diff --git a/README.md b/README.md index 37cbc89..e8320cd 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,71 @@ does not import campaign internals; campaign share/existence checks use the core Platform RBAC and governance rules are documented in `govoplan-core/docs/`. +Managed files can carry source provenance for connector and import workflows. +Upload callers may provide `source_provenance_json` and `source_revision`; the +module stores those values under file metadata, returns normalized +`source_provenance` and `source_revision` fields in file responses, and carries +them into managed campaign attachment matches for frozen execution evidence. + +Connector policy preflight is available through +`POST /api/v1/files/connector-policy/evaluate`, and provenance-bearing uploads +can pass `connector_policy_json` to enforce the same policy before file content +is read. Policy payloads contain ordered `sources`; each source has +`scope_type`, optional `scope_id`, optional `label`, and a `policy` object. The +policy object supports `allow`/`allowlist`/`whitelist` and +`deny`/`denylist`/`blacklist` rules for `connectors`, `providers`, +`external_ids`, `external_paths`, and `external_urls`. Deny rules win across the +hierarchy; allow rules narrow access at each source that defines them. + +Connector endpoint settings are exposed through governed connector profiles. +Profiles can be supplied as JSON through +`GOVOPLAN_FILES_CONNECTOR_PROFILES_JSON`, or from a JSON file path through +`GOVOPLAN_FILES_CONNECTOR_PROFILES_FILE`. Each profile has an `id`, `provider`, +`endpoint_url`, governance `scope_type`/`scope_id`, optional `capabilities`, and +credential references such as `password_env`, `token_env`, or `secret_ref`. +`GET /api/v1/files/connectors/profiles` returns only profiles visible to the +current principal (system, tenant, user, group, or accessible campaign scope) and +redacts secret values and environment variable names. Use the returned +`policy_sources` with connector policy preflight before importing files. +`GET /api/v1/files/connectors/providers` exposes provider descriptors for +Seafile, Nextcloud, WebDAV, SMB, NFS, and local filesystem connectors. The +descriptor declares implementation status, optional dependencies, permission +mapping, sync/index strategy, conflict handling, preview behavior, and audit +events so provider coverage remains visible without forcing every optional +protocol dependency to be installed. +`GET /api/v1/files/connectors/profiles/{profile_id}/browse` provides read-only +connector browsing. Browse entries use a shared `library`/`folder`/`file` shape, +run profile policy with the `browse` operation, and support Seafile, +WebDAV/Nextcloud, and SMB when the optional `smb` extra is installed. Seafile +profiles browse libraries and directories via Seafile's read-only API; profiles +can still opt into WebDAV browsing by setting `metadata.webdav_endpoint_url` or +`metadata.browse_protocol` to `webdav`. +`POST /api/v1/files/connectors/profiles/{profile_id}/import` imports a Seafile +or WebDAV/Nextcloud/SMB file into managed storage through the same governance, +conflict handling, source provenance, and connector audit path as direct +uploads. The Seafile provider uses account-token auth and the native file +download-link API; Nextcloud and generic WebDAV profiles use authenticated `GET` +requests against the configured WebDAV endpoint. SMB profiles use +`smb://server[:port]/share[/path]` endpoints and environment-backed credentials +through `smbprotocol`. + +Local connector development assets live in `dev/connectors/`. The compose stack +boots Nextcloud, Seafile, WebDAV, and SMB endpoints for provider development and +manual interoperability testing. + +Connector and collaboration ownership boundaries are documented in +`docs/CONNECTOR_BOUNDARY.md` and `docs/DOCUMENT_COLLABORATION_BOUNDARY.md`. + +ZIP uploads are processed without buffering the whole archive in memory. The API +spools incoming ZIP request bodies to a bounded temporary file, then extracts +members with per-file and total extracted-size limits before storing managed +files. + +Bulk rename and transfer APIs are owner-scoped: callers must provide the active +user or group file space with `owner_type` and `owner_id`. The storage layer +keeps a named legacy file-only helper for historical callers that lack owner +context, and regression tests cover its write-access checks. + ## Release packaging The repository root includes a `package.json` for git-based WebUI installs. It exports the package `@govoplan/files-webui` from `webui/src` so release builds can depend on tagged git refs instead of local `file:` paths. diff --git a/dev/connectors/.env.example b/dev/connectors/.env.example new file mode 100644 index 0000000..15ed158 --- /dev/null +++ b/dev/connectors/.env.example @@ -0,0 +1,19 @@ +NEXTCLOUD_HOST_PORT=9081 +NEXTCLOUD_DB_ROOT_PASSWORD=govoplan-nextcloud-root +NEXTCLOUD_DB_PASSWORD=govoplan-nextcloud +NEXTCLOUD_ADMIN_USER=admin +NEXTCLOUD_ADMIN_PASSWORD=govoplan-nextcloud-admin + +SEAFILE_HOST_PORT=9082 +SEAFILE_MYSQL_ROOT_PASSWORD=govoplan-seafile-root +SEAFILE_ADMIN_EMAIL=admin@example.local +SEAFILE_ADMIN_PASSWORD=govoplan-seafile-admin + +WEBDAV_HOST_PORT=9083 +WEBDAV_USER=govoplan +WEBDAV_PASSWORD=govoplan-webdav + +SMB_HOST_PORT=1445 +SMB_SHARE_NAME=files +SMB_USER=govoplan +SMB_PASSWORD=govoplan-smb diff --git a/dev/connectors/README.md b/dev/connectors/README.md new file mode 100644 index 0000000..57c73ad --- /dev/null +++ b/dev/connectors/README.md @@ -0,0 +1,130 @@ +# Connector Dev Stack + +This directory keeps local Docker Compose assets for GovOPlaN file connector +development. The stack is intentionally separate from production deployment and +binds services to localhost high ports. + +## Start + +```bash +cd /mnt/DATA/git/govoplan-files/dev/connectors +cp .env.example .env +docker compose up -d nextcloud nextcloud-db webdav smb +docker compose up -d seafile-db seafile-memcached seafile +``` + +If the SMB service was already running before a compose change, recreate it so +the image and share configuration are applied: + +```bash +docker compose rm -sf smb +docker compose up -d --build smb +``` + +Endpoints: + +- Nextcloud: `http://127.0.0.1:9081`, WebDAV root + `http://127.0.0.1:9081/remote.php/dav/files/admin/` +- Seafile: `http://127.0.0.1:9082`, WebDAV root + `http://127.0.0.1:9082/seafdav/` +- WebDAV: `http://127.0.0.1:9083` +- SMB: `smb://127.0.0.1:1445/files` + +The local fixture data under `data/` is ignored by git. + +## GovOPlaN Profile Config + +Connector profiles are read from `GOVOPLAN_FILES_CONNECTOR_PROFILES_JSON` or +`GOVOPLAN_FILES_CONNECTOR_PROFILES_FILE`. Credentials should be referenced by +secret refs or environment variables; profile API responses only expose the +credential source and configured state. + +Example local profile file: + +```json +{ + "profiles": [ + { + "id": "dev-seafile", + "label": "Dev Seafile", + "provider": "seafile", + "endpoint_url": "http://127.0.0.1:9082", + "scope_type": "system", + "credential_mode": "basic", + "username": "admin@example.local", + "password_env": "SEAFILE_ADMIN_PASSWORD", + "capabilities": ["browse", "import"], + "metadata": { "webdav_endpoint_url": "http://127.0.0.1:9082/seafdav/" }, + "policy": { "allow": { "providers": ["seafile"] } } + }, + { + "id": "dev-nextcloud", + "label": "Dev Nextcloud", + "provider": "nextcloud", + "endpoint_url": "http://127.0.0.1:9081/remote.php/dav/files/admin/", + "scope_type": "system", + "credential_mode": "basic", + "username": "admin", + "password_env": "NEXTCLOUD_ADMIN_PASSWORD", + "capabilities": ["browse", "import"], + "policy": { "allow": { "providers": ["nextcloud", "webdav"] } } + }, + { + "id": "dev-webdav", + "label": "Dev WebDAV", + "provider": "webdav", + "endpoint_url": "http://127.0.0.1:9083", + "scope_type": "system", + "credential_mode": "basic", + "username": "govoplan", + "password_env": "WEBDAV_PASSWORD", + "capabilities": ["browse", "import"], + "policy": { "allow": { "providers": ["webdav"] } } + }, + { + "id": "dev-smb", + "label": "Dev SMB", + "provider": "smb", + "endpoint_url": "smb://127.0.0.1:1445/files", + "scope_type": "system", + "credential_mode": "basic", + "username": "govoplan", + "password_env": "SMB_PASSWORD", + "capabilities": ["browse", "import"], + "policy": { "allow": { "providers": ["smb"] } } + } + ] +} +``` + +Start GovOPlaN with: + +```bash +export GOVOPLAN_FILES_CONNECTOR_PROFILES_FILE=/mnt/DATA/git/govoplan-files/dev/connectors/profiles.local.json +``` + +## Smoke Test + +Run the connector helper smoke checks from an environment where +`govoplan-files` and `httpx` are importable: + +```bash +cd /mnt/DATA/git/govoplan-files/dev/connectors +/mnt/DATA/git/govoplan-core/.venv/bin/python smoke.py +``` + +The script reads `.env` from this directory when present, seeds tiny WebDAV, +Nextcloud, and SMB fixtures, then browses and imports them through the connector +helper layer. Pass `--require-smb` after recreating the SMB container to fail on +SMB access errors instead of reporting them as an optional skip. + +SMB smoke checks need the optional Python dependency in the environment running +the script: + +```bash +/mnt/DATA/git/govoplan-core/.venv/bin/python -m pip install -e /mnt/DATA/git/govoplan-files[smb] +``` + +The SMB service is built from `dev/connectors/smb/` so the development share is +deterministic: one `files` share backed by `data/smb`, with the credentials from +`.env`. diff --git a/dev/connectors/data/.gitignore b/dev/connectors/data/.gitignore new file mode 100644 index 0000000..d6b7ef3 --- /dev/null +++ b/dev/connectors/data/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/dev/connectors/docker-compose.yml b/dev/connectors/docker-compose.yml new file mode 100644 index 0000000..4e0a914 --- /dev/null +++ b/dev/connectors/docker-compose.yml @@ -0,0 +1,106 @@ +name: govoplan-files-connectors + +services: + nextcloud-db: + image: mariadb:10.11 + command: --transaction-isolation=READ-COMMITTED --binlog-format=ROW + restart: unless-stopped + environment: + MYSQL_ROOT_PASSWORD: ${NEXTCLOUD_DB_ROOT_PASSWORD:-govoplan-nextcloud-root} + MYSQL_DATABASE: nextcloud + MYSQL_USER: nextcloud + MYSQL_PASSWORD: ${NEXTCLOUD_DB_PASSWORD:-govoplan-nextcloud} + volumes: + - nextcloud-db:/var/lib/mysql + + nextcloud: + image: nextcloud:apache + restart: unless-stopped + depends_on: + - nextcloud-db + ports: + - "127.0.0.1:${NEXTCLOUD_HOST_PORT:-9081}:80" + environment: + MYSQL_HOST: nextcloud-db + MYSQL_DATABASE: nextcloud + MYSQL_USER: nextcloud + MYSQL_PASSWORD: ${NEXTCLOUD_DB_PASSWORD:-govoplan-nextcloud} + NEXTCLOUD_ADMIN_USER: ${NEXTCLOUD_ADMIN_USER:-admin} + NEXTCLOUD_ADMIN_PASSWORD: ${NEXTCLOUD_ADMIN_PASSWORD:-govoplan-nextcloud-admin} + NEXTCLOUD_TRUSTED_DOMAINS: "localhost 127.0.0.1" + volumes: + - nextcloud:/var/www/html + + seafile-db: + image: mariadb:10.11 + restart: unless-stopped + environment: + MYSQL_ROOT_PASSWORD: ${SEAFILE_MYSQL_ROOT_PASSWORD:-govoplan-seafile-root} + MARIADB_AUTO_UPGRADE: "1" + MYSQL_LOG_CONSOLE: "true" + volumes: + - seafile-db:/var/lib/mysql + + seafile-memcached: + image: memcached:1.6 + restart: unless-stopped + command: memcached -m 256 + + seafile: + image: seafileltd/seafile-mc:11.0-latest + restart: unless-stopped + depends_on: + - seafile-db + - seafile-memcached + ports: + - "127.0.0.1:${SEAFILE_HOST_PORT:-9082}:80" + environment: + DB_HOST: seafile-db + DB_ROOT_PASSWD: ${SEAFILE_MYSQL_ROOT_PASSWORD:-govoplan-seafile-root} + TIME_ZONE: Etc/UTC + SEAFILE_ADMIN_EMAIL: ${SEAFILE_ADMIN_EMAIL:-admin@example.local} + SEAFILE_ADMIN_PASSWORD: ${SEAFILE_ADMIN_PASSWORD:-govoplan-seafile-admin} + SEAFILE_SERVER_LETSENCRYPT: "false" + SEAFILE_SERVER_HOSTNAME: "127.0.0.1:${SEAFILE_HOST_PORT:-9082}" + volumes: + - seafile-data:/shared + + webdav: + image: rclone/rclone:latest + restart: unless-stopped + command: + - serve + - webdav + - /data + - --addr + - :8080 + - --user + - ${WEBDAV_USER:-govoplan} + - --pass + - ${WEBDAV_PASSWORD:-govoplan-webdav} + ports: + - "127.0.0.1:${WEBDAV_HOST_PORT:-9083}:8080" + volumes: + - ./data/webdav:/data + + smb: + build: + context: ./smb + image: govoplan-files-samba-dev:latest + restart: unless-stopped + environment: + SMB_SHARE_NAME: ${SMB_SHARE_NAME:-files} + SMB_USER: ${SMB_USER:-govoplan} + SMB_PASSWORD: ${SMB_PASSWORD:-govoplan-smb} + SMB_UID: ${SMB_UID:-1000} + SMB_GID: ${SMB_GID:-1000} + ports: + - "127.0.0.1:${SMB_HOST_PORT:-1445}:445" + volumes: + - ./data/smb:/storage + +volumes: + nextcloud-db: + nextcloud: + seafile-db: + seafile-data: diff --git a/dev/connectors/smb/Dockerfile b/dev/connectors/smb/Dockerfile new file mode 100644 index 0000000..304d5a8 --- /dev/null +++ b/dev/connectors/smb/Dockerfile @@ -0,0 +1,10 @@ +FROM alpine:3.20 + +RUN apk add --no-cache samba-server samba-common-tools + +COPY entrypoint.sh /usr/local/bin/govoplan-samba-entrypoint +RUN chmod +x /usr/local/bin/govoplan-samba-entrypoint + +EXPOSE 445 + +ENTRYPOINT ["/usr/local/bin/govoplan-samba-entrypoint"] diff --git a/dev/connectors/smb/entrypoint.sh b/dev/connectors/smb/entrypoint.sh new file mode 100644 index 0000000..27aa56c --- /dev/null +++ b/dev/connectors/smb/entrypoint.sh @@ -0,0 +1,50 @@ +#!/bin/sh +set -eu + +share_name="${SMB_SHARE_NAME:-files}" +user_name="${SMB_USER:-govoplan}" +password="${SMB_PASSWORD:-govoplan-smb}" +uid="${SMB_UID:-1000}" +gid="${SMB_GID:-1000}" + +mkdir -p /storage /var/lib/samba/private /run/samba + +if ! getent group "$user_name" >/dev/null 2>&1; then + addgroup -g "$gid" "$user_name" +fi + +if ! id "$user_name" >/dev/null 2>&1; then + adduser -D -H -s /sbin/nologin -u "$uid" -G "$user_name" "$user_name" +fi + +chown -R "$user_name:$user_name" /storage + +cat > /etc/samba/smb.conf </dev/null +smbpasswd -e "$user_name" >/dev/null + +exec smbd --foreground --no-process-group --debug-stdout diff --git a/dev/connectors/smoke.py b/dev/connectors/smoke.py new file mode 100644 index 0000000..ec8938e --- /dev/null +++ b/dev/connectors/smoke.py @@ -0,0 +1,215 @@ +from __future__ import annotations + +import argparse +import os +import socket +from pathlib import Path + +import httpx + +from govoplan_files.backend.storage.connector_browse import browse_connector_profile +from govoplan_files.backend.storage.connector_imports import read_connector_file +from govoplan_files.backend.storage.connector_profiles import connector_profiles_from_payload + + +ROOT = Path(__file__).resolve().parent + + +def main() -> int: + parser = argparse.ArgumentParser(description="Smoke-test GovOPlaN connector helpers against the local dev compose stack.") + parser.add_argument("--require-smb", action="store_true", help="Fail if the SMB connector cannot browse/import.") + parser.add_argument("--debug-smb", action="store_true", help="Print direct smbclient probes before the connector smoke check.") + args = parser.parse_args() + + _load_dotenv() + _default_env() + preflight_failures = _preflight_services(require_smb=args.require_smb) + if preflight_failures: + for failure in preflight_failures: + print(f"FAIL {failure}") + print("Start or recreate the connector stack from this directory with: docker compose up -d nextcloud nextcloud-db webdav smb") + return 1 + _seed_webdav_fixture() + _seed_smb_fixture() + try: + _seed_nextcloud_fixture() + except httpx.HTTPError as exc: + print(f"FAIL dev-nextcloud seed failed: {exc}") + return 1 + + profiles = {profile.id: profile for profile in connector_profiles_from_payload({"profiles": _profile_payloads()})} + if args.debug_smb: + _debug_smb(profiles["dev-smb"]) + failures: list[str] = [] + for profile_id, folder, file_path, expected in ( + ("dev-webdav", "GovOPlaN", "GovOPlaN/webdav-live.txt", "webdav live fixture"), + ("dev-nextcloud", "GovOPlaN", "GovOPlaN/nextcloud-live.txt", "nextcloud live fixture"), + ): + try: + _exercise_profile(profiles[profile_id], folder=folder, file_path=file_path, expected=expected) + except Exception as exc: + failures.append(f"{profile_id}: {exc}") + + try: + _exercise_profile(profiles["dev-smb"], folder="GovOPlaN", file_path="GovOPlaN/smb-live.txt", expected="smb live fixture") + except Exception as exc: + message = f"dev-smb: {exc}" + if args.require_smb: + failures.append(message) + else: + print(f"SKIP {message}") + + if failures: + for failure in failures: + print(f"FAIL {failure}") + return 1 + print("OK connector dev stack smoke checks passed") + return 0 + + +def _default_env() -> None: + defaults = { + "NEXTCLOUD_ADMIN_USER": "admin", + "NEXTCLOUD_ADMIN_PASSWORD": "govoplan-nextcloud-admin", + "WEBDAV_USER": "govoplan", + "WEBDAV_PASSWORD": "govoplan-webdav", + "SMB_SHARE_NAME": "files", + "SMB_USER": "govoplan", + "SMB_PASSWORD": "govoplan-smb", + } + for key, value in defaults.items(): + os.environ.setdefault(key, value) + + +def _load_dotenv() -> None: + path = ROOT / ".env" + if not path.exists(): + return + for line in path.read_text(encoding="utf-8").splitlines(): + text = line.strip() + if not text or text.startswith("#") or "=" not in text: + continue + key, value = text.split("=", 1) + key = key.strip() + if not key or key in os.environ: + continue + os.environ[key] = value.strip().strip("\"'") + + +def _preflight_services(*, require_smb: bool) -> list[str]: + failures: list[str] = [] + nextcloud_url = f"http://127.0.0.1:{os.getenv('NEXTCLOUD_HOST_PORT', '9081')}/status.php" + try: + response = httpx.get(nextcloud_url, timeout=3.0) + if response.status_code != 200: + failures.append(f"dev-nextcloud expected HTTP 200 at {nextcloud_url}, got HTTP {response.status_code}") + except httpx.HTTPError as exc: + failures.append(f"dev-nextcloud is not reachable at {nextcloud_url}: {exc}") + + webdav_url = f"http://127.0.0.1:{os.getenv('WEBDAV_HOST_PORT', '9083')}/" + try: + response = httpx.get(webdav_url, timeout=3.0) + if response.status_code not in {200, 401}: + failures.append(f"dev-webdav expected HTTP 200/401 at {webdav_url}, got HTTP {response.status_code}") + except httpx.HTTPError as exc: + failures.append(f"dev-webdav is not reachable at {webdav_url}: {exc}") + + if require_smb: + smb_port = int(os.getenv("SMB_HOST_PORT", "1445")) + try: + with socket.create_connection(("127.0.0.1", smb_port), timeout=3.0): + pass + except OSError as exc: + failures.append(f"dev-smb is not reachable at 127.0.0.1:{smb_port}: {exc}") + return failures + + +def _profile_payloads() -> list[dict[str, object]]: + return [ + { + "id": "dev-webdav", + "provider": "webdav", + "endpoint_url": f"http://127.0.0.1:{os.getenv('WEBDAV_HOST_PORT', '9083')}/", + "credential_mode": "basic", + "username": os.getenv("WEBDAV_USER", "govoplan"), + "password_env": "WEBDAV_PASSWORD", + }, + { + "id": "dev-nextcloud", + "provider": "nextcloud", + "endpoint_url": f"http://127.0.0.1:{os.getenv('NEXTCLOUD_HOST_PORT', '9081')}/remote.php/dav/files/{os.getenv('NEXTCLOUD_ADMIN_USER', 'admin')}/", + "credential_mode": "basic", + "username": os.getenv("NEXTCLOUD_ADMIN_USER", "admin"), + "password_env": "NEXTCLOUD_ADMIN_PASSWORD", + }, + { + "id": "dev-smb", + "provider": "smb", + "endpoint_url": f"smb://127.0.0.1:{os.getenv('SMB_HOST_PORT', '1445')}/{os.getenv('SMB_SHARE_NAME', 'files')}", + "credential_mode": "basic", + "username": os.getenv("SMB_USER", "govoplan"), + "password_env": "SMB_PASSWORD", + }, + ] + + +def _exercise_profile(profile, *, folder: str, file_path: str, expected: str) -> None: + items = browse_connector_profile(profile, path=folder) + paths = {item.path for item in items} + if file_path not in paths: + raise RuntimeError(f"{file_path!r} not found; saw {sorted(paths)!r}") + downloaded = read_connector_file(profile, library_id="", path=file_path, max_bytes=1024 * 1024) + text = downloaded.data.decode("utf-8").strip() + if text != expected: + raise RuntimeError(f"{file_path!r} content mismatch: {text!r}") + print(f"OK {profile.id} browse/import {file_path}") + + +def _debug_smb(profile) -> None: + try: + import smbclient + except ImportError as exc: + print(f"DEBUG smbclient import failed: {exc}") + return + from govoplan_files.backend.storage.connector_browse import _smb_client_kwargs, _smb_location, _smb_unc_path + + location = _smb_location(profile) + kwargs = _smb_client_kwargs(profile, location) + print(f"DEBUG SMB endpoint=//{location.server}:{location.port}/{location.share} root_path={location.root_path!r}") + print(f"DEBUG SMB user={kwargs.get('username')!r} require_signing={kwargs.get('require_signing')!r} auth_protocol={kwargs.get('auth_protocol')!r}") + try: + smbclient.reset_connection_cache() + except Exception as exc: + print(f"DEBUG SMB reset cache failed: {exc}") + for path in (_smb_unc_path(location, ""), _smb_unc_path(location, "GovOPlaN")): + try: + print(f"DEBUG SMB list {path}: {smbclient.listdir(path, **kwargs)}") + except Exception as exc: + print(f"DEBUG SMB list {path} failed: {type(exc).__name__}: {exc}") + + +def _seed_webdav_fixture() -> None: + path = ROOT / "data" / "webdav" / "GovOPlaN" + path.mkdir(parents=True, exist_ok=True) + (path / "webdav-live.txt").write_text("webdav live fixture\n", encoding="utf-8") + + +def _seed_smb_fixture() -> None: + path = ROOT / "data" / "smb" / "GovOPlaN" + path.mkdir(parents=True, exist_ok=True) + (path / "smb-live.txt").write_text("smb live fixture\n", encoding="utf-8") + + +def _seed_nextcloud_fixture() -> None: + base_url = f"http://127.0.0.1:{os.getenv('NEXTCLOUD_HOST_PORT', '9081')}/remote.php/dav/files/{os.getenv('NEXTCLOUD_ADMIN_USER', 'admin')}/GovOPlaN" + auth = (os.getenv("NEXTCLOUD_ADMIN_USER", "admin"), os.getenv("NEXTCLOUD_ADMIN_PASSWORD", "govoplan-nextcloud-admin")) + response = httpx.request("MKCOL", base_url, auth=auth, timeout=15.0) + if response.status_code not in {201, 405}: + raise RuntimeError(f"Nextcloud MKCOL failed with HTTP {response.status_code}: {response.text[:200]}") + response = httpx.put(f"{base_url}/nextcloud-live.txt", content=b"nextcloud live fixture\n", auth=auth, timeout=15.0) + if response.status_code not in {200, 201, 204}: + raise RuntimeError(f"Nextcloud PUT failed with HTTP {response.status_code}: {response.text[:200]}") + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/docs/CONNECTOR_BOUNDARY.md b/docs/CONNECTOR_BOUNDARY.md new file mode 100644 index 0000000..2f17e46 --- /dev/null +++ b/docs/CONNECTOR_BOUNDARY.md @@ -0,0 +1,65 @@ +# File Connector Boundary + +GovOPlaN Files owns the managed-file boundary: frozen blobs, versions, shares, +campaign attachment evidence, provenance, connector policy, connector profiles, +and read-only browse/import APIs that turn external files into managed +GovOPlaN files. + +The built-in connector layer is intentionally on-demand. It does not run remote +indexing, background sync, remote mutation, or upstream permission management. +Connectors may browse an external source, import one selected file, freeze it as +a managed file, record provenance, and audit the access. + +## Built-In Baseline + +The files module may keep small baseline providers when they are needed for +normal product workflows and can share the same governance model: + +- Seafile through the native read/download API +- Nextcloud through WebDAV +- generic WebDAV +- SMB through the optional `smb` extra + +These providers are surfaced through connector descriptors at +`GET /api/v1/files/connectors/providers`. Provider descriptors declare whether +the provider is implemented, whether its optional dependency is installed, and +which browse/import behaviors are available. + +## Separate Connector Module Candidates + +A separate connector module becomes appropriate when integration needs exceed +on-demand browse/import: + +- background indexing or synchronization +- bidirectional remote mutation +- long-running transfer workers +- provider-specific credential lifecycle or OAuth flows +- DMS/eAkte metadata models, registers, retention, or filing plans +- S3-compatible object store administration +- NFS host-mount lifecycle or sidecar coordination +- provider-specific WebUI administration beyond profile fields + +In that model, `govoplan-files` should keep the managed-file import contract, +policy checks, provenance model, and audit events. A connector module should own +provider-specific discovery, synchronization, credentials, health checks, and +any remote write behavior. + +## Provider Responsibilities + +Every provider must: + +- enforce GovOPlaN profile visibility and connector policy before browse/import +- keep credentials as environment variables or secret references, never API + response values +- import external files into managed storage before they are used in campaigns + or workflows +- preserve source provenance and revision metadata +- emit connector audit events for imported or accessed files +- treat remote ACLs as upstream checks, not as a replacement for GovOPlaN policy + +## Non-Goals For Files + +Files does not own collaborative editing, comments, document review workflows, +remote lock orchestration, DMS records management, or external system data +models. Those belong to future document, workflow, DMS, or connector modules and +should integrate through capabilities and managed-file imports. diff --git a/docs/CONNECTOR_SPACES.md b/docs/CONNECTOR_SPACES.md new file mode 100644 index 0000000..65accbf --- /dev/null +++ b/docs/CONNECTOR_SPACES.md @@ -0,0 +1,147 @@ +# Connector Spaces + +GovOPlaN Files should treat external file shares as two separate product +objects: + +1. A governed connection profile defines a reusable remote endpoint such as a + WebDAV server, Nextcloud account, Seafile server, or SMB share. Profiles are + administered in settings at system or tenant scope, with the same + inheritance and limit semantics used by mail-server profiles. +2. A governed credential profile defines reusable authentication material for + one provider or for any compatible provider. Credentials are administered + separately from connection profiles and can carry their own allow/deny + policy. +3. A linked file space binds one concrete remote folder or library path from an + allowed profile to a user or group. Linked spaces appear beside "My files" + and group file spaces in the files module. + +This keeps secrets, endpoint governance, and tenant limits in settings, while +keeping user/group workspaces and concrete folder choices in the files module. + +## Model + +Connection profile: + +- provider: `seafile`, `nextcloud`, `webdav`, `smb`, or a future provider +- endpoint URL and optional base path +- scope: `system` or `tenant` for administered profiles; user/group profiles + may be allowed later only when policy explicitly permits them +- optional credential profile id +- capabilities: browse, sync, import, optional write when a provider supports it +- profile-local policy for allow/deny rules + +Credential profile: + +- provider: a specific provider or any provider +- scope: `system` or `tenant` for administered credentials +- credential mode: anonymous, environment reference, secret reference, or + encrypted stored password/token +- username and redacted secret configuration +- credential-local policy for allow/deny rules + +Connector policy: + +- system policy is the baseline +- tenant policy inherits system policy and may narrow it unless system allows + lower-level relaxation +- future user/group policy may narrow tenant policy for self-service links +- policies can allow or block providers, profile ids, endpoint URLs, external + path prefixes, credential ids, credential inheritance, and local linked-space + creation + +Linked connector space: + +- owner type: user or group +- display name +- connector profile id +- remote library id or share id +- remote root path +- sync mode: manual initially; background sync is future work +- read-only flag from provider/policy +- active/deleted state + +The linked space should be addressable as a normal file space in the files UI. +For the first implementation slice, actions may use the existing connector +browse/sync APIs and managed file storage. A linked space can therefore browse +the remote folder and sync selected files into managed storage. Later slices can +add a native remote listing view or background sync jobs. + +## Policy Semantics + +Use mail-profile terminology because administrators already see it there: + +- must use: lower scopes are restricted to selected profile ids or providers +- can use: lower scopes may choose from inherited allowed profiles +- shall not use anything outside: endpoint URLs and path prefixes are enforced + by deny/allow rules before browse, import, or sync +- may create local links: controls whether users/groups can add linked spaces + from inherited profiles + +Connector policy should provide an explainable effective policy response with +source path entries: system, tenant, user, group, campaign when applicable. + +## Current State + +Implemented: + +- provider descriptors for Seafile, Nextcloud, WebDAV, and SMB +- database-backed connector profiles with system and tenant scope +- database-backed connector credentials with system and tenant scope +- encrypted stored password/token support for database credentials +- JSON/environment-defined connector profiles with system, tenant, user, group, + and campaign visibility +- connector allow/deny policy enforcement before browse/import/sync, including + provider, connection profile, credential profile, and path checks +- read-only browse endpoints +- import and sync into managed files with provenance and revision metadata +- audit events for connector import, sync, and access +- settings/admin UI sections for system and tenant file connections and + credentials +- linked connector-space rows owned by users or groups +- file-space API responses that include linked connector spaces +- Files UI entry point to create linked connector spaces from a browsed remote + profile/folder +- Files UI sync dialog for choosing a profile, browsing a remote folder, and + syncing a selected file into a managed destination folder +- Files UI connector-space view with provider/read-only/manual-sync state +- Docker dev stack smoke checks for WebDAV, Nextcloud, and SMB + +Missing: + +- explicit connector profile policy rows equivalent to mail-profile policies +- effective connector policy/explain UI equivalent to mail-profile policies +- edit/manage actions for existing linked connector spaces +- optional background sync or remote-write semantics + +## Implementation Phases + +1. Persist governed connector profiles and policies. + - `file_connector_profiles` exists for system and tenant profiles. + - `file_connector_credentials` exists for system and tenant credentials. + - Environment JSON profiles remain bootstrap/compatibility profiles. + - Profile and credential CRUD endpoints exist for database-backed records. + - Remaining: `file_connector_policies` plus effective policy/explain output. + +2. Add linked connector spaces. + - `file_connector_spaces` exists with owner user/group, profile id, library + id, remote path, display label, sync mode, and active state. + - Connector policy is enforced before creating or using a link. + - Linked connector spaces are returned from `/api/v1/files/spaces`. + +3. Surface linked spaces in the Files UI. + - Linked spaces appear beside managed user/group spaces. + - The add-space dialog browses an allowed profile and links the current + remote folder/library root. + - The connector browser/sync flow is reused when a linked space is open. + - Remaining: edit/manage actions for existing linked spaces. + +4. Add sync orchestration. + - Manual sync selected file is already available. + - Add folder-level manual sync. + - Add optional scheduled/background sync with conflict reporting. + +5. Add provider-specific expansion. + - OAuth/secret-store credentials. + - Remote write and delete only where policy and provider support it. + - DMS/eAkte metadata integrations in a separate connector or documents + module. diff --git a/docs/DOCUMENT_COLLABORATION_BOUNDARY.md b/docs/DOCUMENT_COLLABORATION_BOUNDARY.md new file mode 100644 index 0000000..3c53cf9 --- /dev/null +++ b/docs/DOCUMENT_COLLABORATION_BOUNDARY.md @@ -0,0 +1,54 @@ +# Document Collaboration Boundary + +GovOPlaN Files is the governed managed-file module. It stores uploaded/imported +blobs, versions, shares, provenance, ZIP imports/exports, connector imports, and +campaign attachment evidence. It should stay reliable, auditable, and simple. + +Collaborative document behavior should be owned by a future documents module or +by provider-specific connector modules when the behavior is delegated to systems +such as Nextcloud, Seafile, Collabora, OnlyOffice, OpenDesk, or DMS/eAkte +platforms. + +## Files Owns + +- managed blobs and immutable version evidence +- owner-scoped user/group file spaces +- shares and access checks for managed files +- connector browse/import into managed storage +- source provenance, source revision, and audit events +- freeze-before-send campaign attachment behavior +- previews generated from managed file versions + +## Documents Or Connectors Should Own + +- co-editing sessions and editor launch URLs +- comments, suggestions, document tasks, and review state +- check-in/check-out, remote locks, and lock timeouts +- semantic document versions beyond blob versions +- template merge flows and generated-document lifecycle +- external DMS metadata, filing plans, record categories, and retention classes +- provider-specific sync state and conflict resolution +- collaborative presence and notification behavior + +## Integration Shape + +The documents layer should integrate with files through stable contracts: + +- create or import a managed file version for evidence points +- attach source provenance when a file came from an external editor or DMS +- ask files for read/download access rather than importing files internals +- emit workflow/audit events when a collaborative document reaches a governed + state such as reviewed, approved, frozen, sent, or archived +- use connector providers for provider-specific browse/import/write behavior + +Files may expose links to a document or connector module, but it should not own +the collaboration state machine. A collaborative document can produce many +working states; GovOPlaN Files should persist the governed snapshots that other +modules can safely use as evidence. + +## Practical Rule + +If a feature is about storing, sharing, importing, downloading, or freezing a +file, it belongs in `govoplan-files`. If a feature is about people jointly +editing, reviewing, locking, commenting on, or routing a document through a +semantic process, it belongs in a documents/workflow/DMS integration module. diff --git a/package.json b/package.json index 82afa27..da4e745 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ ], "peerDependencies": { "@govoplan/core-webui": "^0.1.6", - "lucide-react": "^0.555.0", + "lucide-react": "^1.23.0", "react": "^19.0.0", "react-dom": "^19.0.0", "react-router-dom": "^7.1.1", diff --git a/pyproject.toml b/pyproject.toml index 11a3618..a9b29bb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,12 @@ license = { file = "LICENSE" } authors = [{ name = "GovOPlaN" }] dependencies = [ "govoplan-core>=0.1.6", + "python-multipart>=0.0.31,<1", +] + +[project.optional-dependencies] +smb = [ + "smbprotocol>=1.13", ] [tool.setuptools.packages.find] diff --git a/src/govoplan_files/backend/change_tracking.py b/src/govoplan_files/backend/change_tracking.py new file mode 100644 index 0000000..a0adcd0 --- /dev/null +++ b/src/govoplan_files/backend/change_tracking.py @@ -0,0 +1,199 @@ +from __future__ import annotations + +from datetime import datetime +from typing import Any + +from sqlalchemy import event, inspect +from sqlalchemy.orm import Session as OrmSession + +from govoplan_core.core.change_sequence import record_change +from govoplan_files.backend.db.models import FileAsset, FileFolder, FileShare, new_uuid + +FILES_MODULE_ID = "files" +FILES_ASSETS_COLLECTION = "files.assets" +FILES_FOLDERS_COLLECTION = "files.folders" +FILES_CONNECTOR_PROFILES_COLLECTION = "files.connector_profiles" +FILES_CONNECTOR_CREDENTIALS_COLLECTION = "files.connector_credentials" +FILES_CONNECTOR_POLICIES_COLLECTION = "files.connector_policies" +FILES_CONNECTOR_SPACES_COLLECTION = "files.connector_spaces" + +_REGISTERED = False + + +def register_files_change_tracking() -> None: + global _REGISTERED + if _REGISTERED: + return + event.listen(OrmSession, "before_flush", _record_files_changes) + _REGISTERED = True + + +def _record_files_changes(session: OrmSession, _flush_context: object, _instances: object) -> None: + for obj in tuple(session.new) + tuple(session.dirty): + if isinstance(obj, FileAsset): + _record_asset_change(session, obj) + elif isinstance(obj, FileFolder): + _record_folder_change(session, obj) + elif isinstance(obj, FileShare): + _record_share_visibility_change(session, obj) + + +def _record_asset_change(session: OrmSession, asset: FileAsset) -> None: + operation = _operation_for_soft_deletable( + asset, + changed_attrs=( + "owner_type", + "owner_user_id", + "owner_group_id", + "current_version_id", + "display_path", + "filename", + "description", + "deleted_at", + "metadata_", + ), + ) + if operation is None: + return + resource_id = _ensure_id(asset) + record_change( + session, + module_id=FILES_MODULE_ID, + collection=FILES_ASSETS_COLLECTION, + resource_type="file", + resource_id=resource_id, + operation=operation, + tenant_id=asset.tenant_id, + actor_type="user" if asset.created_by_user_id else None, + actor_id=asset.created_by_user_id, + payload={ + "owner_type": asset.owner_type, + "owner_id": _owner_id(asset.owner_type, asset.owner_user_id, asset.owner_group_id), + "path": asset.display_path, + "previous_path": _previous_value(asset, "display_path"), + "filename": asset.filename, + "deleted_at": _isoformat(asset.deleted_at), + }, + ) + + +def _record_folder_change(session: OrmSession, folder: FileFolder) -> None: + operation = _operation_for_soft_deletable( + folder, + changed_attrs=("owner_type", "owner_user_id", "owner_group_id", "path", "deleted_at", "metadata_"), + ) + if operation is None: + return + resource_id = _ensure_id(folder) + record_change( + session, + module_id=FILES_MODULE_ID, + collection=FILES_FOLDERS_COLLECTION, + resource_type="folder", + resource_id=resource_id, + operation=operation, + tenant_id=folder.tenant_id, + actor_type="user" if folder.created_by_user_id else None, + actor_id=folder.created_by_user_id, + payload={ + "owner_type": folder.owner_type, + "owner_id": _owner_id(folder.owner_type, folder.owner_user_id, folder.owner_group_id), + "path": folder.path, + "previous_path": _previous_value(folder, "path"), + "deleted_at": _isoformat(folder.deleted_at), + }, + ) + + +def _record_share_visibility_change(session: OrmSession, share: FileShare) -> None: + state = inspect(share) + if not share.file_asset_id: + return + if not state.pending and not _has_attr_changes( + state, + ("file_asset_id", "target_type", "target_id", "permission", "revoked_at"), + ): + return + _ensure_id(share) + record_change( + session, + module_id=FILES_MODULE_ID, + collection=FILES_ASSETS_COLLECTION, + resource_type="file", + resource_id=share.file_asset_id, + operation="updated", + tenant_id=share.tenant_id, + actor_type="user" if share.created_by_user_id else None, + actor_id=share.created_by_user_id, + payload={ + "share_id": share.id, + "share_target_type": share.target_type, + "share_target_id": share.target_id, + "share_permission": share.permission, + "share_revoked_at": _isoformat(share.revoked_at), + }, + ) + + +def _operation_for_soft_deletable(obj: object, *, changed_attrs: tuple[str, ...]) -> str | None: + state = inspect(obj) + if state.pending: + return "created" + if not _has_attr_changes(state, changed_attrs): + return None + if "deleted_at" in state.attrs: + history = state.attrs.deleted_at.history + if history.has_changes(): + if any(value is not None for value in history.added): + return "deleted" + if any(value is not None for value in history.deleted): + return "created" + return "updated" + + +def _has_attr_changes(state: Any, attrs: tuple[str, ...]) -> bool: + return any(name in state.attrs and state.attrs[name].history.has_changes() for name in attrs) + + +def _previous_value(obj: object, attr_name: str) -> str | None: + state = inspect(obj) + if attr_name not in state.attrs: + return None + history = state.attrs[attr_name].history + if not history.has_changes() or not history.deleted: + return None + value = history.deleted[0] + return str(value) if value is not None else None + + +def _ensure_id(obj: object) -> str: + resource_id = getattr(obj, "id", None) + if resource_id: + return str(resource_id) + resource_id = new_uuid() + setattr(obj, "id", resource_id) + return resource_id + + +def _owner_id(owner_type: str, owner_user_id: str | None, owner_group_id: str | None) -> str | None: + if owner_type == "user": + return owner_user_id + if owner_type == "group": + return owner_group_id + return None + + +def _isoformat(value: datetime | None) -> str | None: + return value.isoformat() if value else None + + +__all__ = [ + "FILES_ASSETS_COLLECTION", + "FILES_CONNECTOR_CREDENTIALS_COLLECTION", + "FILES_CONNECTOR_POLICIES_COLLECTION", + "FILES_CONNECTOR_PROFILES_COLLECTION", + "FILES_CONNECTOR_SPACES_COLLECTION", + "FILES_FOLDERS_COLLECTION", + "FILES_MODULE_ID", + "register_files_change_tracking", +] diff --git a/src/govoplan_files/backend/db/models.py b/src/govoplan_files/backend/db/models.py index 69c2c94..90e572e 100644 --- a/src/govoplan_files/backend/db/models.py +++ b/src/govoplan_files/backend/db/models.py @@ -4,7 +4,7 @@ import uuid from datetime import datetime from typing import Any -from sqlalchemy import DateTime, ForeignKey, Index, Integer, JSON, String, Text, UniqueConstraint, text +from sqlalchemy import Boolean, DateTime, ForeignKey, Index, Integer, JSON, String, Text, UniqueConstraint, text from sqlalchemy.orm import Mapped, mapped_column from govoplan_core.db.base import Base, TimestampMixin @@ -19,7 +19,7 @@ class FileBlob(Base, TimestampMixin): __table_args__ = (UniqueConstraint("tenant_id", "checksum_sha256", "size_bytes", name="uq_file_blobs_tenant_checksum_size"),) id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid) - tenant_id: Mapped[str] = mapped_column(ForeignKey("tenants.id", ondelete="CASCADE"), nullable=False, index=True) + tenant_id: Mapped[str] = mapped_column(ForeignKey("tenancy_tenants.id", ondelete="CASCADE"), nullable=False, index=True) storage_backend: Mapped[str] = mapped_column(String(50), nullable=False) storage_bucket: Mapped[str | None] = mapped_column(String(255)) storage_key: Mapped[str] = mapped_column(String(1000), nullable=False) @@ -50,12 +50,12 @@ class FileFolder(Base, TimestampMixin): ) id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid) - tenant_id: Mapped[str] = mapped_column(ForeignKey("tenants.id", ondelete="CASCADE"), nullable=False, index=True) + tenant_id: Mapped[str] = mapped_column(ForeignKey("tenancy_tenants.id", ondelete="CASCADE"), nullable=False, index=True) owner_type: Mapped[str] = mapped_column(String(20), nullable=False, index=True) - owner_user_id: Mapped[str | None] = mapped_column(ForeignKey("users.id", ondelete="SET NULL"), nullable=True, index=True) - owner_group_id: Mapped[str | None] = mapped_column(ForeignKey("groups.id", ondelete="SET NULL"), nullable=True, index=True) + owner_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True) + owner_group_id: Mapped[str | None] = mapped_column(ForeignKey("access_groups.id", ondelete="SET NULL"), nullable=True, index=True) path: Mapped[str] = mapped_column(String(1000), nullable=False, index=True) - created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("users.id", ondelete="SET NULL"), nullable=True, index=True) + created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True) deleted_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True) metadata_: Mapped[dict[str, Any] | None] = mapped_column("metadata", JSON, nullable=True) @@ -64,15 +64,15 @@ class FileAsset(Base, TimestampMixin): __tablename__ = "file_assets" id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid) - tenant_id: Mapped[str] = mapped_column(ForeignKey("tenants.id", ondelete="CASCADE"), nullable=False, index=True) + tenant_id: Mapped[str] = mapped_column(ForeignKey("tenancy_tenants.id", ondelete="CASCADE"), nullable=False, index=True) owner_type: Mapped[str] = mapped_column(String(20), nullable=False, index=True) - owner_user_id: Mapped[str | None] = mapped_column(ForeignKey("users.id", ondelete="SET NULL"), nullable=True, index=True) - owner_group_id: Mapped[str | None] = mapped_column(ForeignKey("groups.id", ondelete="SET NULL"), nullable=True, index=True) + owner_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True) + owner_group_id: Mapped[str | None] = mapped_column(ForeignKey("access_groups.id", ondelete="SET NULL"), nullable=True, index=True) current_version_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True) display_path: Mapped[str] = mapped_column(String(1000), nullable=False, index=True) filename: Mapped[str] = mapped_column(String(500), nullable=False, index=True) description: Mapped[str | None] = mapped_column(Text) - created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("users.id", ondelete="SET NULL"), nullable=True, index=True) + created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True) deleted_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True) metadata_: Mapped[dict[str, Any] | None] = mapped_column("metadata", JSON, nullable=True) @@ -82,7 +82,7 @@ class FileVersion(Base, TimestampMixin): __table_args__ = (UniqueConstraint("file_asset_id", "version_number", name="uq_file_versions_asset_number"),) id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid) - tenant_id: Mapped[str] = mapped_column(ForeignKey("tenants.id", ondelete="CASCADE"), nullable=False, index=True) + tenant_id: Mapped[str] = mapped_column(ForeignKey("tenancy_tenants.id", ondelete="CASCADE"), nullable=False, index=True) file_asset_id: Mapped[str] = mapped_column(ForeignKey("file_assets.id", ondelete="CASCADE"), nullable=False, index=True) blob_id: Mapped[str] = mapped_column(ForeignKey("file_blobs.id", ondelete="RESTRICT"), nullable=False, index=True) version_number: Mapped[int] = mapped_column(Integer, nullable=False) @@ -91,7 +91,7 @@ class FileVersion(Base, TimestampMixin): content_type: Mapped[str | None] = mapped_column(String(255)) size_bytes: Mapped[int] = mapped_column(Integer, nullable=False) checksum_sha256: Mapped[str] = mapped_column(String(64), nullable=False, index=True) - created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("users.id", ondelete="SET NULL"), nullable=True, index=True) + created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True) class FileShare(Base, TimestampMixin): @@ -99,21 +99,131 @@ class FileShare(Base, TimestampMixin): __table_args__ = (UniqueConstraint("file_asset_id", "target_type", "target_id", "revoked_at", name="uq_file_shares_active_target"),) id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid) - tenant_id: Mapped[str] = mapped_column(ForeignKey("tenants.id", ondelete="CASCADE"), nullable=False, index=True) + tenant_id: Mapped[str] = mapped_column(ForeignKey("tenancy_tenants.id", ondelete="CASCADE"), nullable=False, index=True) file_asset_id: Mapped[str] = mapped_column(ForeignKey("file_assets.id", ondelete="CASCADE"), nullable=False, index=True) target_type: Mapped[str] = mapped_column(String(20), nullable=False, index=True) target_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True) permission: Mapped[str] = mapped_column(String(20), default="read", nullable=False) - created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("users.id", ondelete="SET NULL"), nullable=True, index=True) + created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True) revoked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True) +class FileConnectorProfile(Base, TimestampMixin): + __tablename__ = "file_connector_profiles" + __table_args__ = ( + Index("ix_file_connector_profiles_scope", "scope_type", "scope_id"), + ) + + id: Mapped[str] = mapped_column(String(255), primary_key=True) + tenant_id: Mapped[str | None] = mapped_column(ForeignKey("tenancy_tenants.id", ondelete="CASCADE"), nullable=True, index=True) + scope_type: Mapped[str] = mapped_column(String(20), default="tenant", nullable=False, index=True) + scope_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True) + label: Mapped[str] = mapped_column(String(255), nullable=False) + provider: Mapped[str] = mapped_column(String(50), nullable=False, index=True) + endpoint_url: Mapped[str | None] = mapped_column(String(1000), nullable=True) + base_path: Mapped[str | None] = mapped_column(String(1000), nullable=True) + enabled: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False, index=True) + credential_profile_id: Mapped[str | None] = mapped_column(String(255), nullable=True, index=True) + credential_mode: Mapped[str] = mapped_column(String(30), default="none", nullable=False) + username: Mapped[str | None] = mapped_column(String(320), nullable=True) + password_encrypted: Mapped[str | None] = mapped_column(Text, nullable=True) + token_encrypted: Mapped[str | None] = mapped_column(Text, nullable=True) + password_env: Mapped[str | None] = mapped_column(String(255), nullable=True) + token_env: Mapped[str | None] = mapped_column(String(255), nullable=True) + secret_ref: Mapped[str | None] = mapped_column(String(1000), nullable=True) + capabilities: Mapped[list[str] | None] = mapped_column(JSON, nullable=True) + policy: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True) + metadata_: Mapped[dict[str, Any] | None] = mapped_column("metadata", JSON, nullable=True) + created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True) + updated_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True) + + +class FileConnectorCredential(Base, TimestampMixin): + __tablename__ = "file_connector_credentials" + __table_args__ = ( + Index("ix_file_connector_credentials_scope", "scope_type", "scope_id"), + ) + + id: Mapped[str] = mapped_column(String(255), primary_key=True) + tenant_id: Mapped[str | None] = mapped_column(ForeignKey("tenancy_tenants.id", ondelete="CASCADE"), nullable=True, index=True) + scope_type: Mapped[str] = mapped_column(String(20), default="tenant", nullable=False, index=True) + scope_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True) + label: Mapped[str] = mapped_column(String(255), nullable=False) + provider: Mapped[str | None] = mapped_column(String(50), nullable=True, index=True) + enabled: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False, index=True) + credential_mode: Mapped[str] = mapped_column(String(30), default="none", nullable=False) + username: Mapped[str | None] = mapped_column(String(320), nullable=True) + password_encrypted: Mapped[str | None] = mapped_column(Text, nullable=True) + token_encrypted: Mapped[str | None] = mapped_column(Text, nullable=True) + password_env: Mapped[str | None] = mapped_column(String(255), nullable=True) + token_env: Mapped[str | None] = mapped_column(String(255), nullable=True) + secret_ref: Mapped[str | None] = mapped_column(String(1000), nullable=True) + policy: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True) + metadata_: Mapped[dict[str, Any] | None] = mapped_column("metadata", JSON, nullable=True) + created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True) + updated_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True) + + +class FileConnectorPolicy(Base, TimestampMixin): + __tablename__ = "file_connector_policies" + __table_args__ = ( + UniqueConstraint("tenant_id", "scope_type", "scope_id", name="uq_file_connector_policies_scope"), + Index("ix_file_connector_policies_scope", "scope_type", "scope_id"), + ) + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid) + tenant_id: Mapped[str | None] = mapped_column(ForeignKey("tenancy_tenants.id", ondelete="CASCADE"), nullable=True, index=True) + scope_type: Mapped[str] = mapped_column(String(20), nullable=False, index=True) + scope_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True) + policy: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict, nullable=False) + created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True) + updated_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True) + + +class FileConnectorSpace(Base, TimestampMixin): + __tablename__ = "file_connector_spaces" + __table_args__ = ( + Index("ix_file_connector_spaces_owner", "tenant_id", "owner_type", "owner_user_id", "owner_group_id"), + Index( + "uq_file_connector_spaces_active_user_label", + "tenant_id", "owner_user_id", "label", + unique=True, + sqlite_where=text("owner_type = 'user' AND deleted_at IS NULL"), + postgresql_where=text("owner_type = 'user' AND deleted_at IS NULL"), + ), + Index( + "uq_file_connector_spaces_active_group_label", + "tenant_id", "owner_group_id", "label", + unique=True, + sqlite_where=text("owner_type = 'group' AND deleted_at IS NULL"), + postgresql_where=text("owner_type = 'group' AND deleted_at IS NULL"), + ), + ) + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid) + tenant_id: Mapped[str] = mapped_column(ForeignKey("tenancy_tenants.id", ondelete="CASCADE"), nullable=False, index=True) + owner_type: Mapped[str] = mapped_column(String(20), nullable=False, index=True) + owner_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True) + owner_group_id: Mapped[str | None] = mapped_column(ForeignKey("access_groups.id", ondelete="SET NULL"), nullable=True, index=True) + label: Mapped[str] = mapped_column(String(255), nullable=False) + connector_profile_id: Mapped[str] = mapped_column(String(255), nullable=False, index=True) + provider: Mapped[str] = mapped_column(String(50), nullable=False, index=True) + library_id: Mapped[str | None] = mapped_column(String(255), nullable=True) + remote_path: Mapped[str] = mapped_column(String(1000), default="", nullable=False) + sync_mode: Mapped[str] = mapped_column(String(30), default="manual", nullable=False) + read_only: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False) + is_active: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False, index=True) + created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True) + deleted_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True) + metadata_: Mapped[dict[str, Any] | None] = mapped_column("metadata", JSON, nullable=True) + + class CampaignAttachmentUse(Base, TimestampMixin): __tablename__ = "campaign_attachment_uses" __table_args__ = (UniqueConstraint("campaign_job_id", "file_version_id", "filename_used", "use_stage", name="uq_campaign_attachment_uses_job_file_stage"),) id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid) - tenant_id: Mapped[str] = mapped_column(ForeignKey("tenants.id", ondelete="CASCADE"), nullable=False, index=True) + tenant_id: Mapped[str] = mapped_column(ForeignKey("tenancy_tenants.id", ondelete="CASCADE"), nullable=False, index=True) campaign_id: Mapped[str] = mapped_column(ForeignKey("campaigns.id", ondelete="CASCADE"), nullable=False, index=True) campaign_version_id: Mapped[str] = mapped_column(ForeignKey("campaign_versions.id", ondelete="CASCADE"), nullable=False, index=True) campaign_job_id: Mapped[str | None] = mapped_column(ForeignKey("campaign_jobs.id", ondelete="SET NULL"), nullable=True, index=True) @@ -134,6 +244,10 @@ __all__ = [ "CampaignAttachmentUse", "FileAsset", "FileBlob", + "FileConnectorCredential", + "FileConnectorPolicy", + "FileConnectorProfile", + "FileConnectorSpace", "FileFolder", "FileShare", "FileVersion", diff --git a/src/govoplan_files/backend/manifest.py b/src/govoplan_files/backend/manifest.py index 55b6e49..d461992 100644 --- a/src/govoplan_files/backend/manifest.py +++ b/src/govoplan_files/backend/manifest.py @@ -5,8 +5,11 @@ from pathlib import Path from govoplan_core.core.module_guards import drop_table_retirement_provider, persistent_table_uninstall_guard from govoplan_core.core.modules import FrontendModule, MigrationSpec, ModuleContext, ModuleManifest, NavItem, PermissionDefinition, RoleTemplate from govoplan_core.db.base import Base +from govoplan_files.backend.change_tracking import register_files_change_tracking from govoplan_files.backend.db import models as file_models # noqa: F401 - populate Files ORM metadata +register_files_change_tracking() + def _permission(scope: str, label: str, description: str) -> PermissionDefinition: module_id, resource, action = scope.split(":", 2) @@ -56,23 +59,33 @@ ROLE_TEMPLATES = ( def _tenant_summary(session, tenant_id: str) -> dict[str, int]: - from govoplan_files.backend.db.models import FileAsset + from govoplan_files.backend.db.models import FileAsset, FileConnectorCredential, FileConnectorPolicy, FileConnectorProfile, FileConnectorSpace - return {"files": session.query(FileAsset).filter(FileAsset.tenant_id == tenant_id).count()} + return { + "files": session.query(FileAsset).filter(FileAsset.tenant_id == tenant_id).count(), + "connector_credentials": session.query(FileConnectorCredential).filter(FileConnectorCredential.tenant_id == tenant_id).count(), + "connector_policies": session.query(FileConnectorPolicy).filter(FileConnectorPolicy.tenant_id == tenant_id).count(), + "connector_profiles": session.query(FileConnectorProfile).filter(FileConnectorProfile.tenant_id == tenant_id).count(), + "connector_spaces": session.query(FileConnectorSpace).filter(FileConnectorSpace.tenant_id == tenant_id).count(), + } def _veto_group_delete(session, tenant_id: str, group_id: str) -> None: - from govoplan_files.backend.db.models import FileAsset, FileFolder, FileShare + from govoplan_files.backend.db.models import FileAsset, FileConnectorSpace, FileFolder, FileShare owned_asset_count = session.query(FileAsset).filter(FileAsset.tenant_id == tenant_id, FileAsset.owner_group_id == group_id).count() owned_folder_count = session.query(FileFolder).filter(FileFolder.tenant_id == tenant_id, FileFolder.owner_group_id == group_id).count() + owned_connector_space_count = session.query(FileConnectorSpace).filter( + FileConnectorSpace.tenant_id == tenant_id, + FileConnectorSpace.owner_group_id == group_id, + ).count() shared_asset_count = session.query(FileShare).filter( FileShare.tenant_id == tenant_id, FileShare.target_type == "group", FileShare.target_id == group_id, ).count() - if owned_asset_count or owned_folder_count or shared_asset_count: - raise ValueError("Cannot remove the group while it owns files, folders or file shares.") + if owned_asset_count or owned_folder_count or owned_connector_space_count or shared_asset_count: + raise ValueError("Cannot remove the group while it owns files, folders, connector spaces or file shares.") def _files_router(context: ModuleContext): @@ -112,6 +125,10 @@ manifest = ModuleManifest( file_models.FileAsset, file_models.FileVersion, file_models.FileShare, + file_models.FileConnectorCredential, + file_models.FileConnectorPolicy, + file_models.FileConnectorProfile, + file_models.FileConnectorSpace, file_models.CampaignAttachmentUse, label="Files", ), @@ -124,6 +141,10 @@ manifest = ModuleManifest( file_models.FileAsset, file_models.FileVersion, file_models.FileShare, + file_models.FileConnectorCredential, + file_models.FileConnectorPolicy, + file_models.FileConnectorProfile, + file_models.FileConnectorSpace, file_models.CampaignAttachmentUse, label="Files", ), diff --git a/src/govoplan_files/backend/migrations/versions/4f5a6b7c8d9e_file_connector_spaces.py b/src/govoplan_files/backend/migrations/versions/4f5a6b7c8d9e_file_connector_spaces.py new file mode 100644 index 0000000..ade96a1 --- /dev/null +++ b/src/govoplan_files/backend/migrations/versions/4f5a6b7c8d9e_file_connector_spaces.py @@ -0,0 +1,114 @@ +"""file connector spaces + +Revision ID: 4f5a6b7c8d9e +Revises: 2e3f4a5b6c7d +Create Date: 2026-07-08 00:00:00.000000 +""" +from __future__ import annotations + +from alembic import op +import sqlalchemy as sa + + +revision = "4f5a6b7c8d9e" +down_revision = "2e3f4a5b6c7d" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + inspector = sa.inspect(op.get_bind()) + tables = set(inspector.get_table_names()) + if "file_connector_spaces" not in tables: + op.create_table( + "file_connector_spaces", + sa.Column("id", sa.String(length=36), nullable=False), + sa.Column("tenant_id", sa.String(length=36), nullable=False), + sa.Column("owner_type", sa.String(length=20), nullable=False), + sa.Column("owner_user_id", sa.String(length=36), nullable=True), + sa.Column("owner_group_id", sa.String(length=36), nullable=True), + sa.Column("label", sa.String(length=255), nullable=False), + sa.Column("connector_profile_id", sa.String(length=255), nullable=False), + sa.Column("provider", sa.String(length=50), nullable=False), + sa.Column("library_id", sa.String(length=255), nullable=True), + sa.Column("remote_path", sa.String(length=1000), nullable=False), + sa.Column("sync_mode", sa.String(length=30), nullable=False), + sa.Column("read_only", sa.Boolean(), nullable=False), + sa.Column("is_active", sa.Boolean(), nullable=False), + sa.Column("created_by_user_id", sa.String(length=36), nullable=True), + sa.Column("deleted_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("metadata", sa.JSON(), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(["created_by_user_id"], ["access_users.id"], name=op.f("fk_file_connector_spaces_created_by_user_id_users"), ondelete="SET NULL"), + sa.ForeignKeyConstraint(["owner_group_id"], ["access_groups.id"], name=op.f("fk_file_connector_spaces_owner_group_id_groups"), ondelete="SET NULL"), + sa.ForeignKeyConstraint(["owner_user_id"], ["access_users.id"], name=op.f("fk_file_connector_spaces_owner_user_id_users"), ondelete="SET NULL"), + sa.ForeignKeyConstraint(["tenant_id"], ["tenancy_tenants.id"], name=op.f("fk_file_connector_spaces_tenant_id_tenants"), ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id", name=op.f("pk_file_connector_spaces")), + ) + + inspector = sa.inspect(op.get_bind()) + indexes = {item["name"] for item in inspector.get_indexes("file_connector_spaces")} + for column in ( + "tenant_id", + "owner_type", + "owner_user_id", + "owner_group_id", + "connector_profile_id", + "provider", + "is_active", + "created_by_user_id", + "deleted_at", + ): + name = op.f(f"ix_file_connector_spaces_{column}") + if name not in indexes: + op.create_index(name, "file_connector_spaces", [column], unique=False) + if "ix_file_connector_spaces_owner" not in indexes: + op.create_index( + "ix_file_connector_spaces_owner", + "file_connector_spaces", + ["tenant_id", "owner_type", "owner_user_id", "owner_group_id"], + unique=False, + ) + if "uq_file_connector_spaces_active_user_label" not in indexes: + op.create_index( + "uq_file_connector_spaces_active_user_label", + "file_connector_spaces", + ["tenant_id", "owner_user_id", "label"], + unique=True, + sqlite_where=sa.text("owner_type = 'user' AND deleted_at IS NULL"), + postgresql_where=sa.text("owner_type = 'user' AND deleted_at IS NULL"), + ) + if "uq_file_connector_spaces_active_group_label" not in indexes: + op.create_index( + "uq_file_connector_spaces_active_group_label", + "file_connector_spaces", + ["tenant_id", "owner_group_id", "label"], + unique=True, + sqlite_where=sa.text("owner_type = 'group' AND deleted_at IS NULL"), + postgresql_where=sa.text("owner_type = 'group' AND deleted_at IS NULL"), + ) + + +def downgrade() -> None: + inspector = sa.inspect(op.get_bind()) + if "file_connector_spaces" not in inspector.get_table_names(): + return + indexes = {item["name"] for item in inspector.get_indexes("file_connector_spaces")} + for name in ( + "uq_file_connector_spaces_active_group_label", + "uq_file_connector_spaces_active_user_label", + "ix_file_connector_spaces_owner", + op.f("ix_file_connector_spaces_deleted_at"), + op.f("ix_file_connector_spaces_created_by_user_id"), + op.f("ix_file_connector_spaces_is_active"), + op.f("ix_file_connector_spaces_provider"), + op.f("ix_file_connector_spaces_connector_profile_id"), + op.f("ix_file_connector_spaces_owner_group_id"), + op.f("ix_file_connector_spaces_owner_user_id"), + op.f("ix_file_connector_spaces_owner_type"), + op.f("ix_file_connector_spaces_tenant_id"), + ): + if name in indexes: + op.drop_index(name, table_name="file_connector_spaces") + op.drop_table("file_connector_spaces") diff --git a/src/govoplan_files/backend/migrations/versions/5a6b7c8d9e0f_file_connector_profiles.py b/src/govoplan_files/backend/migrations/versions/5a6b7c8d9e0f_file_connector_profiles.py new file mode 100644 index 0000000..1172a4c --- /dev/null +++ b/src/govoplan_files/backend/migrations/versions/5a6b7c8d9e0f_file_connector_profiles.py @@ -0,0 +1,88 @@ +"""file connector profiles + +Revision ID: 5a6b7c8d9e0f +Revises: 4f5a6b7c8d9e +Create Date: 2026-07-08 00:00:00.000000 +""" +from __future__ import annotations + +from alembic import op +import sqlalchemy as sa + + +revision = "5a6b7c8d9e0f" +down_revision = "4f5a6b7c8d9e" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + inspector = sa.inspect(op.get_bind()) + if "file_connector_profiles" not in inspector.get_table_names(): + op.create_table( + "file_connector_profiles", + sa.Column("id", sa.String(length=255), nullable=False), + sa.Column("tenant_id", sa.String(length=36), nullable=True), + sa.Column("scope_type", sa.String(length=20), nullable=False), + sa.Column("scope_id", sa.String(length=36), nullable=True), + sa.Column("label", sa.String(length=255), nullable=False), + sa.Column("provider", sa.String(length=50), nullable=False), + sa.Column("endpoint_url", sa.String(length=1000), nullable=True), + sa.Column("base_path", sa.String(length=1000), nullable=True), + sa.Column("enabled", sa.Boolean(), nullable=False), + sa.Column("credential_mode", sa.String(length=30), nullable=False), + sa.Column("username", sa.String(length=320), nullable=True), + sa.Column("password_encrypted", sa.Text(), nullable=True), + sa.Column("token_encrypted", sa.Text(), nullable=True), + sa.Column("password_env", sa.String(length=255), nullable=True), + sa.Column("token_env", sa.String(length=255), nullable=True), + sa.Column("secret_ref", sa.String(length=1000), nullable=True), + sa.Column("capabilities", sa.JSON(), nullable=True), + sa.Column("policy", sa.JSON(), nullable=True), + sa.Column("metadata", sa.JSON(), nullable=True), + sa.Column("created_by_user_id", sa.String(length=36), nullable=True), + sa.Column("updated_by_user_id", sa.String(length=36), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(["created_by_user_id"], ["access_users.id"], name=op.f("fk_file_connector_profiles_created_by_user_id_users"), ondelete="SET NULL"), + sa.ForeignKeyConstraint(["tenant_id"], ["tenancy_tenants.id"], name=op.f("fk_file_connector_profiles_tenant_id_tenants"), ondelete="CASCADE"), + sa.ForeignKeyConstraint(["updated_by_user_id"], ["access_users.id"], name=op.f("fk_file_connector_profiles_updated_by_user_id_users"), ondelete="SET NULL"), + sa.PrimaryKeyConstraint("id", name=op.f("pk_file_connector_profiles")), + ) + + inspector = sa.inspect(op.get_bind()) + indexes = {item["name"] for item in inspector.get_indexes("file_connector_profiles")} + for column in ( + "tenant_id", + "scope_type", + "scope_id", + "provider", + "enabled", + "created_by_user_id", + "updated_by_user_id", + ): + name = op.f(f"ix_file_connector_profiles_{column}") + if name not in indexes: + op.create_index(name, "file_connector_profiles", [column], unique=False) + if "ix_file_connector_profiles_scope" not in indexes: + op.create_index("ix_file_connector_profiles_scope", "file_connector_profiles", ["scope_type", "scope_id"], unique=False) + + +def downgrade() -> None: + inspector = sa.inspect(op.get_bind()) + if "file_connector_profiles" not in inspector.get_table_names(): + return + indexes = {item["name"] for item in inspector.get_indexes("file_connector_profiles")} + for name in ( + "ix_file_connector_profiles_scope", + op.f("ix_file_connector_profiles_updated_by_user_id"), + op.f("ix_file_connector_profiles_created_by_user_id"), + op.f("ix_file_connector_profiles_enabled"), + op.f("ix_file_connector_profiles_provider"), + op.f("ix_file_connector_profiles_scope_id"), + op.f("ix_file_connector_profiles_scope_type"), + op.f("ix_file_connector_profiles_tenant_id"), + ): + if name in indexes: + op.drop_index(name, table_name="file_connector_profiles") + op.drop_table("file_connector_profiles") diff --git a/src/govoplan_files/backend/migrations/versions/6b7c8d9e0f1a_file_connector_credentials.py b/src/govoplan_files/backend/migrations/versions/6b7c8d9e0f1a_file_connector_credentials.py new file mode 100644 index 0000000..5323454 --- /dev/null +++ b/src/govoplan_files/backend/migrations/versions/6b7c8d9e0f1a_file_connector_credentials.py @@ -0,0 +1,106 @@ +"""file connector credentials + +Revision ID: 6b7c8d9e0f1a +Revises: 5a6b7c8d9e0f +Create Date: 2026-07-08 00:00:00.000000 +""" +from __future__ import annotations + +from alembic import op +import sqlalchemy as sa + + +revision = "6b7c8d9e0f1a" +down_revision = "5a6b7c8d9e0f" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + inspector = sa.inspect(op.get_bind()) + table_names = inspector.get_table_names() + + if "file_connector_profiles" in table_names: + columns = {item["name"] for item in inspector.get_columns("file_connector_profiles")} + if "credential_profile_id" not in columns: + op.add_column("file_connector_profiles", sa.Column("credential_profile_id", sa.String(length=255), nullable=True)) + indexes = {item["name"] for item in inspector.get_indexes("file_connector_profiles")} + index_name = op.f("ix_file_connector_profiles_credential_profile_id") + if index_name not in indexes: + op.create_index(index_name, "file_connector_profiles", ["credential_profile_id"], unique=False) + + if "file_connector_credentials" not in table_names: + op.create_table( + "file_connector_credentials", + sa.Column("id", sa.String(length=255), nullable=False), + sa.Column("tenant_id", sa.String(length=36), nullable=True), + sa.Column("scope_type", sa.String(length=20), nullable=False), + sa.Column("scope_id", sa.String(length=36), nullable=True), + sa.Column("label", sa.String(length=255), nullable=False), + sa.Column("provider", sa.String(length=50), nullable=True), + sa.Column("enabled", sa.Boolean(), nullable=False), + sa.Column("credential_mode", sa.String(length=30), nullable=False), + sa.Column("username", sa.String(length=320), nullable=True), + sa.Column("password_encrypted", sa.Text(), nullable=True), + sa.Column("token_encrypted", sa.Text(), nullable=True), + sa.Column("password_env", sa.String(length=255), nullable=True), + sa.Column("token_env", sa.String(length=255), nullable=True), + sa.Column("secret_ref", sa.String(length=1000), nullable=True), + sa.Column("policy", sa.JSON(), nullable=True), + sa.Column("metadata", sa.JSON(), nullable=True), + sa.Column("created_by_user_id", sa.String(length=36), nullable=True), + sa.Column("updated_by_user_id", sa.String(length=36), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(["created_by_user_id"], ["access_users.id"], name=op.f("fk_file_connector_credentials_created_by_user_id_users"), ondelete="SET NULL"), + sa.ForeignKeyConstraint(["tenant_id"], ["tenancy_tenants.id"], name=op.f("fk_file_connector_credentials_tenant_id_tenants"), ondelete="CASCADE"), + sa.ForeignKeyConstraint(["updated_by_user_id"], ["access_users.id"], name=op.f("fk_file_connector_credentials_updated_by_user_id_users"), ondelete="SET NULL"), + sa.PrimaryKeyConstraint("id", name=op.f("pk_file_connector_credentials")), + ) + + inspector = sa.inspect(op.get_bind()) + indexes = {item["name"] for item in inspector.get_indexes("file_connector_credentials")} + for column in ( + "tenant_id", + "scope_type", + "scope_id", + "provider", + "enabled", + "created_by_user_id", + "updated_by_user_id", + ): + name = op.f(f"ix_file_connector_credentials_{column}") + if name not in indexes: + op.create_index(name, "file_connector_credentials", [column], unique=False) + if "ix_file_connector_credentials_scope" not in indexes: + op.create_index("ix_file_connector_credentials_scope", "file_connector_credentials", ["scope_type", "scope_id"], unique=False) + + +def downgrade() -> None: + inspector = sa.inspect(op.get_bind()) + if "file_connector_credentials" in inspector.get_table_names(): + indexes = {item["name"] for item in inspector.get_indexes("file_connector_credentials")} + for name in ( + "ix_file_connector_credentials_scope", + op.f("ix_file_connector_credentials_updated_by_user_id"), + op.f("ix_file_connector_credentials_created_by_user_id"), + op.f("ix_file_connector_credentials_enabled"), + op.f("ix_file_connector_credentials_provider"), + op.f("ix_file_connector_credentials_scope_id"), + op.f("ix_file_connector_credentials_scope_type"), + op.f("ix_file_connector_credentials_tenant_id"), + ): + if name in indexes: + op.drop_index(name, table_name="file_connector_credentials") + op.drop_table("file_connector_credentials") + + inspector = sa.inspect(op.get_bind()) + if "file_connector_profiles" not in inspector.get_table_names(): + return + profile_indexes = {item["name"] for item in inspector.get_indexes("file_connector_profiles")} + profile_index_name = op.f("ix_file_connector_profiles_credential_profile_id") + if profile_index_name in profile_indexes: + op.drop_index(profile_index_name, table_name="file_connector_profiles") + profile_columns = {item["name"] for item in inspector.get_columns("file_connector_profiles")} + if "credential_profile_id" in profile_columns: + op.drop_column("file_connector_profiles", "credential_profile_id") diff --git a/src/govoplan_files/backend/migrations/versions/a7b8c9d0e1f3_file_connector_policies.py b/src/govoplan_files/backend/migrations/versions/a7b8c9d0e1f3_file_connector_policies.py new file mode 100644 index 0000000..b66de23 --- /dev/null +++ b/src/govoplan_files/backend/migrations/versions/a7b8c9d0e1f3_file_connector_policies.py @@ -0,0 +1,71 @@ +"""file connector policies + +Revision ID: a7b8c9d0e1f3 +Revises: 6b7c8d9e0f1a +Create Date: 2026-07-08 00:00:00.000000 +""" +from __future__ import annotations + +from alembic import op +import sqlalchemy as sa + + +revision = "a7b8c9d0e1f3" +down_revision = "6b7c8d9e0f1a" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + inspector = sa.inspect(op.get_bind()) + if "file_connector_policies" not in inspector.get_table_names(): + op.create_table( + "file_connector_policies", + sa.Column("id", sa.String(length=36), nullable=False), + sa.Column("tenant_id", sa.String(length=36), nullable=True), + sa.Column("scope_type", sa.String(length=20), nullable=False), + sa.Column("scope_id", sa.String(length=36), nullable=True), + sa.Column("policy", sa.JSON(), nullable=False), + sa.Column("created_by_user_id", sa.String(length=36), nullable=True), + sa.Column("updated_by_user_id", sa.String(length=36), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(["created_by_user_id"], ["access_users.id"], name=op.f("fk_file_connector_policies_created_by_user_id_users"), ondelete="SET NULL"), + sa.ForeignKeyConstraint(["tenant_id"], ["tenancy_tenants.id"], name=op.f("fk_file_connector_policies_tenant_id_tenants"), ondelete="CASCADE"), + sa.ForeignKeyConstraint(["updated_by_user_id"], ["access_users.id"], name=op.f("fk_file_connector_policies_updated_by_user_id_users"), ondelete="SET NULL"), + sa.PrimaryKeyConstraint("id", name=op.f("pk_file_connector_policies")), + sa.UniqueConstraint("tenant_id", "scope_type", "scope_id", name="uq_file_connector_policies_scope"), + ) + + inspector = sa.inspect(op.get_bind()) + indexes = {item["name"] for item in inspector.get_indexes("file_connector_policies")} + for column in ( + "tenant_id", + "scope_type", + "scope_id", + "created_by_user_id", + "updated_by_user_id", + ): + name = op.f(f"ix_file_connector_policies_{column}") + if name not in indexes: + op.create_index(name, "file_connector_policies", [column], unique=False) + if "ix_file_connector_policies_scope" not in indexes: + op.create_index("ix_file_connector_policies_scope", "file_connector_policies", ["scope_type", "scope_id"], unique=False) + + +def downgrade() -> None: + inspector = sa.inspect(op.get_bind()) + if "file_connector_policies" not in inspector.get_table_names(): + return + indexes = {item["name"] for item in inspector.get_indexes("file_connector_policies")} + for name in ( + "ix_file_connector_policies_scope", + op.f("ix_file_connector_policies_updated_by_user_id"), + op.f("ix_file_connector_policies_created_by_user_id"), + op.f("ix_file_connector_policies_scope_id"), + op.f("ix_file_connector_policies_scope_type"), + op.f("ix_file_connector_policies_tenant_id"), + ): + if name in indexes: + op.drop_index(name, table_name="file_connector_policies") + op.drop_table("file_connector_policies") diff --git a/src/govoplan_files/backend/router.py b/src/govoplan_files/backend/router.py index 27f85d0..3d8af5e 100644 --- a/src/govoplan_files/backend/router.py +++ b/src/govoplan_files/backend/router.py @@ -3,16 +3,45 @@ from __future__ import annotations import json import os import tempfile +from dataclasses import replace +from datetime import datetime from io import BytesIO -from typing import Literal -from urllib.parse import quote -from fastapi import APIRouter, Depends, File as FastAPIFile, Form, HTTPException, UploadFile, status +from typing import Any, Literal +from urllib.parse import quote, urljoin +from fastapi import APIRouter, Depends, File as FastAPIFile, Form, HTTPException, Query, UploadFile, status from fastapi.responses import FileResponse, StreamingResponse from starlette.background import BackgroundTask +from sqlalchemy import func, or_ from sqlalchemy.orm import Session -from govoplan_access.backend.auth.dependencies import ApiPrincipal, has_scope, require_scope +from govoplan_access.auth import ApiPrincipal, has_scope, require_any_scope, require_scope +from govoplan_core.api.v1.schemas import DeltaDeletedItem +from govoplan_core.audit.logging import audit_from_principal from govoplan_core.core.campaigns import CAPABILITY_CAMPAIGNS_ACCESS, CampaignAccessProvider +from govoplan_core.core.change_sequence import ( + ChangeSequenceEntry, + decode_sequence_watermark, + encode_sequence_watermark, + max_sequence_id, + record_change, + sequence_entries_since, + sequence_watermark_is_expired, +) +from govoplan_core.core.pagination import ( + KeysetCursorError, + decode_keyset_cursor, + encode_keyset_cursor, + keyset_query_fingerprint, +) +from govoplan_files.backend.change_tracking import ( + FILES_ASSETS_COLLECTION, + FILES_CONNECTOR_CREDENTIALS_COLLECTION, + FILES_CONNECTOR_POLICIES_COLLECTION, + FILES_CONNECTOR_PROFILES_COLLECTION, + FILES_CONNECTOR_SPACES_COLLECTION, + FILES_FOLDERS_COLLECTION, + FILES_MODULE_ID, +) from govoplan_files.backend.schemas import ( ArchiveRequest, BulkFileShareRequest, @@ -21,6 +50,32 @@ from govoplan_files.backend.schemas import ( BulkDeleteResponse, ConflictResolutionRequest, FileAssetResponse, + FileConnectorBrowseItem, + FileConnectorBrowseResponse, + FileConnectorCredentialCreateRequest, + FileConnectorCredentialResponse, + FileConnectorCredentialsResponse, + FileConnectorCredentialUpdateRequest, + FileConnectorDiscoveryRequest, + FileConnectorDiscoveryResponse, + FileConnectorSettingsDeltaResponse, + FileConnectorSpaceCreateRequest, + FileConnectorSpaceResponse, + FileConnectorSpacesResponse, + FileConnectorSpaceUpdateRequest, + FileConnectorPolicyEvaluateRequest, + FileConnectorPolicyEvaluateResponse, + FileConnectorPolicyResponse, + FileConnectorPolicyUpdateRequest, + FileConnectorProfileCreateRequest, + FileConnectorProfileResponse, + FileConnectorProfilesResponse, + FileConnectorProfileUpdateRequest, + FileConnectorProviderResponse, + FileConnectorProvidersResponse, + FileConnectorImportRequest, + FileConnectorSyncResponse, + FileDeltaResponse, FileFolderCreateRequest, FileFolderDeleteRequest, FileFolderDeleteResponse, @@ -42,30 +97,92 @@ from govoplan_files.backend.schemas import ( TransferResponse, _conflict_resolutions, ) -from govoplan_files.backend.db.models import CampaignAttachmentUse, FileAsset, FileBlob, FileFolder, FileShare, FileVersion +from govoplan_files.backend.db.models import CampaignAttachmentUse, FileAsset, FileBlob, FileConnectorSpace, FileFolder, FileShare, FileVersion from govoplan_core.db.session import get_session from govoplan_files.backend.runtime import get_registry, settings from govoplan_files.backend.storage.paths import UnsafeFilePathError, filename_from_path, normalize_logical_path from govoplan_files.backend.storage.access import ensure_group_access, group_refs_for_ids, user_group_ids from govoplan_files.backend.storage.archives import create_zip_file, extract_zip_upload from govoplan_files.backend.storage.common import FileStorageError +from govoplan_files.backend.storage.connector_credential_store import ( + connector_credential_from_row, + create_connector_credential_row, + deactivate_connector_credential_row, + get_connector_credential_row, + list_database_connector_credentials, + update_connector_credential_row, +) +from govoplan_files.backend.storage.connector_browse import ( + ConnectorBrowseError, + ConnectorBrowseUnsupported, + browse_connector_profile, + normalize_connector_browse_path, +) +from govoplan_files.backend.storage.connector_imports import ConnectorImportError, ConnectorImportUnsupported, read_connector_file +from govoplan_files.backend.storage.connector_profile_store import ( + connector_profile_from_row, + create_connector_profile_row, + deactivate_connector_profile_row, + get_connector_profile_row, + list_database_connector_profiles, + update_connector_profile_row, +) +from govoplan_files.backend.storage.connector_profiles import ConnectorProfile, connector_profiles_from_settings +from govoplan_files.backend.storage.connector_providers import connector_provider_descriptors +from govoplan_files.backend.storage.connector_spaces import ( + connector_space_owner_id, + create_connector_space, + get_connector_space_for_user, + list_connector_spaces_for_user, + soft_delete_connector_space, + update_connector_space, +) +from govoplan_files.backend.storage.connector_policy import ( + ConnectorAccessRequest, + ConnectorPolicyDenied, + connector_policy_decision, + connector_policy_sources_for_fields, + connector_policy_sources_from_payload, + ensure_connector_policy_allows, +) +from govoplan_files.backend.storage.connector_policy_store import ( + connector_policy_response, + effective_connector_policy_sources, + parent_connector_policy, + set_connector_policy, + validate_connector_policy_allowed_by_parent, +) from govoplan_files.backend.storage.files import ( asset_is_audit_relevant, create_file_asset, current_version_and_blob, get_asset_for_user, list_assets_for_user, + list_assets_for_user_window, read_asset_bytes, share_file, share_files, soft_delete_assets, + sync_file_asset_from_source, ) -from govoplan_files.backend.storage.folders import create_folder, list_folders_for_user, soft_delete_folder +from govoplan_files.backend.storage.folders import create_folder, list_folders_for_user, list_folders_for_user_window, soft_delete_folder +from govoplan_files.backend.storage.provenance import source_metadata, source_provenance_from_metadata, source_revision_from_metadata from govoplan_files.backend.storage.search import resolve_patterns from govoplan_files.backend.storage.transfers import rename_selection, transfer_selection router = APIRouter(prefix="/files", tags=["files"]) +FILES_CONNECTOR_SETTINGS_COLLECTIONS = ( + FILES_CONNECTOR_PROFILES_COLLECTION, + FILES_CONNECTOR_CREDENTIALS_COLLECTION, + FILES_CONNECTOR_POLICIES_COLLECTION, + FILES_CONNECTOR_SPACES_COLLECTION, +) +FILES_CONNECTOR_PROFILE_RESOURCE = "file_connector_profile" +FILES_CONNECTOR_CREDENTIAL_RESOURCE = "file_connector_credential" +FILES_CONNECTOR_POLICY_RESOURCE = "file_connector_policy" +FILES_CONNECTOR_SPACE_RESOURCE = "file_connector_space" + def _campaign_access_provider() -> CampaignAccessProvider: registry = get_registry() @@ -81,6 +198,148 @@ def _is_admin(principal: ApiPrincipal) -> bool: return has_scope(principal, "files:file:admin") +def _can_read_disabled_connector_profiles(principal: ApiPrincipal) -> bool: + return any( + has_scope(principal, scope) + for scope in ( + "files:file:admin", + "system:settings:read", + "system:settings:write", + "admin:settings:read", + "admin:settings:write", + ) + ) + + +def _record_connector_settings_change( + session: Session, + *, + collection: str, + resource_type: str, + resource_id: str | None, + operation: str, + principal: ApiPrincipal, + tenant_id: str | None, + payload: dict[str, object] | None = None, +) -> None: + if not resource_id: + return + record_change( + session, + module_id=FILES_MODULE_ID, + collection=collection, + resource_type=resource_type, + resource_id=resource_id, + operation=operation, + tenant_id=tenant_id, + actor_type="user", + actor_id=principal.user.id, + payload=payload or {}, + ) + + +def _file_connector_settings_query(session: Session, *, tenant_id: str, since_sequence: int): + return session.query(ChangeSequenceEntry).filter( + ChangeSequenceEntry.id > since_sequence, + ChangeSequenceEntry.module_id == FILES_MODULE_ID, + ChangeSequenceEntry.collection.in_(FILES_CONNECTOR_SETTINGS_COLLECTIONS), + or_(ChangeSequenceEntry.tenant_id == tenant_id, ChangeSequenceEntry.tenant_id.is_(None)), + ) + + +def _file_connector_settings_watermark(session: Session, *, tenant_id: str) -> str: + sequence_id = _file_connector_settings_query(session, tenant_id=tenant_id, since_sequence=0).with_entities(func.max(ChangeSequenceEntry.id)).scalar() + return encode_sequence_watermark(int(sequence_id or 0)) + + +def _file_connector_settings_entries(session: Session, *, tenant_id: str, since: str, limit: int): + try: + since_sequence = decode_sequence_watermark(since) + except ValueError as exc: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from exc + expired_for_tenant = sequence_watermark_is_expired( + session, + since=since_sequence, + tenant_id=tenant_id, + module_id=FILES_MODULE_ID, + collections=FILES_CONNECTOR_SETTINGS_COLLECTIONS, + ) + expired_for_system = sequence_watermark_is_expired( + session, + since=since_sequence, + tenant_id=None, + module_id=FILES_MODULE_ID, + collections=FILES_CONNECTOR_SETTINGS_COLLECTIONS, + ) + if expired_for_tenant or expired_for_system: + return None, False + entries_plus_one = _file_connector_settings_query( + session, + tenant_id=tenant_id, + since_sequence=since_sequence, + ).order_by(ChangeSequenceEntry.id.asc()).limit(limit + 1).all() + has_more = len(entries_plus_one) > limit + return entries_plus_one[:limit], has_more + + +def _file_connector_settings_response_watermark(session: Session, *, tenant_id: str, entries, has_more: bool) -> str: + return encode_sequence_watermark(entries[-1].id) if has_more and entries else _file_connector_settings_watermark(session, tenant_id=tenant_id) + + +def _connector_deleted_item(entry) -> DeltaDeletedItem: + return DeltaDeletedItem( + id=entry.resource_id, + resource_type=entry.resource_type, + revision=encode_sequence_watermark(entry.id), + deleted_at=entry.created_at if entry.operation == "deleted" else None, + ) + + +def _visible_connector_credentials( + session: Session, + principal: ApiPrincipal, + *, + provider: str | None = None, + include_disabled: bool = False, +): + provider_norm = provider.strip().casefold() if provider else None + credentials = list_database_connector_credentials( + session, + tenant_id=principal.tenant_id, + include_disabled=include_disabled and _can_read_disabled_connector_profiles(principal), + ) + return [ + credential + for credential in credentials + if provider_norm is None or credential.provider in {None, provider_norm} + ] + + +def _visible_connector_spaces( + session: Session, + principal: ApiPrincipal, + *, + owner_type: Literal["user", "group"] | None = None, + owner_id: str | None = None, + include_inactive: bool = False, +) -> list[FileConnectorSpace]: + if not (has_scope(principal, "files:file:read") or has_scope(principal, "files:file:admin")): + return [] + return list_connector_spaces_for_user( + session, + tenant_id=principal.tenant_id, + user_id=principal.user.id, + owner_type=owner_type, + owner_id=owner_id, + include_inactive=include_inactive and _is_admin(principal), + is_admin=_is_admin(principal), + ) + + +def _file_connector_policy_resource_id(scope_type: str, scope_id: str | None) -> str: + return f"{scope_type.strip().casefold()}:{scope_id or ''}" + + async def _read_limited_upload(upload: UploadFile, *, max_bytes: int) -> bytes: data = await upload.read(max_bytes + 1) if len(data) > max_bytes: @@ -91,6 +350,29 @@ async def _read_limited_upload(upload: UploadFile, *, max_bytes: int) -> bytes: return data +async def _spool_limited_upload_to_temp(upload: UploadFile, *, max_bytes: int, suffix: str = ".upload") -> str: + tmp = tempfile.NamedTemporaryFile(prefix="govoplan-upload-", suffix=suffix, delete=False) + total = 0 + try: + while True: + chunk = await upload.read(1024 * 1024) + if not chunk: + break + total += len(chunk) + if total > max_bytes: + raise HTTPException( + status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, + detail=f"Upload exceeds limit of {max_bytes} bytes", + ) + tmp.write(chunk) + except Exception: + tmp.close() + _cleanup_temp_file(tmp.name) + raise + tmp.close() + return tmp.name + + def _cleanup_temp_file(path: str) -> None: @@ -136,12 +418,488 @@ def _http_error(exc: Exception, *, not_found: bool = False) -> HTTPException: return HTTPException(status_code=code, detail=str(exc)) +def _connector_policy_error(exc: ConnectorPolicyDenied) -> HTTPException: + return HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=exc.decision.to_dict()) + + def _owner_id(asset: FileAsset) -> str: return asset.owner_user_id if asset.owner_type == "user" else asset.owner_group_id # type: ignore[return-value] +def _source_metadata_from_form(source_provenance_json: str | None, source_revision: str | None) -> dict[str, object] | None: + source_provenance = json.loads(source_provenance_json) if source_provenance_json else None + return source_metadata(source_provenance=source_provenance, source_revision=source_revision) + + +def _connector_policy_sources_from_form(connector_policy_json: str | None): + if not connector_policy_json: + return [] + return connector_policy_sources_from_payload(json.loads(connector_policy_json)) + + +def _enforce_connector_policy(source_provenance_json: str | None, connector_policy_json: str | None, *, operation: str) -> None: + if not source_provenance_json: + return + sources = _connector_policy_sources_from_form(connector_policy_json) + if not sources: + return + provenance = json.loads(source_provenance_json) + request = ConnectorAccessRequest.from_provenance(provenance if isinstance(provenance, dict) else {}, operation=operation) + ensure_connector_policy_allows(request, sources) + + +def _connector_profile_visible( + session: Session, + principal: ApiPrincipal, + profile: ConnectorProfile, + *, + campaign_id: str | None, + group_ids: set[str], +) -> bool: + if profile.scope_type == "system": + return True + if profile.scope_type == "tenant": + return profile.scope_id == principal.tenant_id + if profile.scope_type == "user": + return profile.scope_id == principal.user.id + if profile.scope_type == "group": + return bool(profile.scope_id and profile.scope_id in group_ids) + if profile.scope_type == "campaign": + if not profile.scope_id: + return False + if campaign_id and profile.scope_id != campaign_id: + return False + try: + _ensure_campaign_file_access(session, principal, profile.scope_id) + except HTTPException: + return False + return True + return False + + +def _visible_connector_profiles( + session: Session, + principal: ApiPrincipal, + *, + provider: str | None = None, + campaign_id: str | None = None, + include_disabled: bool = False, + include_admin_scopes: bool = False, + include_effective_policy: bool = True, +) -> list[ConnectorProfile]: + provider_norm = provider.strip().casefold() if provider else None + group_ids = set(user_group_ids(session, tenant_id=principal.tenant_id, user_id=principal.user.id, include_admin_groups=_is_admin(principal))) + database_profiles = list_database_connector_profiles( + session, + tenant_id=principal.tenant_id, + include_disabled=include_disabled, + ) + env_profiles = connector_profiles_from_settings(settings) + profiles_by_id: dict[str, ConnectorProfile] = {} + for profile in [*database_profiles, *env_profiles]: + if profile.id not in profiles_by_id: + profiles_by_id[profile.id] = profile + profiles = list(profiles_by_id.values()) + visible: list[ConnectorProfile] = [] + for profile in profiles: + if not (profile.enabled or include_disabled): + continue + if provider_norm is not None and profile.provider != provider_norm: + continue + admin_scope_visible = include_admin_scopes and profile.scope_type in {"user", "group", "campaign"} + if not admin_scope_visible and not _connector_profile_visible(session, principal, profile, campaign_id=campaign_id, group_ids=group_ids): + continue + visible.append(_with_effective_connector_policy(session, principal, profile) if include_effective_policy else profile) + return visible + + +def _webdav_discovery_candidates(payload: FileConnectorDiscoveryRequest) -> list[str]: + endpoint_url = str(payload.endpoint_url or "").strip() + if not endpoint_url: + return [] + direct_url = endpoint_url if endpoint_url.endswith("/") else endpoint_url + "/" + candidates = [direct_url] + if payload.provider == "nextcloud": + root_url = _nextcloud_root_url(direct_url) + username = (payload.credentials.username or "").strip() + if username: + candidates.append(urljoin(root_url, f"remote.php/dav/files/{quote(username, safe='')}/")) + candidates.append(urljoin(root_url, "remote.php/webdav/")) + seen: set[str] = set() + unique: list[str] = [] + for candidate in candidates: + normalized = candidate.rstrip("/") + if not normalized or normalized in seen: + continue + seen.add(normalized) + unique.append(candidate) + return unique + + +def _nextcloud_root_url(endpoint_url: str) -> str: + marker = "/remote.php/" + if marker in endpoint_url: + return endpoint_url.split(marker, 1)[0].rstrip("/") + "/" + return endpoint_url.rstrip("/") + "/" + + +def _same_endpoint(left: str, right: str) -> bool: + return left.strip().rstrip("/") == right.strip().rstrip("/") + + +def _discovery_profile_from_payload( + payload: FileConnectorDiscoveryRequest, + endpoint_url: str, + *, + principal: ApiPrincipal, +) -> ConnectorProfile: + credentials = payload.credentials + return ConnectorProfile( + id=f"discovery-{principal.tenant_id}", + label="Connector discovery", + provider=payload.provider, + endpoint_url=endpoint_url, + base_path=payload.base_path, + scope_type="tenant", + scope_id=principal.tenant_id, + credential_mode=payload.credential_mode, + username=credentials.username, + password_env=credentials.password_env, + token_env=credentials.token_env, + secret_ref=credentials.secret_ref, + password_value=credentials.password, + token_value=credentials.token, + capabilities=("browse",), + metadata=payload.metadata, + source_kind="discovery", + ) + + +def _visible_connector_profile( + session: Session, + principal: ApiPrincipal, + profile_id: str, + *, + campaign_id: str | None = None, + include_disabled: bool = False, + include_effective_policy: bool = True, +) -> ConnectorProfile: + for profile in _visible_connector_profiles( + session, + principal, + campaign_id=campaign_id, + include_disabled=include_disabled, + include_admin_scopes=False, + include_effective_policy=include_effective_policy, + ): + if profile.id == profile_id: + return profile + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Connector profile not found") + + +def _with_effective_connector_policy(session: Session, principal: ApiPrincipal, profile: ConnectorProfile) -> ConnectorProfile: + sources = effective_connector_policy_sources( + session, + tenant_id=principal.tenant_id, + scope_type=profile.scope_type, + scope_id=profile.scope_id, + ) + if not sources: + return profile + return replace(profile, policy_sources=tuple([*sources, *profile.policy_sources])) + + +def _require_connector_profile_write(principal: ApiPrincipal, scope_type: str) -> None: + clean_scope = scope_type.strip().casefold() + if clean_scope == "system": + if has_scope(principal, "system:settings:write") or has_scope(principal, "files:file:admin"): + return + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Missing scope: system:settings:write") + if has_scope(principal, "files:file:admin") or has_scope(principal, "admin:settings:write") or has_scope(principal, "system:settings:write"): + return + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Missing scope: files:file:admin") + + +def _require_connector_credential_write(principal: ApiPrincipal, scope_type: str) -> None: + _require_connector_profile_write(principal, scope_type) + + +def _require_connector_policy_read(principal: ApiPrincipal, scope_type: str) -> None: + clean_scope = scope_type.strip().casefold() + if clean_scope == "system": + if any(has_scope(principal, scope) for scope in ("files:file:admin", "system:settings:read", "system:settings:write")): + return + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Missing scope: system:settings:read") + if any(has_scope(principal, scope) for scope in ("files:file:admin", "admin:settings:read", "admin:settings:write", "system:settings:read", "system:settings:write")): + return + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Missing scope: files:file:admin") + + +def _connector_policy_configure_sources( + session: Session, + principal: ApiPrincipal, + *, + scope_type: str, + scope_id: str | None, + fields: tuple[str, ...], +): + sources = effective_connector_policy_sources(session, tenant_id=principal.tenant_id, scope_type=scope_type, scope_id=scope_id) + return connector_policy_sources_for_fields(sources, fields) + + +def _ensure_connector_configuration_allowed( + session: Session, + principal: ApiPrincipal, + *, + connector_id: str | None, + credential_id: str | None, + provider: str | None, + endpoint_url: str | None, + base_path: str | None, + scope_type: str, + scope_id: str | None, + operation: str, +) -> None: + sources = _connector_policy_configure_sources( + session, + principal, + scope_type=scope_type, + scope_id=scope_id, + fields=("connectors", "credentials", "providers", "external_paths", "external_urls"), + ) + if not sources: + return + ensure_connector_policy_allows( + ConnectorAccessRequest( + connector_id=connector_id, + credential_id=credential_id, + provider=provider, + external_path=base_path, + external_url=endpoint_url, + operation=operation, + ), + sources, + ) + + +def _ensure_connector_credential_configuration_allowed( + session: Session, + principal: ApiPrincipal, + *, + credential_id: str, + provider: str | None, + scope_type: str, + scope_id: str | None, + operation: str, +) -> None: + sources = _connector_policy_configure_sources( + session, + principal, + scope_type=scope_type, + scope_id=scope_id, + fields=("credentials", "providers"), + ) + if not sources: + return + ensure_connector_policy_allows( + ConnectorAccessRequest( + credential_id=credential_id, + provider=provider, + operation=operation, + ), + sources, + ) + + +def _ensure_connector_local_policy_allowed( + session: Session, + principal: ApiPrincipal, + *, + scope_type: str, + scope_id: str | None, + policy: dict[str, Any] | None, +) -> None: + if policy is None: + return + validate_connector_policy_allowed_by_parent( + parent_connector_policy(session, tenant_id=principal.tenant_id, scope_type=scope_type, scope_id=scope_id), + policy, + ) + + +def _connector_credential_response(row) -> FileConnectorCredentialResponse: + return FileConnectorCredentialResponse(**connector_credential_from_row(row).to_response()) + + +def _credential_row_for_profile( + session: Session, + principal: ApiPrincipal, + *, + credential_profile_id: str | None, + provider: str, + include_disabled: bool = False, +): + credential_id = (credential_profile_id or "").strip() + if not credential_id: + return None + row = get_connector_credential_row( + session, + tenant_id=principal.tenant_id, + credential_id=credential_id, + include_disabled=include_disabled, + ) + if row.provider and row.provider != provider: + raise FileStorageError(f"Connector credential {credential_id} is limited to {row.provider} connections") + return row + + +def _download_connector_payload( + profile: ConnectorProfile, + payload: FileConnectorImportRequest, + *, + operation: str, +) -> tuple[str, Any, dict[str, Any]]: + source_path = normalize_connector_browse_path(payload.path) + decision = connector_policy_decision( + ConnectorAccessRequest( + connector_id=profile.id, + credential_id=profile.credential_profile_id, + provider=profile.provider, + external_id=f"{payload.library_id}:{source_path}", + external_path=source_path, + external_url=profile.endpoint_url, + operation=operation, + ), + profile.policy_sources, + ) + if not decision.allowed: + raise ConnectorPolicyDenied(decision) + downloaded = read_connector_file( + profile, + library_id=payload.library_id, + path=source_path, + max_bytes=settings.file_upload_max_bytes, + ) + provenance_metadata = { + "profile_id": profile.id, + "library_id": payload.library_id, + "library_path": source_path, + **downloaded.metadata, + **payload.metadata, + } + metadata = source_metadata( + source_provenance={ + "source_type": "connector", + "connector_id": profile.id, + "provider": profile.provider, + "external_id": downloaded.external_id or f"{payload.library_id}:{source_path}", + "external_path": source_path, + "external_url": downloaded.external_url, + "metadata": provenance_metadata, + }, + source_revision=payload.source_revision or downloaded.revision, + ) + return source_path, downloaded, metadata or {} + + +def _connector_audit_details(asset: FileAsset, version: FileVersion, blob: FileBlob, *, operation: str) -> dict[str, object] | None: + metadata = asset.metadata_ or {} + provenance = source_provenance_from_metadata(metadata) + if not provenance: + return None + return { + "operation": operation, + "file_id": asset.id, + "display_path": asset.display_path, + "version_id": version.id, + "blob_id": blob.id, + "checksum_sha256": blob.checksum_sha256, + "size_bytes": blob.size_bytes, + "source_revision": source_revision_from_metadata(metadata), + "source_provenance": provenance, + } + + +def _audit_connector_event( + session: Session, + principal: ApiPrincipal, + *, + action: str, + asset: FileAsset, + version: FileVersion, + blob: FileBlob, + operation: str, + extra_details: dict[str, object] | None = None, + commit: bool = False, +) -> bool: + details = _connector_audit_details(asset, version, blob, operation=operation) + if details is None: + return False + if extra_details: + details.update(extra_details) + audit_from_principal( + session, + principal, + action=action, + object_type="file", + object_id=asset.id, + details=details, + commit=commit, + ) + return True + + +def _audit_connector_imports(session: Session, principal: ApiPrincipal, assets: list[FileAsset]) -> None: + for asset in assets: + version, blob = current_version_and_blob(session, asset) + _audit_connector_event( + session, + principal, + action="files.connector.imported", + asset=asset, + version=version, + blob=blob, + operation="import", + ) + + +def _audit_connector_sync(session: Session, principal: ApiPrincipal, asset: FileAsset, *, sync_action: str, previous_version_id: str | None) -> None: + version, blob = current_version_and_blob(session, asset) + _audit_connector_event( + session, + principal, + action="files.connector.synced", + asset=asset, + version=version, + blob=blob, + operation="sync", + extra_details={ + "sync_action": sync_action, + "previous_version_id": previous_version_id, + }, + ) + + +def _audit_connector_access(session: Session, principal: ApiPrincipal, assets: list[FileAsset], *, operation: str) -> None: + recorded = False + for asset in assets: + version, blob = current_version_and_blob(session, asset) + recorded = _audit_connector_event( + session, + principal, + action="files.connector.accessed", + asset=asset, + version=version, + blob=blob, + operation=operation, + ) or recorded + if recorded: + session.commit() + + def _asset_response(session: Session, asset: FileAsset, *, include_shares: bool = False) -> FileAssetResponse: version, blob = current_version_and_blob(session, asset) + metadata = asset.metadata_ or {} shares: list[FileShareResponse] = [] if include_shares: rows = session.query(FileShare).filter(FileShare.file_asset_id == asset.id).order_by(FileShare.created_at.desc()).all() @@ -172,7 +930,9 @@ def _asset_response(session: Session, asset: FileAsset, *, include_shares: bool updated_at=asset.updated_at.isoformat(), deleted_at=asset.deleted_at.isoformat() if asset.deleted_at else None, audit_relevant=asset_is_audit_relevant(session, asset), - metadata=asset.metadata_ or {}, + metadata=metadata, + source_provenance=source_provenance_from_metadata(metadata), + source_revision=source_revision_from_metadata(metadata), shares=shares, ) @@ -234,6 +994,7 @@ def _asset_list_response(session: Session, assets: list[FileAsset], *, include_s blob = blobs_by_version_id.get(asset.current_version_id or "") if not version or not blob: raise FileStorageError("File version not found") + metadata = asset.metadata_ or {} responses.append( FileAssetResponse( id=asset.id, @@ -251,7 +1012,9 @@ def _asset_list_response(session: Session, assets: list[FileAsset], *, include_s updated_at=asset.updated_at.isoformat(), deleted_at=asset.deleted_at.isoformat() if asset.deleted_at else None, audit_relevant=asset.id in sent_asset_ids, - metadata=asset.metadata_ or {}, + metadata=metadata, + source_provenance=source_provenance_from_metadata(metadata), + source_revision=source_revision_from_metadata(metadata), shares=shares_by_asset_id.get(asset.id, []), ) ) @@ -280,6 +1043,69 @@ def _folder_response(folder: FileFolder) -> FileFolderResponse: ) +def _connector_space_response(space: FileConnectorSpace) -> FileConnectorSpaceResponse: + return FileConnectorSpaceResponse( + id=space.id, + tenant_id=space.tenant_id, + owner_type=space.owner_type, # type: ignore[arg-type] + owner_id=connector_space_owner_id(space), + label=space.label, + connector_profile_id=space.connector_profile_id, + provider=space.provider, + library_id=space.library_id, + remote_path=space.remote_path, + sync_mode=space.sync_mode, + read_only=space.read_only, + is_active=space.is_active, + created_at=space.created_at.isoformat(), + updated_at=space.updated_at.isoformat(), + deleted_at=space.deleted_at.isoformat() if space.deleted_at else None, + metadata=space.metadata_ or {}, + ) + + +def _connector_space_file_space_response(space: FileConnectorSpace) -> FileSpaceResponse: + owner_id = connector_space_owner_id(space) + return FileSpaceResponse( + id=f"connector:{space.id}", + label=space.label, + owner_type=space.owner_type, # type: ignore[arg-type] + owner_id=owner_id, + description=f"Linked {space.provider} folder.", + space_type="connector", + connector_space_id=space.id, + connector_profile_id=space.connector_profile_id, + provider=space.provider, + library_id=space.library_id, + remote_path=space.remote_path, + sync_mode=space.sync_mode, + read_only=space.read_only, + ) + + +def _connector_space_policy_decision( + profile: ConnectorProfile, + *, + library_id: str | None, + remote_path: str | None, + operation: str, +): + browse_path = normalize_connector_browse_path(remote_path) + external_id = f"{library_id}:{browse_path}" if library_id else browse_path or profile.id + return connector_policy_decision( + ConnectorAccessRequest( + connector_id=profile.id, + credential_id=profile.credential_profile_id, + provider=profile.provider, + external_id=external_id, + external_path=browse_path, + external_url=profile.endpoint_url, + operation=operation, + ), + profile.policy_sources, + ) + + def _ensure_list_owner_access(session: Session, principal: ApiPrincipal, owner_type: str | None, owner_id: str | None) -> None: if not owner_type: return @@ -298,6 +1124,430 @@ def _ensure_list_owner_access(session: Session, principal: ApiPrincipal, owner_t raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=str(exc)) from exc +_FILES_DELTA_COLLECTIONS = (FILES_ASSETS_COLLECTION, FILES_FOLDERS_COLLECTION) +FILES_LIST_CURSOR_SCOPE = "files.list.v1" +FOLDERS_LIST_CURSOR_SCOPE = "files.folders.list.v1" +DEFAULT_FILE_LIST_PAGE_SIZE = 500 + + +def _cursor_http_error(exc: Exception) -> HTTPException: + return HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) + + +def _cursor_page_size(scope: str, cursor: str | None, explicit_page_size: int | None) -> int | None: + if explicit_page_size is not None: + return explicit_page_size + if not cursor: + return None + try: + values = decode_keyset_cursor(scope, cursor) + except KeysetCursorError as exc: + raise _cursor_http_error(exc) from exc + raw_page_size = values.get("page_size") + if not isinstance(raw_page_size, int) or raw_page_size < 1 or raw_page_size > 1000: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid pagination cursor") + return raw_page_size + + +def _files_list_fingerprint( + principal: ApiPrincipal, + *, + owner_type: Literal["user", "group"] | None, + owner_id: str | None, + campaign_id: str | None, + path_prefix: str | None, + page_size: int, +) -> str: + return keyset_query_fingerprint( + FILES_LIST_CURSOR_SCOPE, + { + "tenant_id": principal.tenant_id, + "actor": "admin" if _is_admin(principal) else principal.user.id, + "owner_type": owner_type, + "owner_id": owner_id, + "campaign_id": campaign_id, + "path_prefix": path_prefix or "", + "sort": "display_path.asc,updated_at.desc,id.asc", + "page_size": page_size, + }, + ) + + +def _folders_list_fingerprint( + principal: ApiPrincipal, + *, + owner_type: Literal["user", "group"], + owner_id: str, + page_size: int, +) -> str: + return keyset_query_fingerprint( + FOLDERS_LIST_CURSOR_SCOPE, + { + "tenant_id": principal.tenant_id, + "actor": "admin" if _is_admin(principal) else principal.user.id, + "owner_type": owner_type, + "owner_id": owner_id, + "sort": "path.asc,id.asc", + "page_size": page_size, + }, + ) + + +def _file_cursor_values(cursor: str | None, *, fingerprint: str) -> tuple[str | None, datetime | None, str | None]: + try: + values = decode_keyset_cursor(FILES_LIST_CURSOR_SCOPE, cursor, fingerprint=fingerprint) + except KeysetCursorError as exc: + raise _cursor_http_error(exc) from exc + if values is None: + return None, None, None + display_path = values.get("display_path") + updated_at = values.get("updated_at") + asset_id = values.get("id") + if not isinstance(display_path, str) or not isinstance(updated_at, str) or not isinstance(asset_id, str): + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid pagination cursor") + try: + parsed_updated_at = datetime.fromisoformat(updated_at) + except ValueError as exc: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid pagination cursor") from exc + return display_path, parsed_updated_at, asset_id + + +def _folder_cursor_values(cursor: str | None, *, fingerprint: str) -> tuple[str | None, str | None]: + try: + values = decode_keyset_cursor(FOLDERS_LIST_CURSOR_SCOPE, cursor, fingerprint=fingerprint) + except KeysetCursorError as exc: + raise _cursor_http_error(exc) from exc + if values is None: + return None, None + folder_path = values.get("path") + folder_id = values.get("id") + if not isinstance(folder_path, str) or not isinstance(folder_id, str): + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid pagination cursor") + return folder_path, folder_id + + +def _next_file_list_cursor( + principal: ApiPrincipal, + assets: list[FileAsset], + *, + owner_type: Literal["user", "group"] | None, + owner_id: str | None, + campaign_id: str | None, + path_prefix: str | None, + page_size: int, + has_more: bool, +) -> str | None: + if not has_more or not assets: + return None + last = assets[-1] + return encode_keyset_cursor( + FILES_LIST_CURSOR_SCOPE, + fingerprint=_files_list_fingerprint( + principal, + owner_type=owner_type, + owner_id=owner_id, + campaign_id=campaign_id, + path_prefix=path_prefix, + page_size=page_size, + ), + values={"display_path": last.display_path, "updated_at": last.updated_at, "id": last.id, "page_size": page_size}, + ) + + +def _next_folder_list_cursor( + principal: ApiPrincipal, + folders: list[FileFolder], + *, + owner_type: Literal["user", "group"], + owner_id: str, + page_size: int, + has_more: bool, +) -> str | None: + if not has_more or not folders: + return None + last = folders[-1] + return encode_keyset_cursor( + FOLDERS_LIST_CURSOR_SCOPE, + fingerprint=_folders_list_fingerprint(principal, owner_type=owner_type, owner_id=owner_id, page_size=page_size), + values={"path": last.path, "id": last.id, "page_size": page_size}, + ) + + +def _files_delta_watermark(session: Session, tenant_id: str) -> str: + return encode_sequence_watermark( + max_sequence_id( + session, + tenant_id=tenant_id, + module_id=FILES_MODULE_ID, + collections=_FILES_DELTA_COLLECTIONS, + ) + ) + + +def _full_file_delta_response( + session: Session, + *, + principal: ApiPrincipal, + owner_type: Literal["user", "group"] | None, + owner_id: str | None, + campaign_id: str | None, + path_prefix: str | None, +) -> FileDeltaResponse: + assets = list_assets_for_user( + session, + tenant_id=principal.tenant_id, + user_id=principal.user.id, + owner_type=owner_type, + owner_id=owner_id, + campaign_id=campaign_id, + path_prefix=path_prefix, + is_admin=_is_admin(principal), + ) + folders = _visible_folders_for_delta( + session, + principal=principal, + owner_type=owner_type, + owner_id=owner_id, + path_prefix=path_prefix, + ) + return FileDeltaResponse( + files=_asset_list_response(session, assets, include_shares=True), + folders=[_folder_response(folder) for folder in folders], + deleted=[], + watermark=_files_delta_watermark(session, principal.tenant_id), + has_more=False, + full=True, + ) + + +def _visible_folders_for_delta( + session: Session, + *, + principal: ApiPrincipal, + owner_type: Literal["user", "group"] | None, + owner_id: str | None, + path_prefix: str | None, +) -> list[FileFolder]: + if not owner_type or not owner_id: + return [] + folders = list_folders_for_user( + session, + tenant_id=principal.tenant_id, + user_id=principal.user.id, + owner_type=owner_type, + owner_id=owner_id, + is_admin=_is_admin(principal), + ) + if not path_prefix: + return folders + normalized = path_prefix.strip().strip("/") + if not normalized: + return folders + return [folder for folder in folders if folder.path == normalized or folder.path.startswith(f"{normalized}/")] + + +def _entry_path_matches(entry_payload: dict[str, object], path_prefix: str | None) -> bool: + if not path_prefix: + return True + normalized = path_prefix.strip().strip("/") + if not normalized: + return True + for key in ("path", "previous_path"): + value = entry_payload.get(key) + if isinstance(value, str) and (value == normalized or value.startswith(f"{normalized}/")): + return True + return False + + +def _entry_owner_matches( + entry_payload: dict[str, object], + owner_type: Literal["user", "group"] | None, + owner_id: str | None, +) -> bool: + if not owner_type or not owner_id: + return True + return entry_payload.get("owner_type") == owner_type and entry_payload.get("owner_id") == owner_id + + +def _entry_campaign_matches(session: Session, entry, campaign_id: str | None) -> bool: + if not campaign_id: + return True + payload = entry.payload or {} + if payload.get("share_target_type") == "campaign" and payload.get("share_target_id") == campaign_id: + return True + if entry.resource_type != "file": + return False + return ( + session.query(FileShare) + .filter( + FileShare.tenant_id == entry.tenant_id, + FileShare.file_asset_id == entry.resource_id, + FileShare.target_type == "campaign", + FileShare.target_id == campaign_id, + FileShare.revoked_at.is_(None), + ) + .first() + is not None + ) + + +def _principal_group_ids_for_delta(session: Session, principal: ApiPrincipal) -> set[str]: + cache = session.info.setdefault("files_delta_group_ids", {}) + key = (principal.tenant_id, principal.user.id) + if key not in cache: + cache[key] = set(user_group_ids(session, tenant_id=principal.tenant_id, user_id=principal.user.id)) + return cache[key] + + +def _entry_subject_matches_principal(session: Session, principal: ApiPrincipal, entry_payload: dict[str, object]) -> bool: + if _is_admin(principal): + return True + owner_type = entry_payload.get("owner_type") + owner_id = entry_payload.get("owner_id") + if owner_type == "user" and owner_id == principal.user.id: + return True + if owner_type == "group" and isinstance(owner_id, str): + if owner_id in _principal_group_ids_for_delta(session, principal): + return True + share_target_type = entry_payload.get("share_target_type") + share_target_id = entry_payload.get("share_target_id") + if share_target_type == "user" and share_target_id == principal.user.id: + return True + if share_target_type == "tenant" and share_target_id == principal.tenant_id: + return True + if share_target_type == "group" and isinstance(share_target_id, str): + if share_target_id in _principal_group_ids_for_delta(session, principal): + return True + return False + + +def _entry_matches_delta_scope( + session: Session, + principal: ApiPrincipal, + entry, + *, + owner_type: Literal["user", "group"] | None, + owner_id: str | None, + campaign_id: str | None, + path_prefix: str | None, +) -> bool: + payload = entry.payload or {} + if not owner_type and not campaign_id and not _entry_subject_matches_principal(session, principal, payload): + return False + return ( + _entry_owner_matches(payload, owner_type, owner_id) + and _entry_path_matches(payload, path_prefix) + and _entry_campaign_matches(session, entry, campaign_id) + ) + + +def _files_delta_response( + session: Session, + *, + principal: ApiPrincipal, + owner_type: Literal["user", "group"] | None, + owner_id: str | None, + campaign_id: str | None, + path_prefix: str | None, + since: str, + limit: int, +) -> FileDeltaResponse: + try: + since_sequence = decode_sequence_watermark(since) + except ValueError as exc: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from exc + if sequence_watermark_is_expired( + session, + since=since_sequence, + tenant_id=principal.tenant_id, + module_id=FILES_MODULE_ID, + collections=_FILES_DELTA_COLLECTIONS, + ): + return _full_file_delta_response( + session, + principal=principal, + owner_type=owner_type, + owner_id=owner_id, + campaign_id=campaign_id, + path_prefix=path_prefix, + ) + + entries_plus_one = sequence_entries_since( + session, + since=since_sequence, + tenant_id=principal.tenant_id, + module_id=FILES_MODULE_ID, + collections=_FILES_DELTA_COLLECTIONS, + limit=limit + 1, + ) + has_more = len(entries_plus_one) > limit + entries = entries_plus_one[:limit] + + changed_file_ids = list(dict.fromkeys(entry.resource_id for entry in entries if entry.resource_type == "file")) + changed_folder_ids = list(dict.fromkeys(entry.resource_id for entry in entries if entry.resource_type == "folder")) + + visible_assets = { + asset.id: asset + for asset in list_assets_for_user( + session, + tenant_id=principal.tenant_id, + user_id=principal.user.id, + owner_type=owner_type, + owner_id=owner_id, + campaign_id=campaign_id, + path_prefix=path_prefix, + is_admin=_is_admin(principal), + ) + if asset.id in changed_file_ids + } + visible_folders = { + folder.id: folder + for folder in _visible_folders_for_delta( + session, + principal=principal, + owner_type=owner_type, + owner_id=owner_id, + path_prefix=path_prefix, + ) + if folder.id in changed_folder_ids + } + + deleted: dict[tuple[str, str], DeltaDeletedItem] = {} + for entry in entries: + is_visible = ( + (entry.resource_type == "file" and entry.resource_id in visible_assets) + or (entry.resource_type == "folder" and entry.resource_id in visible_folders) + ) + if is_visible: + continue + if not _entry_matches_delta_scope( + session, + principal, + entry, + owner_type=owner_type, + owner_id=owner_id, + campaign_id=campaign_id, + path_prefix=path_prefix, + ): + continue + deleted[(entry.resource_type, entry.resource_id)] = DeltaDeletedItem( + id=entry.resource_id, + resource_type=entry.resource_type, + revision=encode_sequence_watermark(entry.id), + deleted_at=entry.created_at if entry.operation == "deleted" else None, + ) + + watermark = encode_sequence_watermark(entries[-1].id) if has_more and entries else _files_delta_watermark(session, principal.tenant_id) + return FileDeltaResponse( + files=_asset_list_response(session, list(visible_assets.values()), include_shares=True), + folders=[_folder_response(folder) for folder in visible_folders.values()], + deleted=list(deleted.values()), + watermark=watermark, + has_more=has_more, + full=False, + ) + + @router.get("/spaces", response_model=FileSpacesResponse) def list_file_spaces( session: Session = Depends(get_session), @@ -325,26 +1575,234 @@ def list_file_spaces( ) for group in groups ) + connector_spaces = list_connector_spaces_for_user( + session, + tenant_id=principal.tenant_id, + user_id=principal.user.id, + is_admin=_is_admin(principal), + ) + spaces.extend(_connector_space_file_space_response(space) for space in connector_spaces) return FileSpacesResponse(spaces=spaces) +@router.get("/connector-spaces", response_model=FileConnectorSpacesResponse) +def list_file_connector_spaces( + owner_type: Literal["user", "group"] | None = None, + owner_id: str | None = None, + include_inactive: bool = False, + session: Session = Depends(get_session), + principal: ApiPrincipal = Depends(require_scope("files:file:read")), +): + try: + spaces = list_connector_spaces_for_user( + session, + tenant_id=principal.tenant_id, + user_id=principal.user.id, + owner_type=owner_type, + owner_id=owner_id, + include_inactive=include_inactive and _is_admin(principal), + is_admin=_is_admin(principal), + ) + return FileConnectorSpacesResponse(spaces=[_connector_space_response(space) for space in spaces]) + except FileStorageError as exc: + raise _http_error(exc) from exc + + +@router.post("/connector-spaces", response_model=FileConnectorSpaceResponse, status_code=status.HTTP_201_CREATED) +def create_file_connector_space( + payload: FileConnectorSpaceCreateRequest, + session: Session = Depends(get_session), + principal: ApiPrincipal = Depends(require_scope("files:file:organize")), +): + if payload.owner_type == "group" and not payload.owner_id: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="owner_id is required for group connector spaces") + target_owner = payload.owner_id or principal.user.id + try: + profile = _visible_connector_profile(session, principal, payload.connector_profile_id) + decision = _connector_space_policy_decision( + profile, + library_id=payload.library_id, + remote_path=payload.remote_path, + operation="link", + ) + if not decision.allowed: + raise ConnectorPolicyDenied(decision) + space = create_connector_space( + session, + tenant_id=principal.tenant_id, + owner_type=payload.owner_type, + owner_id=target_owner, + user_id=principal.user.id, + label=payload.label, + profile=profile, + library_id=payload.library_id, + remote_path=payload.remote_path, + sync_mode=payload.sync_mode, + metadata=payload.metadata, + is_admin=_is_admin(principal), + ) + _record_connector_settings_change( + session, + collection=FILES_CONNECTOR_SPACES_COLLECTION, + resource_type=FILES_CONNECTOR_SPACE_RESOURCE, + resource_id=space.id, + operation="created", + principal=principal, + tenant_id=space.tenant_id, + payload={"owner_type": space.owner_type, "owner_id": connector_space_owner_id(space)}, + ) + session.commit() + return _connector_space_response(space) + except ConnectorPolicyDenied as exc: + session.rollback() + raise _connector_policy_error(exc) from exc + except (FileStorageError, ValueError, json.JSONDecodeError) as exc: + session.rollback() + raise _http_error(exc) from exc + + +@router.patch("/connector-spaces/{space_id}", response_model=FileConnectorSpaceResponse) +def update_file_connector_space( + space_id: str, + payload: FileConnectorSpaceUpdateRequest, + session: Session = Depends(get_session), + principal: ApiPrincipal = Depends(require_scope("files:file:organize")), +): + try: + space = get_connector_space_for_user( + session, + tenant_id=principal.tenant_id, + user_id=principal.user.id, + space_id=space_id, + include_inactive=_is_admin(principal), + is_admin=_is_admin(principal), + ) + except FileStorageError as exc: + raise _http_error(exc, not_found=True) from exc + try: + if payload.library_id is not None or payload.remote_path is not None: + profile = _visible_connector_profile(session, principal, space.connector_profile_id) + decision = _connector_space_policy_decision( + profile, + library_id=payload.library_id if payload.library_id is not None else space.library_id, + remote_path=payload.remote_path if payload.remote_path is not None else space.remote_path, + operation="link", + ) + if not decision.allowed: + raise ConnectorPolicyDenied(decision) + update_connector_space( + session, + space, + user_id=principal.user.id, + label=payload.label, + library_id=payload.library_id, + remote_path=payload.remote_path, + sync_mode=payload.sync_mode, + is_active=payload.is_active, + metadata=payload.metadata, + is_admin=_is_admin(principal), + ) + _record_connector_settings_change( + session, + collection=FILES_CONNECTOR_SPACES_COLLECTION, + resource_type=FILES_CONNECTOR_SPACE_RESOURCE, + resource_id=space.id, + operation="updated", + principal=principal, + tenant_id=space.tenant_id, + payload={"owner_type": space.owner_type, "owner_id": connector_space_owner_id(space)}, + ) + session.commit() + return _connector_space_response(space) + except ConnectorPolicyDenied as exc: + session.rollback() + raise _connector_policy_error(exc) from exc + except (FileStorageError, ValueError, json.JSONDecodeError) as exc: + session.rollback() + raise _http_error(exc) from exc + + +@router.delete("/connector-spaces/{space_id}", response_model=FileConnectorSpaceResponse) +def delete_file_connector_space( + space_id: str, + session: Session = Depends(get_session), + principal: ApiPrincipal = Depends(require_scope("files:file:organize")), +): + try: + space = get_connector_space_for_user( + session, + tenant_id=principal.tenant_id, + user_id=principal.user.id, + space_id=space_id, + include_inactive=True, + is_admin=_is_admin(principal), + ) + soft_delete_connector_space(session, space, user_id=principal.user.id, is_admin=_is_admin(principal)) + _record_connector_settings_change( + session, + collection=FILES_CONNECTOR_SPACES_COLLECTION, + resource_type=FILES_CONNECTOR_SPACE_RESOURCE, + resource_id=space.id, + operation="deleted", + principal=principal, + tenant_id=space.tenant_id, + payload={"owner_type": space.owner_type, "owner_id": connector_space_owner_id(space)}, + ) + session.commit() + return _connector_space_response(space) + except FileStorageError as exc: + session.rollback() + raise _http_error(exc, not_found=True) from exc + + @router.get("/folders", response_model=FileFoldersResponse) def list_file_folders( owner_type: Literal["user", "group"], owner_id: str, + page_size: int | None = Query(default=None, ge=1, le=1000), + cursor: str | None = None, session: Session = Depends(get_session), principal: ApiPrincipal = Depends(require_scope("files:file:read")), ): try: - folders = list_folders_for_user( + watermark = _files_delta_watermark(session, principal.tenant_id) + effective_page_size = _cursor_page_size(FOLDERS_LIST_CURSOR_SCOPE, cursor, page_size) + if effective_page_size is None: + folders = list_folders_for_user( + session, + tenant_id=principal.tenant_id, + user_id=principal.user.id, + owner_type=owner_type, + owner_id=owner_id, + is_admin=_is_admin(principal), + ) + return FileFoldersResponse(folders=[_folder_response(folder) for folder in folders], watermark=watermark) + fingerprint = _folders_list_fingerprint(principal, owner_type=owner_type, owner_id=owner_id, page_size=effective_page_size) + after_path, after_id = _folder_cursor_values(cursor, fingerprint=fingerprint) + folders, has_more = list_folders_for_user_window( session, tenant_id=principal.tenant_id, user_id=principal.user.id, owner_type=owner_type, owner_id=owner_id, is_admin=_is_admin(principal), + page_size=effective_page_size, + after_path=after_path, + after_id=after_id, + ) + return FileFoldersResponse( + folders=[_folder_response(folder) for folder in folders], + cursor=cursor, + next_cursor=_next_folder_list_cursor( + principal, + folders, + owner_type=owner_type, + owner_id=owner_id, + page_size=effective_page_size, + has_more=has_more, + ), + watermark=watermark, ) - return FileFoldersResponse(folders=[_folder_response(folder) for folder in folders]) except FileStorageError as exc: raise _http_error(exc) from exc @@ -396,17 +1854,94 @@ def delete_file_folder( raise _http_error(exc) from exc +@router.get("/delta", response_model=FileDeltaResponse) +def files_delta( + owner_type: Literal["user", "group"] | None = None, + owner_id: str | None = None, + campaign_id: str | None = None, + path_prefix: str | None = None, + since: str | None = None, + limit: int = Query(default=500, ge=1, le=1000), + session: Session = Depends(get_session), + principal: ApiPrincipal = Depends(require_scope("files:file:read")), +): + _ensure_list_owner_access(session, principal, owner_type, owner_id) + _ensure_campaign_file_access(session, principal, campaign_id) + if since is None: + return _full_file_delta_response( + session, + principal=principal, + owner_type=owner_type, + owner_id=owner_id, + campaign_id=campaign_id, + path_prefix=path_prefix, + ) + return _files_delta_response( + session, + principal=principal, + owner_type=owner_type, + owner_id=owner_id, + campaign_id=campaign_id, + path_prefix=path_prefix, + since=since, + limit=limit, + ) + + @router.get("", response_model=FileListResponse) def list_files( owner_type: Literal["user", "group"] | None = None, owner_id: str | None = None, campaign_id: str | None = None, path_prefix: str | None = None, + page_size: int | None = Query(default=None, ge=1, le=1000), + cursor: str | None = None, session: Session = Depends(get_session), principal: ApiPrincipal = Depends(require_scope("files:file:read")), ): _ensure_list_owner_access(session, principal, owner_type, owner_id) _ensure_campaign_file_access(session, principal, campaign_id) + watermark = _files_delta_watermark(session, principal.tenant_id) + effective_page_size = _cursor_page_size(FILES_LIST_CURSOR_SCOPE, cursor, page_size) + if effective_page_size is not None: + fingerprint = _files_list_fingerprint( + principal, + owner_type=owner_type, + owner_id=owner_id, + campaign_id=campaign_id, + path_prefix=path_prefix, + page_size=effective_page_size, + ) + after_display_path, after_updated_at, after_id = _file_cursor_values(cursor, fingerprint=fingerprint) + assets, has_more = list_assets_for_user_window( + session, + tenant_id=principal.tenant_id, + user_id=principal.user.id, + owner_type=owner_type, + owner_id=owner_id, + campaign_id=campaign_id, + path_prefix=path_prefix, + is_admin=_is_admin(principal), + page_size=effective_page_size, + after_display_path=after_display_path, + after_updated_at=after_updated_at, + after_id=after_id, + ) + return FileListResponse( + files=_asset_list_response(session, assets, include_shares=True), + cursor=cursor, + next_cursor=_next_file_list_cursor( + principal, + assets, + owner_type=owner_type, + owner_id=owner_id, + campaign_id=campaign_id, + path_prefix=path_prefix, + page_size=effective_page_size, + has_more=has_more, + ), + watermark=watermark, + ) assets = list_assets_for_user( session, tenant_id=principal.tenant_id, @@ -417,7 +1952,7 @@ def list_files( path_prefix=path_prefix, is_admin=_is_admin(principal), ) - return FileListResponse(files=_asset_list_response(session, assets, include_shares=True)) + return FileListResponse(files=_asset_list_response(session, assets, include_shares=True), watermark=watermark) @router.post("/upload", response_model=FileUploadResponse) @@ -430,6 +1965,9 @@ async def upload_files( unpack_zip: bool = Form(default=False), conflict_strategy: Literal["reject", "overwrite", "rename"] = Form(default="reject"), conflict_resolutions_json: str | None = Form(default=None), + source_provenance_json: str | None = Form(default=None), + source_revision: str | None = Form(default=None), + connector_policy_json: str | None = Form(default=None), session: Session = Depends(get_session), principal: ApiPrincipal = Depends(require_scope("files:file:upload")), ): @@ -438,29 +1976,36 @@ async def upload_files( try: raw_resolutions = json.loads(conflict_resolutions_json) if conflict_resolutions_json else [] upload_resolutions = _conflict_resolutions([ConflictResolutionRequest(**item) for item in raw_resolutions]) + _enforce_connector_policy(source_provenance_json, connector_policy_json, operation="import") + metadata = _source_metadata_from_form(source_provenance_json, source_revision) for upload in files: filename = upload.filename or "file" content_type = upload.content_type or None upload_limit = settings.file_upload_zip_max_bytes if unpack_zip and filename.lower().endswith(".zip") else settings.file_upload_max_bytes - data = await _read_limited_upload(upload, max_bytes=upload_limit) if unpack_zip and filename.lower().endswith(".zip"): - extracted = extract_zip_upload( - session, - tenant_id=principal.tenant_id, - owner_type=owner_type, - owner_id=target_owner, - user_id=principal.user.id, - zip_data=data, - folder=path, - campaign_id=campaign_id, - conflict_strategy=conflict_strategy, - conflict_resolutions=upload_resolutions, - is_admin=_is_admin(principal), - max_file_bytes=settings.file_upload_max_bytes, - max_total_bytes=settings.file_upload_zip_max_bytes, - ) + zip_path = await _spool_limited_upload_to_temp(upload, max_bytes=upload_limit, suffix=".zip") + try: + extracted = extract_zip_upload( + session, + tenant_id=principal.tenant_id, + owner_type=owner_type, + owner_id=target_owner, + user_id=principal.user.id, + zip_data=zip_path, + folder=path, + campaign_id=campaign_id, + conflict_strategy=conflict_strategy, + conflict_resolutions=upload_resolutions, + metadata=metadata, + is_admin=_is_admin(principal), + max_file_bytes=settings.file_upload_max_bytes, + max_total_bytes=settings.file_upload_zip_max_bytes, + ) + finally: + _cleanup_temp_file(zip_path) uploaded_assets.extend(item.asset for item in extracted) continue + data = await _read_limited_upload(upload, max_bytes=upload_limit) stored = create_file_asset( session, tenant_id=principal.tenant_id, @@ -474,10 +2019,15 @@ async def upload_files( campaign_id=campaign_id, conflict_strategy=conflict_strategy, conflict_resolutions=upload_resolutions, + metadata=metadata, is_admin=_is_admin(principal), ) uploaded_assets.append(stored.asset) + _audit_connector_imports(session, principal, uploaded_assets) session.commit() + except ConnectorPolicyDenied as exc: + session.rollback() + raise _connector_policy_error(exc) from exc except (FileStorageError, UnsafeFilePathError, ValueError) as exc: session.rollback() raise _http_error(exc) from exc @@ -493,36 +2043,959 @@ async def upload_zip( campaign_id: str | None = Form(default=None), conflict_strategy: Literal["reject", "overwrite", "rename"] = Form(default="reject"), conflict_resolutions_json: str | None = Form(default=None), + source_provenance_json: str | None = Form(default=None), + source_revision: str | None = Form(default=None), + connector_policy_json: str | None = Form(default=None), session: Session = Depends(get_session), principal: ApiPrincipal = Depends(require_scope("files:file:upload")), ): - data = await _read_limited_upload(file, max_bytes=settings.file_upload_zip_max_bytes) target_owner = owner_id or principal.user.id + zip_path: str | None = None try: raw_resolutions = json.loads(conflict_resolutions_json) if conflict_resolutions_json else [] upload_resolutions = _conflict_resolutions([ConflictResolutionRequest(**item) for item in raw_resolutions]) + _enforce_connector_policy(source_provenance_json, connector_policy_json, operation="import") + metadata = _source_metadata_from_form(source_provenance_json, source_revision) + zip_path = await _spool_limited_upload_to_temp(file, max_bytes=settings.file_upload_zip_max_bytes, suffix=".zip") extracted = extract_zip_upload( session, tenant_id=principal.tenant_id, owner_type=owner_type, owner_id=target_owner, user_id=principal.user.id, - zip_data=data, + zip_data=zip_path, folder=path, campaign_id=campaign_id, conflict_strategy=conflict_strategy, conflict_resolutions=upload_resolutions, + metadata=metadata, is_admin=_is_admin(principal), max_file_bytes=settings.file_upload_max_bytes, max_total_bytes=settings.file_upload_zip_max_bytes, ) + _audit_connector_imports(session, principal, [item.asset for item in extracted]) session.commit() + except ConnectorPolicyDenied as exc: + session.rollback() + raise _connector_policy_error(exc) from exc except (FileStorageError, UnsafeFilePathError, ValueError) as exc: session.rollback() raise _http_error(exc) from exc + finally: + if zip_path: + _cleanup_temp_file(zip_path) return FileUploadResponse(files=[_asset_response(session, item.asset, include_shares=True) for item in extracted]) +@router.post("/connector-policy/evaluate", response_model=FileConnectorPolicyEvaluateResponse) +def evaluate_connector_policy( + payload: FileConnectorPolicyEvaluateRequest, + principal: ApiPrincipal = Depends(require_any_scope("files:file:read", "files:file:upload", "files:file:download")), +): + del principal + sources = connector_policy_sources_from_payload([item.model_dump(mode="json") for item in payload.policy_sources]) + request = ConnectorAccessRequest.from_provenance(payload.source_provenance.model_dump(mode="json", exclude_none=True), operation=payload.operation) + return FileConnectorPolicyEvaluateResponse(decision=connector_policy_decision(request, sources).to_dict()) + + +def _full_file_connector_settings_delta_response( + session: Session, + principal: ApiPrincipal, + *, + scope_type: str, + scope_id: str | None, + provider: str | None, + campaign_id: str | None, + include_disabled: bool, + include_inactive: bool, + owner_type: Literal["user", "group"] | None, + owner_id: str | None, +) -> FileConnectorSettingsDeltaResponse: + if campaign_id: + _ensure_campaign_file_access(session, principal, campaign_id) + profiles = _visible_connector_profiles( + session, + principal, + provider=provider, + campaign_id=campaign_id, + include_disabled=include_disabled and _can_read_disabled_connector_profiles(principal), + include_admin_scopes=_can_read_disabled_connector_profiles(principal), + include_effective_policy=False, + ) + credentials = _visible_connector_credentials( + session, + principal, + provider=provider, + include_disabled=include_disabled, + ) + spaces = _visible_connector_spaces( + session, + principal, + owner_type=owner_type, + owner_id=owner_id, + include_inactive=include_inactive, + ) + return FileConnectorSettingsDeltaResponse( + profiles=[FileConnectorProfileResponse(**profile.to_response()) for profile in profiles], + credentials=[FileConnectorCredentialResponse(**credential.to_response()) for credential in credentials], + spaces=[_connector_space_response(space) for space in spaces], + policy=FileConnectorPolicyResponse(**connector_policy_response(session, tenant_id=principal.tenant_id, scope_type=scope_type, scope_id=scope_id)), + changed_sections=["profiles", "credentials", "spaces", "policy"], + deleted=[], + watermark=_file_connector_settings_watermark(session, tenant_id=principal.tenant_id), + has_more=False, + full=True, + ) + + +@router.get("/connectors/settings/delta", response_model=FileConnectorSettingsDeltaResponse) +def connector_settings_delta( + scope_type: str = Query(default="tenant"), + scope_id: str | None = Query(default=None), + provider: str | None = None, + campaign_id: str | None = None, + include_disabled: bool = False, + include_inactive: bool = False, + owner_type: Literal["user", "group"] | None = None, + owner_id: str | None = None, + since: str | None = None, + limit: int = Query(default=100, ge=1, le=500), + session: Session = Depends(get_session), + principal: ApiPrincipal = Depends(require_any_scope("files:file:admin", "system:settings:read", "admin:settings:read")), +): + scope_type = scope_type.strip().casefold() + _require_connector_policy_read(principal, scope_type) + try: + if since is None: + return _full_file_connector_settings_delta_response( + session, + principal, + scope_type=scope_type, + scope_id=scope_id, + provider=provider, + campaign_id=campaign_id, + include_disabled=include_disabled, + include_inactive=include_inactive, + owner_type=owner_type, + owner_id=owner_id, + ) + entries, has_more = _file_connector_settings_entries(session, tenant_id=principal.tenant_id, since=since, limit=limit) + if entries is None: + return _full_file_connector_settings_delta_response( + session, + principal, + scope_type=scope_type, + scope_id=scope_id, + provider=provider, + campaign_id=campaign_id, + include_disabled=include_disabled, + include_inactive=include_inactive, + owner_type=owner_type, + owner_id=owner_id, + ) + changed_profile_ids = { + entry.resource_id + for entry in entries + if entry.collection == FILES_CONNECTOR_PROFILES_COLLECTION and entry.resource_type == FILES_CONNECTOR_PROFILE_RESOURCE + } + changed_credential_ids = { + entry.resource_id + for entry in entries + if entry.collection == FILES_CONNECTOR_CREDENTIALS_COLLECTION and entry.resource_type == FILES_CONNECTOR_CREDENTIAL_RESOURCE + } + changed_space_ids = { + entry.resource_id + for entry in entries + if entry.collection == FILES_CONNECTOR_SPACES_COLLECTION and entry.resource_type == FILES_CONNECTOR_SPACE_RESOURCE + } + policy_changed = any(entry.collection == FILES_CONNECTOR_POLICIES_COLLECTION for entry in entries) + profiles = _visible_connector_profiles( + session, + principal, + provider=provider, + campaign_id=campaign_id, + include_disabled=include_disabled and _can_read_disabled_connector_profiles(principal), + include_admin_scopes=_can_read_disabled_connector_profiles(principal), + include_effective_policy=False, + ) + visible_profiles = { + profile.id: profile + for profile in profiles + if profile.id in changed_profile_ids or (profile.credential_profile_id and profile.credential_profile_id in changed_credential_ids) + } + credentials = _visible_connector_credentials( + session, + principal, + provider=provider, + include_disabled=include_disabled, + ) + visible_credentials = { + credential.id: credential + for credential in credentials + if credential.id in changed_credential_ids + } + spaces = _visible_connector_spaces( + session, + principal, + owner_type=owner_type, + owner_id=owner_id, + include_inactive=include_inactive, + ) + visible_spaces = { + space.id: space + for space in spaces + if space.id in changed_space_ids + } + changed_sections = [] + if changed_profile_ids or any(profile.credential_profile_id and profile.credential_profile_id in changed_credential_ids for profile in profiles): + changed_sections.append("profiles") + if changed_credential_ids: + changed_sections.append("credentials") + if changed_space_ids: + changed_sections.append("spaces") + if policy_changed: + changed_sections.append("policy") + deleted = [ + _connector_deleted_item(entry) + for entry in entries + if ( + entry.resource_type == FILES_CONNECTOR_PROFILE_RESOURCE + and entry.resource_id not in visible_profiles + ) + or ( + entry.resource_type == FILES_CONNECTOR_CREDENTIAL_RESOURCE + and entry.resource_id not in visible_credentials + ) + or ( + entry.resource_type == FILES_CONNECTOR_SPACE_RESOURCE + and entry.resource_id not in visible_spaces + ) + ] + return FileConnectorSettingsDeltaResponse( + profiles=[FileConnectorProfileResponse(**profile.to_response()) for profile in visible_profiles.values()], + credentials=[FileConnectorCredentialResponse(**credential.to_response()) for credential in visible_credentials.values()], + spaces=[_connector_space_response(space) for space in visible_spaces.values()], + policy=FileConnectorPolicyResponse(**connector_policy_response(session, tenant_id=principal.tenant_id, scope_type=scope_type, scope_id=scope_id)) if policy_changed else None, + changed_sections=changed_sections, + deleted=deleted, + watermark=_file_connector_settings_response_watermark(session, tenant_id=principal.tenant_id, entries=entries, has_more=has_more), + has_more=has_more, + full=False, + ) + except (FileStorageError, ValueError, json.JSONDecodeError) as exc: + raise _http_error(exc) from exc + + +@router.get("/connectors/providers", response_model=FileConnectorProvidersResponse) +def list_connector_providers( + principal: ApiPrincipal = Depends(require_any_scope("files:file:read", "files:file:upload", "files:file:admin")), +): + del principal + return FileConnectorProvidersResponse(providers=[FileConnectorProviderResponse(**item.to_response()) for item in connector_provider_descriptors()]) + + +@router.post("/connectors/discover", response_model=FileConnectorDiscoveryResponse) +def discover_connector_endpoint( + payload: FileConnectorDiscoveryRequest, + principal: ApiPrincipal = Depends(require_any_scope("files:file:admin", "system:settings:write", "admin:settings:write")), +): + if payload.provider not in {"webdav", "nextcloud"}: + return FileConnectorDiscoveryResponse( + provider=payload.provider, + endpoint_url=None, + base_path=payload.base_path, + status="unsupported", + message=f"Discovery is not implemented for {payload.provider} connectors yet", + ) + candidates: list[dict[str, str]] = [] + for endpoint_url in _webdav_discovery_candidates(payload): + profile = _discovery_profile_from_payload(payload, endpoint_url, principal=principal) + try: + browse_connector_profile(profile, path=payload.base_path or "") + except (ConnectorBrowseError, ConnectorBrowseUnsupported, OSError, ValueError, json.JSONDecodeError) as exc: + message = str(exc) + if "credentials were rejected" in message.casefold(): + if payload.require_valid_credentials: + candidates.append({"endpoint_url": endpoint_url, "status": "credentials_rejected", "message": "The endpoint exists, but the credentials were rejected."}) + return FileConnectorDiscoveryResponse( + provider=payload.provider, + endpoint_url=endpoint_url, + base_path=payload.base_path, + status="credentials_rejected", + message="The endpoint was found, but login failed with these credentials.", + candidates=candidates, + metadata={**payload.metadata, "discovered_by": "webdav-auth-challenge"}, + ) + candidates.append({"endpoint_url": endpoint_url, "status": "found", "message": "The endpoint exists, but credentials are required or were rejected."}) + return FileConnectorDiscoveryResponse( + provider=payload.provider, + endpoint_url=endpoint_url, + base_path=payload.base_path, + status="found", + message="The endpoint was found. Add working credentials before saving or testing the connection.", + candidates=candidates, + metadata={**payload.metadata, "discovered_by": "webdav-auth-challenge"}, + ) + candidates.append({"endpoint_url": endpoint_url, "status": "failed", "message": message}) + continue + status_value = "usable" if _same_endpoint(endpoint_url, payload.endpoint_url) else "found" + message = "The supplied URL is directly usable." if status_value == "usable" else "A usable connector endpoint was discovered." + candidates.append({"endpoint_url": endpoint_url, "status": status_value, "message": message}) + return FileConnectorDiscoveryResponse( + provider=payload.provider, + endpoint_url=endpoint_url, + base_path=payload.base_path, + status=status_value, + message=message, + candidates=candidates, + metadata={**payload.metadata, "discovered_by": "webdav-propfind"}, + ) + return FileConnectorDiscoveryResponse( + provider=payload.provider, + endpoint_url=None, + base_path=payload.base_path, + status="not_found", + message="No usable WebDAV endpoint was found for this server URL.", + candidates=candidates, + ) + + +@router.get("/connectors/credentials", response_model=FileConnectorCredentialsResponse) +def list_connector_credentials( + provider: str | None = None, + include_disabled: bool = False, + session: Session = Depends(get_session), + principal: ApiPrincipal = Depends(require_any_scope("files:file:admin", "system:settings:read", "admin:settings:read")), +): + provider_norm = provider.strip().casefold() if provider else None + credentials = list_database_connector_credentials( + session, + tenant_id=principal.tenant_id, + include_disabled=include_disabled and _can_read_disabled_connector_profiles(principal), + ) + return FileConnectorCredentialsResponse( + credentials=[ + FileConnectorCredentialResponse(**credential.to_response()) + for credential in credentials + if provider_norm is None or credential.provider in {None, provider_norm} + ] + ) + + +@router.get("/connectors/policies/{scope_type}", response_model=FileConnectorPolicyResponse) +def read_connector_policy( + scope_type: str, + scope_id: str | None = Query(default=None), + session: Session = Depends(get_session), + principal: ApiPrincipal = Depends(require_any_scope("files:file:admin", "system:settings:read", "admin:settings:read")), +): + _require_connector_policy_read(principal, scope_type) + try: + return FileConnectorPolicyResponse(**connector_policy_response(session, tenant_id=principal.tenant_id, scope_type=scope_type, scope_id=scope_id)) + except (FileStorageError, ValueError, json.JSONDecodeError) as exc: + raise _http_error(exc, not_found=True) from exc + + +@router.put("/connectors/policies/{scope_type}", response_model=FileConnectorPolicyResponse) +def write_connector_policy( + scope_type: str, + payload: FileConnectorPolicyUpdateRequest, + scope_id: str | None = Query(default=None), + session: Session = Depends(get_session), + principal: ApiPrincipal = Depends(require_any_scope("files:file:admin", "system:settings:write", "admin:settings:write")), +): + _require_connector_profile_write(principal, scope_type) + try: + set_connector_policy( + session, + tenant_id=principal.tenant_id, + user_id=principal.user.id, + scope_type=scope_type, + scope_id=scope_id, + policy=payload.policy, + ) + _record_connector_settings_change( + session, + collection=FILES_CONNECTOR_POLICIES_COLLECTION, + resource_type=FILES_CONNECTOR_POLICY_RESOURCE, + resource_id=_file_connector_policy_resource_id(scope_type, scope_id), + operation="updated", + principal=principal, + tenant_id=None if scope_type.strip().casefold() == "system" else principal.tenant_id, + payload={"scope_type": scope_type.strip().casefold(), "scope_id": scope_id}, + ) + session.commit() + return FileConnectorPolicyResponse(**connector_policy_response(session, tenant_id=principal.tenant_id, scope_type=scope_type, scope_id=scope_id)) + except (FileStorageError, ValueError, json.JSONDecodeError) as exc: + session.rollback() + raise _http_error(exc) from exc + + +@router.post("/connectors/credentials", response_model=FileConnectorCredentialResponse, status_code=status.HTTP_201_CREATED) +def create_connector_credential( + payload: FileConnectorCredentialCreateRequest, + session: Session = Depends(get_session), + principal: ApiPrincipal = Depends(require_any_scope("files:file:admin", "system:settings:write", "admin:settings:write")), +): + _require_connector_credential_write(principal, payload.scope_type) + credentials = payload.credentials + try: + _ensure_connector_credential_configuration_allowed( + session, + principal, + credential_id=payload.id, + provider=payload.provider, + scope_type=payload.scope_type, + scope_id=payload.scope_id, + operation="configure_credentials", + ) + _ensure_connector_local_policy_allowed( + session, + principal, + scope_type=payload.scope_type, + scope_id=payload.scope_id, + policy=payload.policy, + ) + row = create_connector_credential_row( + session, + tenant_id=principal.tenant_id, + user_id=principal.user.id, + credential_id=payload.id, + label=payload.label, + provider=payload.provider, + scope_type=payload.scope_type, + scope_id=payload.scope_id, + enabled=payload.enabled, + credential_mode=payload.credential_mode, + username=credentials.username, + password=credentials.password, + token=credentials.token, + password_env=credentials.password_env, + token_env=credentials.token_env, + secret_ref=credentials.secret_ref, + policy=payload.policy, + metadata=payload.metadata, + ) + _record_connector_settings_change( + session, + collection=FILES_CONNECTOR_CREDENTIALS_COLLECTION, + resource_type=FILES_CONNECTOR_CREDENTIAL_RESOURCE, + resource_id=row.id, + operation="created", + principal=principal, + tenant_id=row.tenant_id, + payload={"scope_type": row.scope_type, "scope_id": row.scope_id, "provider": row.provider}, + ) + session.commit() + session.refresh(row) + return _connector_credential_response(row) + except ConnectorPolicyDenied as exc: + session.rollback() + raise _connector_policy_error(exc) from exc + except (FileStorageError, ValueError, json.JSONDecodeError) as exc: + session.rollback() + raise _http_error(exc) from exc + + +@router.get("/connectors/credentials/{credential_id}", response_model=FileConnectorCredentialResponse) +def get_connector_credential( + credential_id: str, + include_disabled: bool = False, + session: Session = Depends(get_session), + principal: ApiPrincipal = Depends(require_any_scope("files:file:admin", "system:settings:read", "admin:settings:read")), +): + try: + row = get_connector_credential_row( + session, + tenant_id=principal.tenant_id, + credential_id=credential_id, + include_disabled=include_disabled and _can_read_disabled_connector_profiles(principal), + ) + return _connector_credential_response(row) + except FileStorageError as exc: + raise _http_error(exc, not_found=True) from exc + + +@router.patch("/connectors/credentials/{credential_id}", response_model=FileConnectorCredentialResponse) +def update_connector_credential( + credential_id: str, + payload: FileConnectorCredentialUpdateRequest, + session: Session = Depends(get_session), + principal: ApiPrincipal = Depends(require_any_scope("files:file:admin", "system:settings:write", "admin:settings:write")), +): + try: + row = get_connector_credential_row(session, tenant_id=principal.tenant_id, credential_id=credential_id, include_disabled=True) + except FileStorageError as exc: + raise _http_error(exc, not_found=True) from exc + _require_connector_credential_write(principal, row.scope_type) + credentials = payload.credentials + try: + provider = payload.provider if payload.provider is not None else row.provider + _ensure_connector_credential_configuration_allowed( + session, + principal, + credential_id=row.id, + provider=provider, + scope_type=row.scope_type, + scope_id=row.scope_id, + operation="configure_credentials", + ) + if payload.policy is not None: + _ensure_connector_local_policy_allowed( + session, + principal, + scope_type=row.scope_type, + scope_id=row.scope_id, + policy=payload.policy, + ) + update_connector_credential_row( + session, + row, + user_id=principal.user.id, + label=payload.label, + provider=payload.provider, + enabled=payload.enabled, + credential_mode=payload.credential_mode, + username=credentials.username if credentials else None, + password=credentials.password if credentials else None, + token=credentials.token if credentials else None, + password_env=credentials.password_env if credentials else None, + token_env=credentials.token_env if credentials else None, + secret_ref=credentials.secret_ref if credentials else None, + clear_password=payload.clear_password, + clear_token=payload.clear_token, + policy=payload.policy, + metadata=payload.metadata, + ) + _record_connector_settings_change( + session, + collection=FILES_CONNECTOR_CREDENTIALS_COLLECTION, + resource_type=FILES_CONNECTOR_CREDENTIAL_RESOURCE, + resource_id=row.id, + operation="updated", + principal=principal, + tenant_id=row.tenant_id, + payload={"scope_type": row.scope_type, "scope_id": row.scope_id, "provider": row.provider}, + ) + session.commit() + session.refresh(row) + return _connector_credential_response(row) + except ConnectorPolicyDenied as exc: + session.rollback() + raise _connector_policy_error(exc) from exc + except (FileStorageError, ValueError, json.JSONDecodeError) as exc: + session.rollback() + raise _http_error(exc) from exc + + +@router.delete("/connectors/credentials/{credential_id}", response_model=FileConnectorCredentialResponse) +def deactivate_connector_credential( + credential_id: str, + session: Session = Depends(get_session), + principal: ApiPrincipal = Depends(require_any_scope("files:file:admin", "system:settings:write", "admin:settings:write")), +): + try: + row = get_connector_credential_row(session, tenant_id=principal.tenant_id, credential_id=credential_id, include_disabled=True) + except FileStorageError as exc: + raise _http_error(exc, not_found=True) from exc + _require_connector_credential_write(principal, row.scope_type) + try: + deactivate_connector_credential_row(session, row, user_id=principal.user.id) + _record_connector_settings_change( + session, + collection=FILES_CONNECTOR_CREDENTIALS_COLLECTION, + resource_type=FILES_CONNECTOR_CREDENTIAL_RESOURCE, + resource_id=row.id, + operation="deleted", + principal=principal, + tenant_id=row.tenant_id, + payload={"scope_type": row.scope_type, "scope_id": row.scope_id, "provider": row.provider}, + ) + session.commit() + session.refresh(row) + return _connector_credential_response(row) + except FileStorageError as exc: + session.rollback() + raise _http_error(exc) from exc + + +@router.get("/connectors/profiles", response_model=FileConnectorProfilesResponse) +def list_connector_profiles( + provider: str | None = None, + campaign_id: str | None = None, + include_disabled: bool = False, + session: Session = Depends(get_session), + principal: ApiPrincipal = Depends(require_any_scope("files:file:read", "files:file:upload", "files:file:download", "files:file:admin", "system:settings:read", "admin:settings:read")), +): + try: + if campaign_id: + _ensure_campaign_file_access(session, principal, campaign_id) + profiles = _visible_connector_profiles( + session, + principal, + provider=provider, + campaign_id=campaign_id, + include_disabled=include_disabled and _can_read_disabled_connector_profiles(principal), + include_admin_scopes=_can_read_disabled_connector_profiles(principal), + include_effective_policy=False, + ) + except (OSError, ValueError, json.JSONDecodeError) as exc: + raise _http_error(exc) from exc + return FileConnectorProfilesResponse(profiles=[FileConnectorProfileResponse(**profile.to_response()) for profile in profiles]) + + +@router.post("/connectors/profiles", response_model=FileConnectorProfileResponse, status_code=status.HTTP_201_CREATED) +def create_connector_profile( + payload: FileConnectorProfileCreateRequest, + session: Session = Depends(get_session), + principal: ApiPrincipal = Depends(require_any_scope("files:file:admin", "system:settings:write", "admin:settings:write")), +): + _require_connector_profile_write(principal, payload.scope_type) + credentials = payload.credentials + try: + credential_row = _credential_row_for_profile( + session, + principal, + credential_profile_id=payload.credential_profile_id, + provider=payload.provider, + ) + _ensure_connector_configuration_allowed( + session, + principal, + connector_id=payload.id, + credential_id=payload.credential_profile_id, + provider=payload.provider, + endpoint_url=payload.endpoint_url, + base_path=payload.base_path, + scope_type=payload.scope_type, + scope_id=payload.scope_id, + operation="configure", + ) + _ensure_connector_local_policy_allowed( + session, + principal, + scope_type=payload.scope_type, + scope_id=payload.scope_id, + policy=payload.policy, + ) + row = create_connector_profile_row( + session, + tenant_id=principal.tenant_id, + user_id=principal.user.id, + profile_id=payload.id, + label=payload.label, + provider=payload.provider, + scope_type=payload.scope_type, + scope_id=payload.scope_id, + endpoint_url=payload.endpoint_url, + base_path=payload.base_path, + enabled=payload.enabled, + credential_profile_id=payload.credential_profile_id, + credential_mode=payload.credential_mode, + username=credentials.username, + password=credentials.password, + token=credentials.token, + password_env=credentials.password_env, + token_env=credentials.token_env, + secret_ref=credentials.secret_ref, + capabilities=payload.capabilities, + policy=payload.policy, + metadata=payload.metadata, + ) + _record_connector_settings_change( + session, + collection=FILES_CONNECTOR_PROFILES_COLLECTION, + resource_type=FILES_CONNECTOR_PROFILE_RESOURCE, + resource_id=row.id, + operation="created", + principal=principal, + tenant_id=row.tenant_id, + payload={"scope_type": row.scope_type, "scope_id": row.scope_id, "provider": row.provider}, + ) + session.commit() + session.refresh(row) + return FileConnectorProfileResponse(**connector_profile_from_row(row, credential_row=credential_row).to_response()) + except ConnectorPolicyDenied as exc: + session.rollback() + raise _connector_policy_error(exc) from exc + except (FileStorageError, ValueError, json.JSONDecodeError) as exc: + session.rollback() + raise _http_error(exc) from exc + + +@router.post("/connectors/profiles/{profile_id}/import", response_model=FileUploadResponse) +def import_connector_file( + profile_id: str, + payload: FileConnectorImportRequest, + session: Session = Depends(get_session), + principal: ApiPrincipal = Depends(require_scope("files:file:upload")), +): + try: + if payload.campaign_id: + _ensure_campaign_file_access(session, principal, payload.campaign_id) + profile = _visible_connector_profile(session, principal, profile_id, campaign_id=payload.campaign_id) + _source_path, downloaded, metadata = _download_connector_payload(profile, payload, operation="import") + target_owner = payload.owner_id or principal.user.id + stored = create_file_asset( + session, + tenant_id=principal.tenant_id, + owner_type=payload.owner_type, + owner_id=target_owner, + user_id=principal.user.id, + filename=downloaded.filename, + data=downloaded.data, + folder=payload.target_folder, + display_path=payload.target_path, + content_type=downloaded.content_type, + metadata=metadata, + campaign_id=payload.campaign_id, + conflict_strategy=payload.conflict_strategy, + is_admin=_is_admin(principal), + ) + _audit_connector_imports(session, principal, [stored.asset]) + session.commit() + except ConnectorPolicyDenied as exc: + session.rollback() + raise _connector_policy_error(exc) from exc + except ConnectorImportUnsupported as exc: + session.rollback() + raise HTTPException(status_code=status.HTTP_501_NOT_IMPLEMENTED, detail=str(exc)) from exc + except (ConnectorImportError, FileStorageError, UnsafeFilePathError, ValueError, json.JSONDecodeError) as exc: + session.rollback() + raise _http_error(exc) from exc + return FileUploadResponse(files=[_asset_response(session, stored.asset, include_shares=True)]) + + +@router.post("/connectors/profiles/{profile_id}/sync", response_model=FileConnectorSyncResponse) +def sync_connector_file( + profile_id: str, + payload: FileConnectorImportRequest, + session: Session = Depends(get_session), + principal: ApiPrincipal = Depends(require_scope("files:file:upload")), +): + try: + if payload.campaign_id: + _ensure_campaign_file_access(session, principal, payload.campaign_id) + profile = _visible_connector_profile(session, principal, profile_id, campaign_id=payload.campaign_id) + _source_path, downloaded, metadata = _download_connector_payload(profile, payload, operation="sync") + target_owner = payload.owner_id or principal.user.id + stored, sync_action, previous_version_id = sync_file_asset_from_source( + session, + tenant_id=principal.tenant_id, + owner_type=payload.owner_type, + owner_id=target_owner, + user_id=principal.user.id, + filename=downloaded.filename, + data=downloaded.data, + folder=payload.target_folder, + display_path=payload.target_path, + content_type=downloaded.content_type, + metadata=metadata, + campaign_id=payload.campaign_id, + conflict_strategy=payload.conflict_strategy, + is_admin=_is_admin(principal), + ) + _audit_connector_sync(session, principal, stored.asset, sync_action=sync_action, previous_version_id=previous_version_id) + session.commit() + except ConnectorPolicyDenied as exc: + session.rollback() + raise _connector_policy_error(exc) from exc + except ConnectorImportUnsupported as exc: + session.rollback() + raise HTTPException(status_code=status.HTTP_501_NOT_IMPLEMENTED, detail=str(exc)) from exc + except (ConnectorImportError, FileStorageError, UnsafeFilePathError, ValueError, json.JSONDecodeError) as exc: + session.rollback() + raise _http_error(exc) from exc + return FileConnectorSyncResponse( + file=_asset_response(session, stored.asset, include_shares=True), + action=sync_action, + previous_version_id=previous_version_id, + current_version_id=stored.version.id, + ) + + +@router.get("/connectors/profiles/{profile_id}/browse", response_model=FileConnectorBrowseResponse) +def browse_connector_profile_items( + profile_id: str, + path: str | None = None, + library_id: str | None = None, + campaign_id: str | None = None, + session: Session = Depends(get_session), + principal: ApiPrincipal = Depends(require_any_scope("files:file:read", "files:file:upload", "files:file:download", "files:file:admin", "system:settings:read", "admin:settings:read")), +): + try: + profile = _visible_connector_profile(session, principal, profile_id, campaign_id=campaign_id) + browse_path = normalize_connector_browse_path(path) + decision = connector_policy_decision( + ConnectorAccessRequest( + connector_id=profile.id, + credential_id=profile.credential_profile_id, + provider=profile.provider, + external_path=browse_path, + external_url=profile.endpoint_url, + operation="browse", + ), + profile.policy_sources, + ) + if not decision.allowed: + raise ConnectorPolicyDenied(decision) + items = browse_connector_profile(profile, path=browse_path, library_id=library_id) + except ConnectorPolicyDenied as exc: + raise _connector_policy_error(exc) from exc + except ConnectorBrowseUnsupported as exc: + raise HTTPException(status_code=status.HTTP_501_NOT_IMPLEMENTED, detail=str(exc)) from exc + except (ConnectorBrowseError, OSError, ValueError, json.JSONDecodeError) as exc: + raise _http_error(exc) from exc + return FileConnectorBrowseResponse( + profile_id=profile.id, + provider=profile.provider, + path=browse_path, + library_id=library_id, + decision=decision.to_dict(), + items=[FileConnectorBrowseItem(**item.to_response()) for item in items], + ) + + +@router.get("/connectors/profiles/{profile_id}", response_model=FileConnectorProfileResponse) +def get_connector_profile( + profile_id: str, + campaign_id: str | None = None, + include_disabled: bool = False, + session: Session = Depends(get_session), + principal: ApiPrincipal = Depends(require_any_scope("files:file:read", "files:file:upload", "files:file:download", "files:file:admin", "system:settings:read", "admin:settings:read")), +): + try: + profile = _visible_connector_profile( + session, + principal, + profile_id, + campaign_id=campaign_id, + include_disabled=include_disabled and _can_read_disabled_connector_profiles(principal), + include_effective_policy=False, + ) + except (OSError, ValueError, json.JSONDecodeError) as exc: + raise _http_error(exc) from exc + return FileConnectorProfileResponse(**profile.to_response()) + + +@router.patch("/connectors/profiles/{profile_id}", response_model=FileConnectorProfileResponse) +def update_connector_profile( + profile_id: str, + payload: FileConnectorProfileUpdateRequest, + session: Session = Depends(get_session), + principal: ApiPrincipal = Depends(require_any_scope("files:file:admin", "system:settings:write", "admin:settings:write")), +): + try: + row = get_connector_profile_row(session, tenant_id=principal.tenant_id, profile_id=profile_id, include_disabled=True) + except FileStorageError as exc: + raise _http_error(exc, not_found=True) from exc + _require_connector_profile_write(principal, row.scope_type) + credentials = payload.credentials + try: + credential_profile_id = payload.credential_profile_id if payload.credential_profile_id is not None else row.credential_profile_id + provider = payload.provider if payload.provider is not None else row.provider + credential_row = _credential_row_for_profile( + session, + principal, + credential_profile_id=credential_profile_id, + provider=provider, + include_disabled=True, + ) + _ensure_connector_configuration_allowed( + session, + principal, + connector_id=row.id, + credential_id=credential_profile_id, + provider=provider, + endpoint_url=payload.endpoint_url if payload.endpoint_url is not None else row.endpoint_url, + base_path=payload.base_path if payload.base_path is not None else row.base_path, + scope_type=row.scope_type, + scope_id=row.scope_id, + operation="configure", + ) + if payload.policy is not None: + _ensure_connector_local_policy_allowed( + session, + principal, + scope_type=row.scope_type, + scope_id=row.scope_id, + policy=payload.policy, + ) + update_connector_profile_row( + session, + row, + user_id=principal.user.id, + label=payload.label, + provider=payload.provider, + endpoint_url=payload.endpoint_url, + base_path=payload.base_path, + enabled=payload.enabled, + credential_profile_id=payload.credential_profile_id, + credential_mode=payload.credential_mode, + username=credentials.username if credentials else None, + password=credentials.password if credentials else None, + token=credentials.token if credentials else None, + password_env=credentials.password_env if credentials else None, + token_env=credentials.token_env if credentials else None, + secret_ref=credentials.secret_ref if credentials else None, + clear_password=payload.clear_password, + clear_token=payload.clear_token, + capabilities=payload.capabilities, + policy=payload.policy, + metadata=payload.metadata, + ) + _record_connector_settings_change( + session, + collection=FILES_CONNECTOR_PROFILES_COLLECTION, + resource_type=FILES_CONNECTOR_PROFILE_RESOURCE, + resource_id=row.id, + operation="updated", + principal=principal, + tenant_id=row.tenant_id, + payload={"scope_type": row.scope_type, "scope_id": row.scope_id, "provider": row.provider}, + ) + session.commit() + session.refresh(row) + return FileConnectorProfileResponse(**connector_profile_from_row(row, credential_row=credential_row).to_response()) + except ConnectorPolicyDenied as exc: + session.rollback() + raise _connector_policy_error(exc) from exc + except (FileStorageError, ValueError, json.JSONDecodeError) as exc: + session.rollback() + raise _http_error(exc) from exc + + +@router.delete("/connectors/profiles/{profile_id}", response_model=FileConnectorProfileResponse) +def deactivate_connector_profile( + profile_id: str, + session: Session = Depends(get_session), + principal: ApiPrincipal = Depends(require_any_scope("files:file:admin", "system:settings:write", "admin:settings:write")), +): + try: + row = get_connector_profile_row(session, tenant_id=principal.tenant_id, profile_id=profile_id, include_disabled=True) + except FileStorageError as exc: + raise _http_error(exc, not_found=True) from exc + _require_connector_profile_write(principal, row.scope_type) + try: + deactivate_connector_profile_row(session, row, user_id=principal.user.id) + _record_connector_settings_change( + session, + collection=FILES_CONNECTOR_PROFILES_COLLECTION, + resource_type=FILES_CONNECTOR_PROFILE_RESOURCE, + resource_id=row.id, + operation="deleted", + principal=principal, + tenant_id=row.tenant_id, + payload={"scope_type": row.scope_type, "scope_id": row.scope_id, "provider": row.provider}, + ) + session.commit() + session.refresh(row) + return FileConnectorProfileResponse(**connector_profile_from_row(row).to_response()) + except FileStorageError as exc: + session.rollback() + raise _http_error(exc) from exc + + @router.get("/{file_id}", response_model=FileAssetResponse) def get_file( file_id: str, @@ -544,7 +3017,17 @@ def download_file( ): try: asset = get_asset_for_user(session, tenant_id=principal.tenant_id, user_id=principal.user.id, asset_id=file_id, is_admin=_is_admin(principal)) - data, _, blob = read_asset_bytes(session, asset) + data, version, blob = read_asset_bytes(session, asset) + _audit_connector_event( + session, + principal, + action="files.connector.accessed", + asset=asset, + version=version, + blob=blob, + operation="download", + commit=True, + ) except FileStorageError as exc: raise _http_error(exc, not_found=True) from exc headers = {"Content-Disposition": _attachment_disposition(asset.filename)} @@ -755,6 +3238,7 @@ def download_archive( except Exception: _cleanup_temp_file(tmp_path) raise + _audit_connector_access(session, principal, assets, operation="archive") except FileStorageError as exc: raise _http_error(exc) from exc filename = filename_from_path(normalize_logical_path(payload.filename, fallback_filename="files.zip")) diff --git a/src/govoplan_files/backend/schemas.py b/src/govoplan_files/backend/schemas.py index c373e41..bd4496e 100644 --- a/src/govoplan_files/backend/schemas.py +++ b/src/govoplan_files/backend/schemas.py @@ -4,6 +4,7 @@ from typing import Any, Literal from pydantic import BaseModel, Field +from govoplan_core.api.v1.schemas import DeltaDeletedItem from govoplan_files.backend.storage.common import FileConflictResolution @@ -13,12 +14,63 @@ class FileSpaceResponse(BaseModel): owner_type: Literal["user", "group"] owner_id: str description: str | None = None + space_type: Literal["managed", "connector"] = "managed" + connector_space_id: str | None = None + connector_profile_id: str | None = None + provider: str | None = None + library_id: str | None = None + remote_path: str | None = None + sync_mode: str | None = None + read_only: bool = False class FileSpacesResponse(BaseModel): spaces: list[FileSpaceResponse] +class FileConnectorSpaceCreateRequest(BaseModel): + owner_type: Literal["user", "group"] = "user" + owner_id: str | None = None + label: str + connector_profile_id: str + library_id: str | None = None + remote_path: str = "" + sync_mode: Literal["manual"] = "manual" + metadata: dict[str, Any] = Field(default_factory=dict) + + +class FileConnectorSpaceUpdateRequest(BaseModel): + label: str | None = None + library_id: str | None = None + remote_path: str | None = None + sync_mode: Literal["manual"] | None = None + is_active: bool | None = None + metadata: dict[str, Any] | None = None + + +class FileConnectorSpaceResponse(BaseModel): + id: str + tenant_id: str + owner_type: Literal["user", "group"] + owner_id: str + label: str + connector_profile_id: str + provider: str + library_id: str | None = None + remote_path: str = "" + sync_mode: str + read_only: bool + is_active: bool + created_at: str + updated_at: str + deleted_at: str | None = None + metadata: dict[str, Any] = Field(default_factory=dict) + + +class FileConnectorSpacesResponse(BaseModel): + spaces: list[FileConnectorSpaceResponse] = Field(default_factory=list) + + class FileShareResponse(BaseModel): id: str target_type: str @@ -28,6 +80,20 @@ class FileShareResponse(BaseModel): revoked_at: str | None = None +class FileSourceProvenance(BaseModel): + source_type: str | None = None + connector_id: str | None = None + provider: str | None = None + external_id: str | None = None + external_path: str | None = None + external_url: str | None = None + revision: str | None = None + revision_label: str | None = None + observed_at: str | None = None + imported_at: str | None = None + metadata: dict[str, Any] = Field(default_factory=dict) + + class FileAssetResponse(BaseModel): id: str tenant_id: str @@ -45,6 +111,8 @@ class FileAssetResponse(BaseModel): deleted_at: str | None = None audit_relevant: bool = False metadata: dict[str, Any] | None = None + source_provenance: FileSourceProvenance | None = None + source_revision: str | None = None shares: list[FileShareResponse] = Field(default_factory=list) @@ -61,6 +129,9 @@ class FileFolderResponse(BaseModel): class FileFoldersResponse(BaseModel): folders: list[FileFolderResponse] + cursor: str | None = None + next_cursor: str | None = None + watermark: str | None = None class FileFolderCreateRequest(BaseModel): @@ -83,12 +154,280 @@ class FileFolderDeleteResponse(BaseModel): class FileListResponse(BaseModel): files: list[FileAssetResponse] + cursor: str | None = None + next_cursor: str | None = None + watermark: str | None = None + + +class FileDeltaResponse(BaseModel): + files: list[FileAssetResponse] = Field(default_factory=list) + folders: list[FileFolderResponse] = Field(default_factory=list) + deleted: list[DeltaDeletedItem] = Field(default_factory=list) + watermark: str | None = None + has_more: bool = False + full: bool = False class FileUploadResponse(BaseModel): files: list[FileAssetResponse] +class FileConnectorPolicySource(BaseModel): + scope_type: Literal["system", "tenant", "user", "group", "campaign"] = "system" + scope_id: str | None = None + label: str | None = None + policy: dict[str, Any] = Field(default_factory=dict) + + +class FileConnectorPolicyEvaluateRequest(BaseModel): + source_provenance: FileSourceProvenance + operation: str = "access" + policy_sources: list[FileConnectorPolicySource] = Field(default_factory=list) + + +class FileConnectorPolicyEvaluateResponse(BaseModel): + decision: dict[str, Any] + + +class FileConnectorPolicyUpdateRequest(BaseModel): + policy: dict[str, Any] = Field(default_factory=dict) + + +class FileConnectorPolicyStepResponse(BaseModel): + scope_type: str + scope_id: str | None = None + path: str + label: str + applied_fields: list[str] = Field(default_factory=list) + policy: dict[str, Any] = Field(default_factory=dict) + + +class FileConnectorPolicyResponse(BaseModel): + scope_type: Literal["system", "tenant", "user", "group", "campaign"] + scope_id: str | None = None + policy: dict[str, Any] = Field(default_factory=dict) + effective_policy: dict[str, Any] | None = None + parent_policy: dict[str, Any] | None = None + effective_policy_sources: list[FileConnectorPolicyStepResponse] = Field(default_factory=list) + parent_policy_sources: list[FileConnectorPolicyStepResponse] = Field(default_factory=list) + + +class FileConnectorProfileResponse(BaseModel): + id: str + label: str + provider: str + endpoint_url: str | None = None + base_path: str | None = None + enabled: bool = True + scope_type: Literal["system", "tenant", "user", "group", "campaign"] = "system" + scope_id: str | None = None + source_path: str + credential_profile_id: str | None = None + credential_profile_label: str | None = None + credential_mode: str = "none" + credential_source: str | None = None + credentials_configured: bool = False + username: str | None = None + capabilities: list[str] = Field(default_factory=list) + policy_sources: list[FileConnectorPolicySource] = Field(default_factory=list) + metadata: dict[str, Any] = Field(default_factory=dict) + source_kind: str = "settings" + + +class FileConnectorProfilesResponse(BaseModel): + profiles: list[FileConnectorProfileResponse] + + +class FileConnectorCredentialResponse(BaseModel): + id: str + label: str + provider: str | None = None + enabled: bool = True + scope_type: Literal["system", "tenant", "user", "group", "campaign"] = "system" + scope_id: str | None = None + source_path: str + credential_mode: str = "none" + credential_secret_source: str | None = None + credentials_configured: bool = False + username: str | None = None + policy_sources: list[FileConnectorPolicySource] = Field(default_factory=list) + metadata: dict[str, Any] = Field(default_factory=dict) + source_kind: str = "database" + + +class FileConnectorCredentialsResponse(BaseModel): + credentials: list[FileConnectorCredentialResponse] + + +class FileConnectorSettingsDeltaResponse(BaseModel): + profiles: list[FileConnectorProfileResponse] = Field(default_factory=list) + credentials: list[FileConnectorCredentialResponse] = Field(default_factory=list) + spaces: list[FileConnectorSpaceResponse] = Field(default_factory=list) + policy: FileConnectorPolicyResponse | None = None + changed_sections: list[str] = Field(default_factory=list) + deleted: list[DeltaDeletedItem] = Field(default_factory=list) + watermark: str | None = None + has_more: bool = False + full: bool = False + + +class FileConnectorProfileCredentialsRequest(BaseModel): + username: str | None = None + password: str | None = None + token: str | None = None + password_env: str | None = None + token_env: str | None = None + secret_ref: str | None = None + + +class FileConnectorDiscoveryCandidate(BaseModel): + endpoint_url: str + status: Literal["failed", "usable", "found", "credentials_rejected"] + message: str + + +class FileConnectorDiscoveryRequest(BaseModel): + provider: Literal["seafile", "nextcloud", "webdav", "smb", "nfs", "dms", "generic"] + endpoint_url: str + base_path: str | None = None + credential_mode: Literal["none", "anonymous", "basic", "token", "secret_ref"] = "none" + credentials: FileConnectorProfileCredentialsRequest = Field(default_factory=FileConnectorProfileCredentialsRequest) + metadata: dict[str, Any] = Field(default_factory=dict) + require_valid_credentials: bool = False + + +class FileConnectorDiscoveryResponse(BaseModel): + provider: str + endpoint_url: str | None = None + base_path: str | None = None + status: Literal["usable", "found", "credentials_rejected", "not_found", "unsupported"] + message: str + candidates: list[FileConnectorDiscoveryCandidate] = Field(default_factory=list) + metadata: dict[str, Any] = Field(default_factory=dict) + + +class FileConnectorProfileCreateRequest(BaseModel): + id: str + label: str + provider: Literal["seafile", "nextcloud", "webdav", "smb", "nfs", "dms", "generic"] + endpoint_url: str | None = None + base_path: str | None = None + enabled: bool = True + scope_type: Literal["system", "tenant", "user", "group", "campaign"] = "tenant" + scope_id: str | None = None + credential_profile_id: str | None = None + credential_mode: Literal["none", "anonymous", "basic", "token", "secret_ref"] = "none" + credentials: FileConnectorProfileCredentialsRequest = Field(default_factory=FileConnectorProfileCredentialsRequest) + capabilities: list[str] = Field(default_factory=lambda: ["browse", "import", "sync"]) + policy: dict[str, Any] = Field(default_factory=dict) + metadata: dict[str, Any] = Field(default_factory=dict) + + +class FileConnectorProfileUpdateRequest(BaseModel): + label: str | None = None + provider: Literal["seafile", "nextcloud", "webdav", "smb", "nfs", "dms", "generic"] | None = None + endpoint_url: str | None = None + base_path: str | None = None + enabled: bool | None = None + credential_profile_id: str | None = None + credential_mode: Literal["none", "anonymous", "basic", "token", "secret_ref"] | None = None + credentials: FileConnectorProfileCredentialsRequest | None = None + clear_password: bool = False + clear_token: bool = False + capabilities: list[str] | None = None + policy: dict[str, Any] | None = None + metadata: dict[str, Any] | None = None + + +class FileConnectorCredentialCreateRequest(BaseModel): + id: str + label: str + provider: Literal["seafile", "nextcloud", "webdav", "smb", "nfs", "dms", "generic"] | None = None + enabled: bool = True + scope_type: Literal["system", "tenant", "user", "group", "campaign"] = "tenant" + scope_id: str | None = None + credential_mode: Literal["none", "anonymous", "basic", "token", "secret_ref"] = "none" + credentials: FileConnectorProfileCredentialsRequest = Field(default_factory=FileConnectorProfileCredentialsRequest) + policy: dict[str, Any] = Field(default_factory=dict) + metadata: dict[str, Any] = Field(default_factory=dict) + + +class FileConnectorCredentialUpdateRequest(BaseModel): + label: str | None = None + provider: Literal["seafile", "nextcloud", "webdav", "smb", "nfs", "dms", "generic"] | None = None + enabled: bool | None = None + credential_mode: Literal["none", "anonymous", "basic", "token", "secret_ref"] | None = None + credentials: FileConnectorProfileCredentialsRequest | None = None + clear_password: bool = False + clear_token: bool = False + policy: dict[str, Any] | None = None + metadata: dict[str, Any] | None = None + + +class FileConnectorProviderResponse(BaseModel): + provider: str + label: str + protocol: str + implemented: bool + installed: bool + browse_supported: bool + import_supported: bool + optional_dependency: str | None = None + permission_model: str + sync_strategy: str + conflict_strategy: str + preview_strategy: str + audit_events: list[str] = Field(default_factory=list) + notes: str | None = None + + +class FileConnectorProvidersResponse(BaseModel): + providers: list[FileConnectorProviderResponse] + + +class FileConnectorBrowseItem(BaseModel): + kind: Literal["library", "folder", "file"] + name: str + path: str + external_id: str | None = None + external_url: str | None = None + size_bytes: int | None = None + content_type: str | None = None + modified_at: str | None = None + etag: str | None = None + metadata: dict[str, Any] = Field(default_factory=dict) + + +class FileConnectorBrowseResponse(BaseModel): + profile_id: str + provider: str + path: str = "" + library_id: str | None = None + read_only: bool = True + decision: dict[str, Any] + items: list[FileConnectorBrowseItem] + + +class FileConnectorImportRequest(BaseModel): + library_id: str + path: str + owner_type: Literal["user", "group"] = "user" + owner_id: str | None = None + target_folder: str | None = None + target_path: str | None = None + campaign_id: str | None = None + conflict_strategy: Literal["reject", "overwrite", "rename"] = "reject" + source_revision: str | None = None + metadata: dict[str, Any] = Field(default_factory=dict) + + +class FileConnectorSyncResponse(BaseModel): + file: FileAssetResponse + action: Literal["created", "updated", "unchanged"] + previous_version_id: str | None = None + current_version_id: str + + class BulkDeleteRequest(BaseModel): file_ids: list[str] @@ -128,8 +467,8 @@ class BulkFileShareResponse(BaseModel): class RenameRequest(BaseModel): file_ids: list[str] = Field(default_factory=list) folder_paths: list[str] = Field(default_factory=list) - owner_type: Literal["user", "group"] | None = None - owner_id: str | None = None + owner_type: Literal["user", "group"] + owner_id: str mode: Literal["direct", "prefix", "suffix", "replace"] new_name: str | None = None find: str | None = None diff --git a/src/govoplan_files/backend/storage/archives.py b/src/govoplan_files/backend/storage/archives.py index 64ab773..4a0695c 100644 --- a/src/govoplan_files/backend/storage/archives.py +++ b/src/govoplan_files/backend/storage/archives.py @@ -3,8 +3,9 @@ from __future__ import annotations import mimetypes import zipfile from io import BytesIO +from os import PathLike from pathlib import Path -from typing import Iterable +from typing import Any, Iterable from sqlalchemy.orm import Session @@ -68,11 +69,12 @@ def extract_zip_upload( owner_type: str, owner_id: str, user_id: str, - zip_data: bytes, + zip_data: bytes | str | PathLike[str], folder: str | None, campaign_id: str | None, conflict_strategy: str = "reject", conflict_resolutions: Iterable[FileConflictResolution] | None = None, + metadata: dict[str, Any] | None = None, is_admin: bool = False, max_files: int = 1000, max_file_bytes: int = 50 * 1024 * 1024, @@ -82,7 +84,8 @@ def extract_zip_upload( total = 0 base_folder = normalize_folder(folder) try: - with zipfile.ZipFile(BytesIO(zip_data)) as archive: + source = BytesIO(zip_data) if isinstance(zip_data, bytes) else zip_data + with zipfile.ZipFile(source) as archive: infos = [info for info in archive.infolist() if not info.is_dir()] if len(infos) > max_files: raise FileStorageError(f"ZIP contains too many files (limit {max_files})") @@ -115,6 +118,7 @@ def extract_zip_upload( data=data, display_path=target_path, content_type=mimetypes.guess_type(inner_path)[0] or "application/octet-stream", + metadata=metadata, campaign_id=campaign_id, conflict_strategy=conflict_strategy, conflict_resolutions=conflict_resolutions, diff --git a/src/govoplan_files/backend/storage/campaign_attachments.py b/src/govoplan_files/backend/storage/campaign_attachments.py index 9897c2a..09ed152 100644 --- a/src/govoplan_files/backend/storage/campaign_attachments.py +++ b/src/govoplan_files/backend/storage/campaign_attachments.py @@ -17,6 +17,7 @@ from govoplan_files.backend.storage.backends import StorageBackendError, get_sto from govoplan_files.backend.storage.common import FileStorageError from govoplan_files.backend.storage.files import current_versions_and_blobs, list_assets_for_user from govoplan_files.backend.storage.paths import normalize_folder, normalize_logical_path, safe_storage_component +from govoplan_files.backend.storage.provenance import source_provenance_from_metadata, source_revision_from_metadata MANAGED_SOURCE_PREFIX = "managed:" @@ -36,10 +37,16 @@ class ManagedAttachmentFile: checksum_sha256: str size_bytes: int content_type: str | None + source_provenance: dict[str, Any] | None = None + source_revision: str | None = None def as_dict(self) -> dict[str, Any]: payload = asdict(self) payload.pop("local_path", None) + if payload.get("source_provenance") is None: + payload.pop("source_provenance", None) + if payload.get("source_revision") is None: + payload.pop("source_revision", None) return payload @@ -233,6 +240,8 @@ def prepare_campaign_snapshot( checksum_sha256=blob.checksum_sha256, size_bytes=blob.size_bytes, content_type=blob.content_type, + source_provenance=source_provenance_from_metadata(asset.metadata_), + source_revision=source_revision_from_metadata(asset.metadata_), ) for rule in _iter_rule_dicts(attachments, prepared_json): diff --git a/src/govoplan_files/backend/storage/connector_browse.py b/src/govoplan_files/backend/storage/connector_browse.py new file mode 100644 index 0000000..0047940 --- /dev/null +++ b/src/govoplan_files/backend/storage/connector_browse.py @@ -0,0 +1,600 @@ +from __future__ import annotations + +import os +from collections.abc import Mapping +from dataclasses import dataclass, field +from datetime import datetime, timezone +from email.utils import parsedate_to_datetime +from importlib import import_module +import mimetypes +from typing import Any +from urllib.parse import quote, unquote, urljoin, urlsplit +import xml.etree.ElementTree as ET + +import httpx + +from govoplan_files.backend.storage.connector_profiles import ConnectorProfile + + +class ConnectorBrowseError(RuntimeError): + pass + + +class ConnectorBrowseUnsupported(ConnectorBrowseError): + pass + + +@dataclass(frozen=True, slots=True) +class ConnectorBrowseItem: + kind: str + name: str + path: str + external_id: str | None = None + external_url: str | None = None + size_bytes: int | None = None + content_type: str | None = None + modified_at: str | None = None + etag: str | None = None + metadata: Mapping[str, Any] = field(default_factory=dict) + + def to_response(self) -> dict[str, Any]: + return { + "kind": self.kind, + "name": self.name, + "path": self.path, + "external_id": self.external_id, + "external_url": self.external_url, + "size_bytes": self.size_bytes, + "content_type": self.content_type, + "modified_at": self.modified_at, + "etag": self.etag, + "metadata": dict(self.metadata), + } + + +def browse_connector_profile(profile: ConnectorProfile, *, path: str | None = None, library_id: str | None = None) -> list[ConnectorBrowseItem]: + browse_path = normalize_connector_browse_path(path) + static_items = _static_listing(profile, path=browse_path, library_id=library_id) + if static_items is not None: + return static_items + if profile.provider == "seafile": + if _metadata_string(profile, "browse_protocol") == "webdav" or _metadata_string(profile, "webdav_endpoint_url"): + return _browse_webdav(profile, path=browse_path) + return _browse_seafile(profile, path=browse_path, library_id=library_id) + if profile.provider in {"webdav", "nextcloud"} or _metadata_string(profile, "browse_protocol") == "webdav": + return _browse_webdav(profile, path=browse_path) + if profile.provider == "smb": + return _browse_smb(profile, path=browse_path) + raise ConnectorBrowseUnsupported(f"Read-only browsing is not implemented for {profile.provider} connector profiles yet") + + +def parse_webdav_multistatus(*, root_url: str, current_path: str, payload: str | bytes) -> list[ConnectorBrowseItem]: + try: + root = ET.fromstring(payload) + except ET.ParseError as exc: + raise ConnectorBrowseError("Connector returned invalid WebDAV XML") from exc + root_path = _url_path_root(root_url) + current = normalize_connector_browse_path(current_path) + items: list[ConnectorBrowseItem] = [] + for response in root.findall("{DAV:}response"): + href = response.findtext("{DAV:}href") + if not href: + continue + relative_path = _relative_href_path(root_path, href) + if relative_path == current: + continue + if current and not relative_path.startswith(current.rstrip("/") + "/"): + continue + parent = relative_path.rsplit("/", 1)[0] if "/" in relative_path else "" + if parent != current: + continue + prop = _webdav_prop(response) + is_collection = prop.find("{DAV:}resourcetype/{DAV:}collection") is not None if prop is not None else False + name = _webdav_display_name(prop) or _path_name(relative_path) + if not name: + continue + items.append( + ConnectorBrowseItem( + kind="folder" if is_collection else "file", + name=name, + path=relative_path, + external_id=_text(prop, "{DAV:}getetag") or relative_path, + size_bytes=None if is_collection else _int(_text(prop, "{DAV:}getcontentlength")), + content_type=None if is_collection else _text(prop, "{DAV:}getcontenttype"), + modified_at=_http_date(_text(prop, "{DAV:}getlastmodified")), + etag=_text(prop, "{DAV:}getetag"), + ) + ) + return sorted(items, key=lambda item: (item.kind != "library", item.kind != "folder", item.name.casefold(), item.path.casefold())) + + +def seafile_libraries_from_payload(payload: object) -> list[ConnectorBrowseItem]: + if not isinstance(payload, list): + raise ConnectorBrowseError("Seafile library response must be a list") + libraries = [_seafile_library_item(item) for item in payload if isinstance(item, Mapping)] + return sorted(libraries, key=lambda item: item.name.casefold()) + + +def seafile_directory_items_from_payload(payload: object, *, path: str | None = None) -> list[ConnectorBrowseItem]: + if payload == "uptodate": + return [] + if not isinstance(payload, list): + raise ConnectorBrowseError("Seafile directory response must be a list") + browse_path = normalize_connector_browse_path(path) + items = [_seafile_directory_item(item, parent_path=browse_path) for item in payload if isinstance(item, Mapping)] + return sorted(items, key=lambda item: (item.kind != "folder", item.name.casefold(), item.path.casefold())) + + +def _browse_seafile(profile: ConnectorProfile, *, path: str, library_id: str | None) -> list[ConnectorBrowseItem]: + repo_id, dir_path = _seafile_repo_and_path(path=path, library_id=library_id) + headers = _seafile_headers(profile) + if not repo_id: + payload = _request_json("GET", _seafile_url(profile, "api2/repos/"), headers=headers) + return seafile_libraries_from_payload(payload) + payload = _request_json( + "GET", + _seafile_url(profile, f"api2/repos/{quote(repo_id, safe='')}/dir/"), + headers=headers, + params={"p": "/" + dir_path if dir_path else "/"}, + ) + return seafile_directory_items_from_payload(payload, path=dir_path) + + +def _browse_webdav(profile: ConnectorProfile, *, path: str) -> list[ConnectorBrowseItem]: + root_url = _metadata_string(profile, "webdav_endpoint_url") or profile.endpoint_url + if not root_url: + raise ConnectorBrowseError("Connector profile does not define an endpoint URL") + url = _webdav_url(root_url, path) + headers = {"Depth": "1", "Content-Type": "application/xml; charset=utf-8"} + auth: tuple[str, str] | None = None + password = _profile_password(profile) + token = _profile_token(profile) + if profile.username and password: + auth = (profile.username, password) + elif token: + headers["Authorization"] = f"Bearer {token}" + elif profile.credential_mode.casefold() not in {"", "none", "anonymous"} and profile.secret_ref: + raise ConnectorBrowseError("Secret-ref connector credentials need a runtime secret resolver before live browsing") + body = """ + + + + + + + + + +""" + try: + response = httpx.request("PROPFIND", url, headers=headers, content=body, auth=auth, timeout=15.0) + except httpx.HTTPError as exc: + raise ConnectorBrowseError(f"Connector browse failed: {exc}") from exc + if response.status_code in {401, 403}: + raise ConnectorBrowseError("Connector credentials were rejected") + if response.status_code not in {200, 207}: + raise ConnectorBrowseError(f"Connector browse failed with HTTP {response.status_code}") + return parse_webdav_multistatus(root_url=root_url, current_path=path, payload=response.text) + + +def _browse_smb(profile: ConnectorProfile, *, path: str) -> list[ConnectorBrowseItem]: + location = _smb_location(profile) + unc_path = _smb_unc_path(location, path) + smbclient = _smbclient_module() + try: + entries = smbclient.scandir(unc_path, **_smb_client_kwargs(profile, location)) + except Exception as exc: # pragma: no cover - concrete exception types are dependency-version specific + raise ConnectorBrowseError(f"SMB connector browse failed: {exc}") from exc + items: list[ConnectorBrowseItem] = [] + try: + with entries as iterator: + for entry in iterator: + name = _clean(getattr(entry, "name", None)) + if not name or name in {".", ".."}: + continue + try: + is_dir = bool(entry.is_dir()) + except Exception: + is_dir = False + stat_result = _smb_entry_stat(entry) + item_path = _join_browse_path(path, name) + items.append( + ConnectorBrowseItem( + kind="folder" if is_dir else "file", + name=name, + path=item_path, + external_id=f"{location.share}:{item_path}", + size_bytes=None if is_dir else _smb_stat_size(stat_result), + content_type=None if is_dir else mimetypes.guess_type(name)[0], + modified_at=_smb_stat_modified_at(stat_result), + etag=_smb_stat_revision(stat_result), + metadata={ + "share": location.share, + **({"server": location.server} if _metadata_bool(profile, "expose_server_metadata", default=False) else {}), + }, + ) + ) + except ConnectorBrowseError: + raise + except Exception as exc: # pragma: no cover - concrete exception types are dependency-version specific + raise ConnectorBrowseError(f"SMB connector browse failed: {exc}") from exc + return sorted(items, key=lambda item: (item.kind != "folder", item.name.casefold(), item.path.casefold())) + + +def _seafile_headers(profile: ConnectorProfile) -> dict[str, str]: + token = _seafile_token(profile) + return {"Authorization": f"Token {token}", "Accept": "application/json"} + + +def _seafile_token(profile: ConnectorProfile) -> str: + token = _profile_token(profile) + if token: + return token + password = _profile_password(profile) + if profile.username and password: + payload = _request_json( + "POST", + _seafile_url(profile, "api2/auth-token/"), + data={"username": profile.username, "password": password}, + ) + if not isinstance(payload, Mapping) or not _clean(payload.get("token")): + raise ConnectorBrowseError("Seafile did not return an account token") + return _clean(payload.get("token")) or "" + if profile.secret_ref: + raise ConnectorBrowseError("Secret-ref Seafile credentials need a runtime secret resolver before live browsing") + raise ConnectorBrowseError("Seafile connector profiles require token credentials or username plus password credentials") + + +def _seafile_url(profile: ConnectorProfile, suffix: str) -> str: + if not profile.endpoint_url: + raise ConnectorBrowseError("Seafile connector profile does not define endpoint_url") + return urljoin(profile.endpoint_url.rstrip("/") + "/", suffix.lstrip("/")) + + +def _request_json( + method: str, + url: str, + *, + headers: Mapping[str, str] | None = None, + params: Mapping[str, str] | None = None, + data: Mapping[str, str] | None = None, +) -> object: + try: + response = httpx.request(method, url, headers=dict(headers or {}), params=params, data=data, timeout=15.0) + except httpx.HTTPError as exc: + raise ConnectorBrowseError(f"Connector browse failed: {exc}") from exc + if response.status_code in {401, 403}: + raise ConnectorBrowseError("Connector credentials were rejected") + if response.status_code not in {200, 201}: + raise ConnectorBrowseError(f"Connector browse failed with HTTP {response.status_code}") + try: + return response.json() + except ValueError as exc: + raise ConnectorBrowseError("Connector returned invalid JSON") from exc + + +def _seafile_repo_and_path(*, path: str, library_id: str | None) -> tuple[str | None, str]: + repo_id = _clean(library_id) + if repo_id: + return repo_id, path + if not path: + return None, "" + first, _, rest = path.partition("/") + return first, rest + + +def _seafile_library_item(value: Mapping[str, Any]) -> ConnectorBrowseItem: + repo_id = _clean(value.get("id") or value.get("repo_id")) + name = _clean(value.get("name") or value.get("repo_name") or repo_id) + if not repo_id or not name: + raise ConnectorBrowseError("Seafile library entries require id and name") + return ConnectorBrowseItem( + kind="library", + name=name, + path=repo_id, + external_id=repo_id, + size_bytes=_int(value.get("size") or value.get("repo_size")), + modified_at=_timestamp(value.get("mtime")), + metadata={ + key: value[key] + for key in ("type", "permission", "encrypted", "owner", "file_count") + if key in value + }, + ) + + +def _seafile_directory_item(value: Mapping[str, Any], *, parent_path: str) -> ConnectorBrowseItem: + name = _clean(value.get("name") or value.get("obj_name")) + if not name: + raise ConnectorBrowseError("Seafile directory entries require name") + item_type = str(value.get("type") or ("dir" if value.get("is_dir") else "file")).casefold() + kind = "folder" if item_type in {"dir", "folder"} else "file" + path = _join_browse_path(parent_path, name) + content_type = None if kind == "folder" else mimetypes.guess_type(name)[0] + return ConnectorBrowseItem( + kind=kind, + name=name, + path=path, + external_id=_clean(value.get("id") or value.get("obj_id")), + size_bytes=None if kind == "folder" else _int(value.get("size")), + content_type=content_type, + modified_at=_timestamp(value.get("mtime") or value.get("modified")), + etag=_clean(value.get("id") or value.get("obj_id")), + metadata={ + key: value[key] + for key in ("permission", "modifier_email", "modifier_name") + if key in value + }, + ) + + +def _static_listing(profile: ConnectorProfile, *, path: str, library_id: str | None) -> list[ConnectorBrowseItem] | None: + listing = profile.metadata.get("static_listing") + if listing is None: + return None + if isinstance(listing, list): + raw_items = listing if path == "" else [] + elif isinstance(listing, Mapping): + key = library_id or path + raw_items = listing.get(key) + if raw_items is None and key == "": + raw_items = listing.get("/") + else: + raise ConnectorBrowseError("Connector static_listing metadata must be a list or object") + if raw_items is None: + return [] + if not isinstance(raw_items, list): + raise ConnectorBrowseError("Connector static_listing entries must be lists") + return sorted( + [_static_item(item, parent_path=path) for item in raw_items if isinstance(item, Mapping)], + key=lambda item: (item.kind != "library", item.kind != "folder", item.name.casefold(), item.path.casefold()), + ) + + +def _static_item(value: Mapping[str, Any], *, parent_path: str) -> ConnectorBrowseItem: + kind = str(value.get("kind") or "file").strip().casefold() + if kind not in {"library", "folder", "file"}: + raise ConnectorBrowseError("Connector browse items must be library, folder or file") + name = str(value.get("name") or value.get("path") or "").strip() + if not name: + raise ConnectorBrowseError("Connector browse items require name") + path = normalize_connector_browse_path(value.get("path")) + if not path: + path = _join_browse_path(parent_path, name) + return ConnectorBrowseItem( + kind=kind, + name=name, + path=path, + external_id=_clean(value.get("external_id")), + external_url=_clean(value.get("external_url")), + size_bytes=_int(value.get("size_bytes")), + content_type=_clean(value.get("content_type")), + modified_at=_clean(value.get("modified_at")), + etag=_clean(value.get("etag")), + metadata=value.get("metadata") if isinstance(value.get("metadata"), Mapping) else {}, + ) + + +def _webdav_url(root_url: str, path: str) -> str: + base = root_url if root_url.endswith("/") else root_url + "/" + if not path: + return base + quoted = "/".join(quote(part, safe="") for part in path.split("/") if part) + return urljoin(base, quoted + "/") + + +def _url_path_root(root_url: str) -> str: + path = unquote(urlsplit(root_url).path or "/") + return path if path.endswith("/") else path + "/" + + +def _relative_href_path(root_path: str, href: str) -> str: + href_path = unquote(urlsplit(href).path or href).strip() + if href_path.startswith(root_path): + return normalize_connector_browse_path(href_path[len(root_path):]) + return normalize_connector_browse_path(href_path.rsplit("/", 1)[-1]) + + +def _webdav_prop(response: ET.Element) -> ET.Element: + prop = response.find("{DAV:}propstat/{DAV:}prop") + if prop is not None: + return prop + prop = response.find(".//{DAV:}prop") + return prop if prop is not None else ET.Element("prop") + + +def _webdav_display_name(prop: ET.Element | None) -> str | None: + return _clean(_text(prop, "{DAV:}displayname")) + + +def _text(prop: ET.Element | None, tag: str) -> str | None: + if prop is None: + return None + child = prop.find(tag) + return child.text.strip() if child is not None and child.text else None + + +def _http_date(value: str | None) -> str | None: + if not value: + return None + try: + return parsedate_to_datetime(value).isoformat() + except (TypeError, ValueError): + return value + + +def _timestamp(value: object) -> str | None: + number = _int(value) + if number is None: + return _clean(value) + return datetime.fromtimestamp(number, tz=timezone.utc).isoformat() + + +def _metadata_string(profile: ConnectorProfile, key: str) -> str | None: + return _clean(profile.metadata.get(key)) + + +def _metadata_bool(profile: ConnectorProfile, key: str, *, default: bool = False) -> bool: + value = profile.metadata.get(key) + if value is None: + return default + if isinstance(value, bool): + return value + return str(value).strip().casefold() in {"1", "true", "yes", "on"} + + +def _env_required(name: str, profile_id: str) -> str: + value = _clean(os.environ.get(name)) + if not value: + raise ConnectorBrowseError(f"Connector profile {profile_id} is missing required credential environment") + return value + + +def normalize_connector_browse_path(value: object) -> str: + if value is None: + return "" + path = str(value).replace("\\", "/").strip().strip("/") + parts = [part for part in path.split("/") if part and part not in {"."}] + if any(part == ".." for part in parts): + raise ConnectorBrowseError("Connector browse paths cannot contain parent directory segments") + return "/".join(parts) + + +def _join_browse_path(parent: str, name: str) -> str: + child = normalize_connector_browse_path(name) + if not parent: + return child + return f"{parent.rstrip('/')}/{child}" + + +def _path_name(path: str) -> str: + return path.rstrip("/").rsplit("/", 1)[-1] + + +def _int(value: object) -> int | None: + if value is None or value == "": + return None + try: + return int(value) + except (TypeError, ValueError): + return None + + +def _clean(value: object) -> str | None: + if value is None: + return None + text = str(value).strip() + return text or None + + +@dataclass(frozen=True, slots=True) +class _SmbLocation: + server: str + share: str + port: int + root_path: str = "" + + +def _smb_location(profile: ConnectorProfile) -> _SmbLocation: + if not profile.endpoint_url: + raise ConnectorBrowseError("SMB connector profile does not define endpoint_url") + parsed = urlsplit(profile.endpoint_url) + if parsed.scheme.casefold() != "smb": + raise ConnectorBrowseError("SMB connector endpoint_url must use smb://") + server = _clean(parsed.hostname) + if not server: + raise ConnectorBrowseError("SMB connector endpoint_url must include a server") + path_parts = [part for part in unquote(parsed.path or "").strip("/").split("/") if part] + if not path_parts: + raise ConnectorBrowseError("SMB connector endpoint_url must include a share name") + root_parts = path_parts[1:] + if profile.base_path: + root_parts.extend(normalize_connector_browse_path(profile.base_path).split("/")) + return _SmbLocation( + server=server, + share=path_parts[0], + port=parsed.port or _int(profile.metadata.get("port")) or 445, + root_path=normalize_connector_browse_path("/".join(root_parts)), + ) + + +def _smb_unc_path(location: _SmbLocation, path: str) -> str: + parts = [part for part in (location.root_path, normalize_connector_browse_path(path)) if part] + suffix = "\\".join(part.replace("/", "\\") for part in parts) + base = f"\\\\{location.server}\\{location.share}" + return f"{base}\\{suffix}" if suffix else base + + +def _smb_client_kwargs(profile: ConnectorProfile, location: _SmbLocation) -> dict[str, object]: + kwargs: dict[str, object] = { + "port": location.port, + "require_signing": _metadata_bool(profile, "require_signing", default=True), + "auth_protocol": _metadata_string(profile, "auth_protocol") or "ntlm", + } + if "encrypt" in profile.metadata: + kwargs["encrypt"] = _metadata_bool(profile, "encrypt", default=False) + password = _profile_password(profile) + token = _profile_token(profile) + if profile.username and password: + kwargs["username"] = profile.username + kwargs["password"] = password + elif token: + raise ConnectorBrowseError("SMB connector profiles do not support bearer-token credentials") + elif profile.credential_mode.casefold() not in {"", "none", "anonymous"} and profile.secret_ref: + raise ConnectorBrowseError("Secret-ref SMB credentials need a runtime secret resolver before live browsing") + return kwargs + + +def _profile_password(profile: ConnectorProfile) -> str | None: + if profile.password_value: + return profile.password_value + if profile.password_env: + return _env_required(profile.password_env, profile.id) + return None + + +def _profile_token(profile: ConnectorProfile) -> str | None: + if profile.token_value: + return profile.token_value + if profile.token_env: + return _env_required(profile.token_env, profile.id) + return None + + +def _smbclient_module() -> Any: + try: + return import_module("smbclient") + except ImportError as exc: + raise ConnectorBrowseUnsupported("SMB connector browsing requires the optional smbprotocol dependency") from exc + + +def _smb_entry_stat(entry: object) -> object | None: + try: + return entry.stat() # type: ignore[attr-defined] + except Exception: + return None + + +def _smb_stat_size(stat_result: object | None) -> int | None: + return _int(getattr(stat_result, "st_size", None)) + + +def _smb_stat_modified_at(stat_result: object | None) -> str | None: + value = getattr(stat_result, "st_mtime", None) + if value is None: + return None + try: + return datetime.fromtimestamp(float(value), tz=timezone.utc).isoformat() + except (TypeError, ValueError, OSError, OverflowError): + return None + + +def _smb_stat_revision(stat_result: object | None) -> str | None: + mtime_ns = getattr(stat_result, "st_mtime_ns", None) + size = getattr(stat_result, "st_size", None) + if mtime_ns is not None: + return f"{mtime_ns}:{size or 0}" + modified = getattr(stat_result, "st_mtime", None) + if modified is not None: + return f"{modified}:{size or 0}" + return None diff --git a/src/govoplan_files/backend/storage/connector_credential_store.py b/src/govoplan_files/backend/storage/connector_credential_store.py new file mode 100644 index 0000000..44bd3c8 --- /dev/null +++ b/src/govoplan_files/backend/storage/connector_credential_store.py @@ -0,0 +1,340 @@ +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Any + +from sqlalchemy.orm import Session + +from govoplan_core.core.policy import normalize_policy_scope_type, policy_source_path +from govoplan_core.security.secrets import decrypt_secret, encrypt_secret +from govoplan_files.backend.db.models import FileConnectorCredential +from govoplan_files.backend.storage.common import FileStorageError +from govoplan_files.backend.storage.connector_policy import ConnectorPolicySource, connector_policy_sources_from_payload +from govoplan_files.backend.storage.connector_profiles import supported_connector_providers + + +@dataclass(frozen=True, slots=True) +class ConnectorCredential: + id: str + label: str + scope_type: str + scope_id: str | None = None + provider: str | None = None + enabled: bool = True + credential_mode: str = "none" + username: str | None = None + password_env: str | None = None + token_env: str | None = None + secret_ref: str | None = None + password_value: str | None = None + token_value: str | None = None + policy_sources: tuple[ConnectorPolicySource, ...] = () + metadata: Mapping[str, Any] | None = None + source_kind: str = "database" + + @property + def source_path(self) -> str: + return policy_source_path(self.scope_type, self.scope_id) + + @property + def credential_secret_source(self) -> str | None: + if self.secret_ref: + return "secret_ref" + if self.password_value or self.token_value: + return "stored" + if self.token_env: + return "token_env" + if self.password_env: + return "password_env" + return None + + @property + def credentials_configured(self) -> bool: + if self.credential_mode.casefold() in {"", "none", "anonymous"}: + return True + return bool(self.secret_ref or self.password_value or self.token_value or self.password_env or self.token_env) + + def to_response(self) -> dict[str, Any]: + return { + "id": self.id, + "label": self.label, + "provider": self.provider, + "enabled": self.enabled, + "scope_type": self.scope_type, + "scope_id": self.scope_id, + "source_path": self.source_path, + "credential_mode": self.credential_mode, + "credential_secret_source": self.credential_secret_source, + "credentials_configured": self.credentials_configured, + "username": self.username, + "policy_sources": [_policy_source_response(source) for source in self.policy_sources], + "metadata": dict(self.metadata or {}), + "source_kind": self.source_kind, + } + + +def list_database_connector_credentials( + session: Session, + *, + tenant_id: str, + include_disabled: bool = False, +) -> list[ConnectorCredential]: + return [connector_credential_from_row(row) for row in list_connector_credential_rows(session, tenant_id=tenant_id, include_disabled=include_disabled)] + + +def list_connector_credential_rows( + session: Session, + *, + tenant_id: str, + include_disabled: bool = False, +) -> list[FileConnectorCredential]: + query = session.query(FileConnectorCredential).filter( + (FileConnectorCredential.scope_type == "system") + | (FileConnectorCredential.tenant_id == tenant_id) + ) + if not include_disabled: + query = query.filter(FileConnectorCredential.enabled.is_(True)) + return query.order_by(FileConnectorCredential.scope_type.asc(), FileConnectorCredential.label.asc()).all() + + +def connector_credential_from_row(row: FileConnectorCredential) -> ConnectorCredential: + policy_sources = [] + if row.policy: + policy_sources = connector_policy_sources_from_payload({ + "scope_type": row.scope_type, + "scope_id": row.scope_id, + "label": row.label, + "policy": row.policy, + }) + return ConnectorCredential( + id=row.id, + label=row.label, + scope_type=row.scope_type, + scope_id=row.scope_id, + provider=row.provider, + enabled=row.enabled, + credential_mode=row.credential_mode, + username=row.username, + password_env=row.password_env, + token_env=row.token_env, + secret_ref=row.secret_ref, + password_value=decrypt_secret(row.password_encrypted), + token_value=decrypt_secret(row.token_encrypted), + policy_sources=tuple(policy_sources), + metadata=dict(row.metadata_ or {}), + ) + + +def get_connector_credential_row( + session: Session, + *, + tenant_id: str, + credential_id: str, + include_disabled: bool = False, +) -> FileConnectorCredential: + row = session.get(FileConnectorCredential, credential_id) + if row is None or (row.scope_type != "system" and row.tenant_id != tenant_id): + raise FileStorageError("Connector credential not found") + if not include_disabled and not row.enabled: + raise FileStorageError("Connector credential not found") + return row + + +def credential_rows_by_id( + session: Session, + *, + tenant_id: str, + credential_ids: set[str], + include_disabled: bool = False, +) -> dict[str, FileConnectorCredential]: + if not credential_ids: + return {} + rows = session.query(FileConnectorCredential).filter(FileConnectorCredential.id.in_(credential_ids)).all() + result: dict[str, FileConnectorCredential] = {} + for row in rows: + if row.scope_type != "system" and row.tenant_id != tenant_id: + continue + if not include_disabled and not row.enabled: + continue + result[row.id] = row + return result + + +def create_connector_credential_row( + session: Session, + *, + tenant_id: str, + user_id: str | None, + credential_id: str, + label: str, + scope_type: str, + scope_id: str | None = None, + provider: str | None = None, + enabled: bool = True, + credential_mode: str = "none", + username: str | None = None, + password: str | None = None, + token: str | None = None, + password_env: str | None = None, + token_env: str | None = None, + secret_ref: str | None = None, + policy: Mapping[str, Any] | None = None, + metadata: Mapping[str, Any] | None = None, +) -> FileConnectorCredential: + clean_id = _normalize_id(credential_id) + if session.get(FileConnectorCredential, clean_id) is not None: + raise FileStorageError(f"Connector credential already exists: {clean_id}") + clean_scope_type, clean_scope_id, row_tenant_id = _normalize_scope(tenant_id=tenant_id, scope_type=scope_type, scope_id=scope_id) + row = FileConnectorCredential( + id=clean_id, + tenant_id=row_tenant_id, + scope_type=clean_scope_type, + scope_id=clean_scope_id, + label=_normalize_label(label), + provider=_normalize_provider(provider), + enabled=bool(enabled), + credential_mode=_normalize_credential_mode(credential_mode), + username=_clean(username), + password_encrypted=encrypt_secret(_clean(password)), + token_encrypted=encrypt_secret(_clean(token)), + password_env=_clean(password_env), + token_env=_clean(token_env), + secret_ref=_clean(secret_ref), + policy=dict(policy or {}), + metadata_=dict(metadata or {}), + created_by_user_id=user_id, + updated_by_user_id=user_id, + ) + session.add(row) + session.flush() + return row + + +def update_connector_credential_row( + session: Session, + row: FileConnectorCredential, + *, + user_id: str | None, + label: str | None = None, + provider: str | None = None, + enabled: bool | None = None, + credential_mode: str | None = None, + username: str | None = None, + password: str | None = None, + token: str | None = None, + password_env: str | None = None, + token_env: str | None = None, + secret_ref: str | None = None, + policy: Mapping[str, Any] | None = None, + metadata: Mapping[str, Any] | None = None, + clear_password: bool = False, + clear_token: bool = False, +) -> FileConnectorCredential: + if label is not None: + row.label = _normalize_label(label) + if provider is not None: + row.provider = _normalize_provider(provider) + if enabled is not None: + row.enabled = bool(enabled) + if credential_mode is not None: + row.credential_mode = _normalize_credential_mode(credential_mode) + if username is not None: + row.username = _clean(username) + if password is not None: + row.password_encrypted = encrypt_secret(_clean(password)) + elif clear_password: + row.password_encrypted = None + if token is not None: + row.token_encrypted = encrypt_secret(_clean(token)) + elif clear_token: + row.token_encrypted = None + if password_env is not None: + row.password_env = _clean(password_env) + if token_env is not None: + row.token_env = _clean(token_env) + if secret_ref is not None: + row.secret_ref = _clean(secret_ref) + if policy is not None: + row.policy = dict(policy) + if metadata is not None: + row.metadata_ = dict(metadata) + row.updated_by_user_id = user_id + session.add(row) + session.flush() + return row + + +def deactivate_connector_credential_row(session: Session, row: FileConnectorCredential, *, user_id: str | None) -> FileConnectorCredential: + row.enabled = False + row.updated_by_user_id = user_id + session.add(row) + session.flush() + return row + + +def _normalize_scope(*, tenant_id: str, scope_type: str, scope_id: str | None) -> tuple[str, str | None, str | None]: + clean_scope_type = normalize_policy_scope_type(scope_type) + clean_scope_id = _clean(scope_id) + if clean_scope_type == "system": + return "system", None, None + if clean_scope_type == "tenant": + return "tenant", tenant_id, tenant_id + if clean_scope_type in {"user", "group", "campaign"}: + if not clean_scope_id: + raise FileStorageError(f"{clean_scope_type.capitalize()} connector credentials require scope_id") + return clean_scope_type, clean_scope_id, tenant_id + raise FileStorageError("Unsupported connector credential scope") + + +def _normalize_id(value: str) -> str: + clean = _clean(value) + if not clean: + raise FileStorageError("Connector credential id is required") + if len(clean) > 255: + raise FileStorageError("Connector credential id is too long") + if any(char.isspace() for char in clean): + raise FileStorageError("Connector credential id cannot contain whitespace") + return clean + + +def _normalize_label(value: str) -> str: + clean = value.strip() + if not clean: + raise FileStorageError("Connector credential label is required") + if len(clean) > 255: + raise FileStorageError("Connector credential label is too long") + return clean + + +def _normalize_provider(value: str | None) -> str | None: + clean = _clean(value) + if clean is None: + return None + clean = clean.casefold() + if clean not in supported_connector_providers(): + raise FileStorageError(f"Unsupported connector credential provider: {value}") + return clean + + +def _normalize_credential_mode(value: str) -> str: + clean = value.strip().casefold() or "none" + if clean not in {"none", "anonymous", "basic", "token", "secret_ref"}: + raise FileStorageError(f"Unsupported connector credential mode: {value}") + return clean + + +def _policy_source_response(source: ConnectorPolicySource) -> dict[str, Any]: + return { + "scope_type": source.scope_type, + "scope_id": source.scope_id, + "label": source.label, + "policy": dict(source.policy or {}), + } + + +def _clean(value: object) -> str | None: + if value is None: + return None + text = str(value).strip() + return text or None diff --git a/src/govoplan_files/backend/storage/connector_imports.py b/src/govoplan_files/backend/storage/connector_imports.py new file mode 100644 index 0000000..31a8833 --- /dev/null +++ b/src/govoplan_files/backend/storage/connector_imports.py @@ -0,0 +1,222 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +import mimetypes +from typing import Any + +import httpx + +from govoplan_files.backend.storage.connector_browse import ( + ConnectorBrowseError, + ConnectorBrowseUnsupported, + _clean, + _metadata_string, + _profile_password, + _profile_token, + _request_json, + _smb_client_kwargs, + _smb_location, + _smb_stat_modified_at, + _smb_stat_revision, + _smb_stat_size, + _smb_unc_path, + _smbclient_module, + _seafile_headers, + _seafile_url, + _webdav_url, + normalize_connector_browse_path, +) +from govoplan_files.backend.storage.connector_profiles import ConnectorProfile +from govoplan_files.backend.storage.paths import filename_from_path + + +class ConnectorImportError(RuntimeError): + pass + + +class ConnectorImportUnsupported(ConnectorImportError): + pass + + +@dataclass(frozen=True, slots=True) +class ConnectorDownloadedFile: + filename: str + data: bytes + content_type: str | None = None + revision: str | None = None + external_id: str | None = None + external_url: str | None = None + metadata: dict[str, Any] = field(default_factory=dict) + + +def read_connector_file( + profile: ConnectorProfile, + *, + library_id: str, + path: str, + max_bytes: int, +) -> ConnectorDownloadedFile: + if profile.provider == "seafile": + if _metadata_string(profile, "browse_protocol") == "webdav" or _metadata_string(profile, "webdav_endpoint_url"): + return _read_webdav_file(profile, path=path, max_bytes=max_bytes) + return _read_seafile_file(profile, library_id=library_id, path=path, max_bytes=max_bytes) + if profile.provider in {"webdav", "nextcloud"}: + return _read_webdav_file(profile, path=path, max_bytes=max_bytes) + if profile.provider == "smb": + return _read_smb_file(profile, path=path, max_bytes=max_bytes) + raise ConnectorImportUnsupported(f"Connector file import is not implemented for {profile.provider} profiles yet") + + +def _read_seafile_file(profile: ConnectorProfile, *, library_id: str, path: str, max_bytes: int) -> ConnectorDownloadedFile: + repo_id = _clean(library_id) + if not repo_id: + raise ConnectorImportError("Seafile import requires library_id") + file_path = "/" + normalize_connector_browse_path(path) + if file_path == "/": + raise ConnectorImportError("Seafile import requires a file path") + try: + headers = _seafile_headers(profile) + detail = _request_json( + "GET", + _seafile_url(profile, f"api2/repos/{repo_id}/file/detail/"), + headers=headers, + params={"p": file_path}, + ) + if isinstance(detail, dict): + size = _int(detail.get("size")) + if size is not None and size > max_bytes: + raise ConnectorImportError(f"Seafile file exceeds limit of {max_bytes} bytes") + download_url = _request_json( + "GET", + _seafile_url(profile, f"api2/repos/{repo_id}/file/"), + headers=headers, + params={"p": file_path, "reuse": "1"}, + ) + except ConnectorBrowseError as exc: + raise ConnectorImportError(str(exc)) from exc + if not isinstance(download_url, str) or not download_url.strip(): + raise ConnectorImportError("Seafile did not return a file download URL") + try: + response = httpx.request("GET", download_url, timeout=30.0) + except httpx.HTTPError as exc: + raise ConnectorImportError(f"Seafile file download failed: {exc}") from exc + if response.status_code != 200: + raise ConnectorImportError(f"Seafile file download failed with HTTP {response.status_code}") + data = response.content + if len(data) > max_bytes: + raise ConnectorImportError(f"Seafile file exceeds limit of {max_bytes} bytes") + detail = detail if isinstance(detail, dict) else {} + filename = filename_from_path(str(detail.get("name") or path)) + content_type = response.headers.get("content-type") or mimetypes.guess_type(filename)[0] + external_id = f"{repo_id}:{normalize_connector_browse_path(path)}" + return ConnectorDownloadedFile( + filename=filename, + data=data, + content_type=content_type, + revision=_clean(detail.get("id") or detail.get("mtime") or detail.get("last_modified")), + external_id=external_id, + external_url=download_url, + metadata={ + "library_id": repo_id, + "library_path": normalize_connector_browse_path(path), + "size": len(data), + **({key: detail[key] for key in ("permission", "last_modified", "mtime", "uploader_email", "last_modifier_email") if key in detail}), + }, + ) + + +def _read_webdav_file(profile: ConnectorProfile, *, path: str, max_bytes: int) -> ConnectorDownloadedFile: + root_url = _metadata_string(profile, "webdav_endpoint_url") or profile.endpoint_url + if not root_url: + raise ConnectorImportError("WebDAV connector profile does not define endpoint_url") + file_path = normalize_connector_browse_path(path) + if not file_path: + raise ConnectorImportError("WebDAV import requires a file path") + url = _webdav_url(root_url, file_path).rstrip("/") + headers: dict[str, str] = {} + auth: tuple[str, str] | None = None + password = _profile_password(profile) + token = _profile_token(profile) + if profile.username and password: + auth = (profile.username, password) + elif token: + headers["Authorization"] = f"Bearer {token}" + elif profile.credential_mode.casefold() not in {"", "none", "anonymous"} and profile.secret_ref: + raise ConnectorImportError("Secret-ref WebDAV credentials need a runtime secret resolver before live import") + try: + response = httpx.request("GET", url, headers=headers, auth=auth, timeout=30.0) + except httpx.HTTPError as exc: + raise ConnectorImportError(f"WebDAV file download failed: {exc}") from exc + if response.status_code in {401, 403}: + raise ConnectorImportError("Connector credentials were rejected") + if response.status_code != 200: + raise ConnectorImportError(f"WebDAV file download failed with HTTP {response.status_code}") + length = _int(response.headers.get("content-length")) + if length is not None and length > max_bytes: + raise ConnectorImportError(f"WebDAV file exceeds limit of {max_bytes} bytes") + data = response.content + if len(data) > max_bytes: + raise ConnectorImportError(f"WebDAV file exceeds limit of {max_bytes} bytes") + filename = filename_from_path(file_path) + revision = _clean(response.headers.get("etag") or response.headers.get("last-modified")) + return ConnectorDownloadedFile( + filename=filename, + data=data, + content_type=response.headers.get("content-type") or mimetypes.guess_type(filename)[0], + revision=revision, + external_id=file_path, + external_url=url, + metadata={"path": file_path, "size": len(data)}, + ) + + +def _read_smb_file(profile: ConnectorProfile, *, path: str, max_bytes: int) -> ConnectorDownloadedFile: + location = _smb_location(profile) + file_path = normalize_connector_browse_path(path) + if not file_path: + raise ConnectorImportError("SMB import requires a file path") + unc_path = _smb_unc_path(location, file_path) + try: + smbclient = _smbclient_module() + kwargs = _smb_client_kwargs(profile, location) + stat_result = smbclient.stat(unc_path, **kwargs) + size = _smb_stat_size(stat_result) + if size is not None and size > max_bytes: + raise ConnectorImportError(f"SMB file exceeds limit of {max_bytes} bytes") + with smbclient.open_file(unc_path, mode="rb", **kwargs) as handle: + data = handle.read(max_bytes + 1) + except ConnectorBrowseUnsupported as exc: + raise ConnectorImportUnsupported(str(exc)) from exc + except ConnectorBrowseError as exc: + raise ConnectorImportError(str(exc)) from exc + except ConnectorImportError: + raise + except Exception as exc: # pragma: no cover - concrete exception types are dependency-version specific + raise ConnectorImportError(f"SMB file download failed: {exc}") from exc + if len(data) > max_bytes: + raise ConnectorImportError(f"SMB file exceeds limit of {max_bytes} bytes") + filename = filename_from_path(file_path) + revision = _smb_stat_revision(stat_result) + return ConnectorDownloadedFile( + filename=filename, + data=data, + content_type=mimetypes.guess_type(filename)[0], + revision=revision, + external_id=f"{location.share}:{file_path}", + external_url=f"smb://{location.server}:{location.port}/{location.share}/{file_path}", + metadata={ + "share": location.share, + "path": file_path, + "size": len(data), + "modified_at": _smb_stat_modified_at(stat_result), + }, + ) + + +def _int(value: object) -> int | None: + if value is None or value == "": + return None + try: + return int(value) + except (TypeError, ValueError): + return None diff --git a/src/govoplan_files/backend/storage/connector_policy.py b/src/govoplan_files/backend/storage/connector_policy.py new file mode 100644 index 0000000..74373d1 --- /dev/null +++ b/src/govoplan_files/backend/storage/connector_policy.py @@ -0,0 +1,310 @@ +from __future__ import annotations + +from collections.abc import Iterable, Mapping +from dataclasses import dataclass +from fnmatch import fnmatchcase +from typing import Any + +from govoplan_core.core.policy import PolicyDecision, PolicySourceStep, normalize_policy_scope_type + + +_ALLOW_KEYS = ("allow", "allowlist", "whitelist") +_DENY_KEYS = ("deny", "denylist", "blacklist") + +_FIELD_ALIASES = { + "connectors": ("connectors", "connector_ids", "connector_id"), + "credentials": ("credentials", "credential_ids", "credential_id"), + "providers": ("providers", "provider"), + "external_ids": ("external_ids", "external_id"), + "external_paths": ("external_paths", "paths", "path_prefixes", "external_path"), + "external_urls": ("external_urls", "urls", "external_url"), +} + +CONNECTOR_POLICY_FIELDS = tuple(_FIELD_ALIASES) + + +class ConnectorPolicyDenied(RuntimeError): + def __init__(self, decision: PolicyDecision) -> None: + super().__init__(decision.reason or "Connector policy denied") + self.decision = decision + + +@dataclass(frozen=True, slots=True) +class ConnectorAccessRequest: + connector_id: str | None = None + credential_id: str | None = None + provider: str | None = None + external_id: str | None = None + external_path: str | None = None + external_url: str | None = None + operation: str = "access" + + @classmethod + def from_provenance(cls, value: Mapping[str, Any] | None, *, operation: str = "access") -> "ConnectorAccessRequest": + payload = value or {} + return cls( + connector_id=_clean(payload.get("connector_id")), + credential_id=_clean(payload.get("credential_id") or payload.get("connector_credential_id")), + provider=_clean(payload.get("provider") or payload.get("source_type")), + external_id=_clean(payload.get("external_id")), + external_path=_clean(payload.get("external_path")), + external_url=_clean(payload.get("external_url")), + operation=_clean(operation) or "access", + ) + + def to_dict(self) -> dict[str, str | None]: + return { + "connector_id": self.connector_id, + "credential_id": self.credential_id, + "provider": self.provider, + "external_id": self.external_id, + "external_path": self.external_path, + "external_url": self.external_url, + "operation": self.operation, + } + + +@dataclass(frozen=True, slots=True) +class ConnectorPolicySource: + scope_type: str + label: str + scope_id: str | None = None + policy: Mapping[str, Any] | None = None + + def source_step(self, applied_fields: Iterable[str]) -> PolicySourceStep: + return PolicySourceStep( + scope_type=normalize_policy_scope_type(self.scope_type), + scope_id=self.scope_id, + label=self.label, + applied_fields=tuple(dict.fromkeys(applied_fields)), + policy=dict(self.policy or {}), + ) + + +def connector_policy_sources_from_payload(value: object) -> list[ConnectorPolicySource]: + if value is None or value == "": + return [] + if isinstance(value, Mapping): + raw_sources = value.get("sources") + if raw_sources is None: + raw_sources = [value] + elif isinstance(value, list): + raw_sources = value + else: + raise ValueError("connector_policy must be a JSON object or list") + if not isinstance(raw_sources, list): + raise ValueError("connector_policy.sources must be a list") + return [_source_from_mapping(item) for item in raw_sources if isinstance(item, Mapping)] + + +def connector_policy_applied_fields(policy: Mapping[str, Any] | None) -> tuple[str, ...]: + return _applied_fields(_policy_mapping(policy)) + + +def filtered_connector_policy_source(source: ConnectorPolicySource, fields: Iterable[str]) -> ConnectorPolicySource: + allowed_fields = {field for field in fields if field in _FIELD_ALIASES} + if not allowed_fields: + return ConnectorPolicySource(scope_type=source.scope_type, scope_id=source.scope_id, label=source.label, policy={}) + policy = _policy_mapping(source.policy) + filtered: dict[str, Any] = {} + for prefix, keys in (("allow", _ALLOW_KEYS), ("deny", _DENY_KEYS)): + rules = _merged_rule(policy, keys) + selected = {field: _string_list(rules.get(field)) for field in allowed_fields} + selected = {field: values for field, values in selected.items() if values} + if selected: + filtered[prefix] = selected + return ConnectorPolicySource(scope_type=source.scope_type, scope_id=source.scope_id, label=source.label, policy=filtered) + + +def connector_policy_sources_for_fields( + sources: Iterable[ConnectorPolicySource], + fields: Iterable[str], +) -> list[ConnectorPolicySource]: + return [filtered_connector_policy_source(source, fields) for source in sources] + + +def connector_policy_decision( + request: ConnectorAccessRequest, + sources: Iterable[ConnectorPolicySource], +) -> PolicyDecision: + source_list = list(sources) + source_steps: list[PolicySourceStep] = [] + deny_matches: list[dict[str, Any]] = [] + allow_misses: list[dict[str, Any]] = [] + + for source in source_list: + policy = _policy_mapping(source.policy) + applied_fields = _applied_fields(policy) + if applied_fields: + source_steps.append(source.source_step(applied_fields)) + + deny_policy = _merged_rule(policy, _DENY_KEYS) + for field, patterns in _rules_by_field(deny_policy).items(): + if _matches_field(request, field, patterns): + deny_matches.append({ + "scope": source.source_step((f"deny.{field}",)).to_dict(), + "field": field, + "patterns": patterns, + }) + + allow_policy = _merged_rule(policy, _ALLOW_KEYS) + for field, patterns in _rules_by_field(allow_policy).items(): + if not _matches_field(request, field, patterns): + allow_misses.append({ + "scope": source.source_step((f"allow.{field}",)).to_dict(), + "field": field, + "patterns": patterns, + }) + + details = { + "request": request.to_dict(), + "deny_matches": deny_matches, + "allow_misses": allow_misses, + } + if deny_matches: + return PolicyDecision( + allowed=False, + reason="Connector access is denied by a blacklist rule.", + source_path=tuple(source_steps), + requirements=("connector_policy_denylist",), + details=details, + ) + if allow_misses: + return PolicyDecision( + allowed=False, + reason="Connector access is not allowed by the configured whitelist.", + source_path=tuple(source_steps), + requirements=("connector_policy_allowlist",), + details=details, + ) + return PolicyDecision( + allowed=True, + reason=None, + source_path=tuple(source_steps), + requirements=(), + details=details, + ) + + +def ensure_connector_policy_allows( + request: ConnectorAccessRequest, + sources: Iterable[ConnectorPolicySource], +) -> PolicyDecision: + decision = connector_policy_decision(request, sources) + if not decision.allowed: + raise ConnectorPolicyDenied(decision) + return decision + + +def _source_from_mapping(value: Mapping[str, Any]) -> ConnectorPolicySource: + scope_type = normalize_policy_scope_type(str(value.get("scope_type") or "system")) + scope_id = _clean(value.get("scope_id")) + if scope_type == "system": + scope_id = None + elif not scope_id: + raise ValueError(f"{scope_type} connector policy sources require scope_id") + label = _clean(value.get("label")) or scope_type.capitalize() + policy = value.get("policy") + if policy is None: + policy = {key: value[key] for key in (*_ALLOW_KEYS, *_DENY_KEYS) if key in value} + if policy is not None and not isinstance(policy, Mapping): + raise ValueError("connector policy source policy must be a JSON object") + return ConnectorPolicySource(scope_type=scope_type, scope_id=scope_id, label=label, policy=policy or {}) + + +def _policy_mapping(value: Mapping[str, Any] | None) -> Mapping[str, Any]: + return value if isinstance(value, Mapping) else {} + + +def _merged_rule(policy: Mapping[str, Any], keys: Iterable[str]) -> dict[str, Any]: + merged: dict[str, Any] = {} + for key in keys: + raw = policy.get(key) + if isinstance(raw, Mapping): + merged.update(raw) + return merged + + +def _rules_by_field(value: Mapping[str, Any]) -> dict[str, list[str]]: + result: dict[str, list[str]] = {} + for field, aliases in _FIELD_ALIASES.items(): + items: list[str] = [] + for alias in aliases: + items.extend(_string_list(value.get(alias))) + if items: + result[field] = list(dict.fromkeys(items)) + return result + + +def _applied_fields(policy: Mapping[str, Any]) -> tuple[str, ...]: + fields: list[str] = [] + for prefix, keys in (("allow", _ALLOW_KEYS), ("deny", _DENY_KEYS)): + rules = _merged_rule(policy, keys) + fields.extend(f"{prefix}.{field}" for field in _rules_by_field(rules)) + return tuple(dict.fromkeys(fields)) + + +def _matches_field(request: ConnectorAccessRequest, field: str, patterns: list[str]) -> bool: + if field == "connectors": + return _matches_exact(request.connector_id, patterns) + if field == "credentials": + return _matches_exact(request.credential_id, patterns) + if field == "providers": + return _matches_exact(request.provider, patterns) + if field == "external_ids": + return _matches_glob(request.external_id, patterns) + if field == "external_paths": + return _matches_path(request.external_path, patterns) + if field == "external_urls": + return _matches_glob(request.external_url, patterns) + return False + + +def _matches_exact(value: str | None, patterns: list[str]) -> bool: + if value is None: + return False + clean = value.casefold() + return any(pattern == "*" or clean == pattern.casefold() for pattern in patterns) + + +def _matches_glob(value: str | None, patterns: list[str]) -> bool: + if value is None: + return False + clean = value.casefold() + return any(fnmatchcase(clean, pattern.casefold()) for pattern in patterns) + + +def _matches_path(value: str | None, patterns: list[str]) -> bool: + if value is None: + return False + clean = value.replace("\\", "/").strip() + for pattern in patterns: + normalized = pattern.replace("\\", "/").strip() + if normalized == "*": + return True + if any(token in normalized for token in "*?[]"): + if fnmatchcase(clean.casefold(), normalized.casefold()): + return True + continue + if clean.casefold() == normalized.casefold() or clean.casefold().startswith(normalized.rstrip("/").casefold() + "/"): + return True + return False + + +def _string_list(value: object) -> list[str]: + if value is None: + return [] + if isinstance(value, str): + values = [value] + elif isinstance(value, Iterable) and not isinstance(value, (bytes, bytearray, Mapping)): + values = [str(item) for item in value] + else: + values = [str(value)] + return [item.strip() for item in values if item.strip()] + + +def _clean(value: object) -> str | None: + if value is None: + return None + text = str(value).strip() + return text or None diff --git a/src/govoplan_files/backend/storage/connector_policy_store.py b/src/govoplan_files/backend/storage/connector_policy_store.py new file mode 100644 index 0000000..469ec46 --- /dev/null +++ b/src/govoplan_files/backend/storage/connector_policy_store.py @@ -0,0 +1,334 @@ +from __future__ import annotations + +from collections.abc import Iterable, Mapping +from typing import Any + +from sqlalchemy.orm import Session + +from govoplan_core.core.policy import normalize_policy_scope_type +from govoplan_files.backend.db.models import FileConnectorPolicy +from govoplan_files.backend.storage.common import FileStorageError +from govoplan_files.backend.storage.connector_policy import ( + CONNECTOR_POLICY_FIELDS, + ConnectorPolicySource, + connector_policy_applied_fields, +) + +_RULE_GROUPS = ("allow", "deny") +_FIELD_ALIASES = { + "connectors": ("connectors", "connector_ids", "connector_id"), + "credentials": ("credentials", "credential_ids", "credential_id"), + "providers": ("providers", "provider"), + "external_ids": ("external_ids", "external_id"), + "external_paths": ("external_paths", "paths", "path_prefixes", "external_path"), + "external_urls": ("external_urls", "urls", "external_url"), +} + + +def get_connector_policy( + session: Session, + *, + tenant_id: str, + scope_type: str, + scope_id: str | None = None, +) -> dict[str, Any]: + row = _connector_policy_row(session, tenant_id=tenant_id, scope_type=scope_type, scope_id=scope_id) + return _normalize_connector_policy(row.policy if row is not None else None) + + +def set_connector_policy( + session: Session, + *, + tenant_id: str, + user_id: str | None, + scope_type: str, + scope_id: str | None = None, + policy: Mapping[str, Any] | None = None, +) -> dict[str, Any]: + normalized = _normalize_connector_policy(policy) + parent = parent_connector_policy(session, tenant_id=tenant_id, scope_type=scope_type, scope_id=scope_id) + if parent is not None: + _validate_lower_level_limits(parent, normalized) + row_tenant_id, row_scope_type, row_scope_id = _policy_scope_key(tenant_id=tenant_id, scope_type=scope_type, scope_id=scope_id) + row = _connector_policy_row(session, tenant_id=tenant_id, scope_type=row_scope_type, scope_id=row_scope_id) + if row is None: + row = FileConnectorPolicy( + tenant_id=row_tenant_id, + scope_type=row_scope_type, + scope_id=row_scope_id, + policy=normalized, + created_by_user_id=user_id, + updated_by_user_id=user_id, + ) + else: + row.policy = normalized + row.updated_by_user_id = user_id + session.add(row) + session.flush() + return normalized + + +def connector_policy_response( + session: Session, + *, + tenant_id: str, + scope_type: str, + scope_id: str | None = None, +) -> dict[str, Any]: + clean_scope_type, clean_scope_id = _policy_scope_ref(tenant_id=tenant_id, scope_type=scope_type, scope_id=scope_id) + policy = get_connector_policy(session, tenant_id=tenant_id, scope_type=clean_scope_type, scope_id=clean_scope_id) + effective_sources = effective_connector_policy_sources(session, tenant_id=tenant_id, scope_type=clean_scope_type, scope_id=clean_scope_id) + parent_policy = parent_connector_policy(session, tenant_id=tenant_id, scope_type=clean_scope_type, scope_id=clean_scope_id) + parent_sources = parent_connector_policy_sources(session, tenant_id=tenant_id, scope_type=clean_scope_type, scope_id=clean_scope_id) + return { + "scope_type": clean_scope_type, + "scope_id": clean_scope_id, + "policy": policy, + "effective_policy": merge_connector_policies(source.policy for source in effective_sources), + "parent_policy": parent_policy, + "effective_policy_sources": [_source_step(source) for source in effective_sources], + "parent_policy_sources": [_source_step(source) for source in parent_sources], + } + + +def effective_connector_policy_sources( + session: Session, + *, + tenant_id: str, + scope_type: str, + scope_id: str | None = None, +) -> list[ConnectorPolicySource]: + clean_scope_type, clean_scope_id = _policy_scope_ref(tenant_id=tenant_id, scope_type=scope_type, scope_id=scope_id) + sources: list[ConnectorPolicySource] = [] + for source_scope_type, source_scope_id in _policy_source_chain(tenant_id=tenant_id, scope_type=clean_scope_type, scope_id=clean_scope_id): + row = _connector_policy_row(session, tenant_id=tenant_id, scope_type=source_scope_type, scope_id=source_scope_id) + if row is None: + continue + policy = _normalize_connector_policy(row.policy) + if _policy_is_empty(policy): + continue + sources.append( + ConnectorPolicySource( + scope_type=source_scope_type, + scope_id=source_scope_id, + label=_policy_source_label(source_scope_type), + policy=policy, + ) + ) + return sources + + +def parent_connector_policy_sources( + session: Session, + *, + tenant_id: str, + scope_type: str, + scope_id: str | None = None, +) -> list[ConnectorPolicySource]: + clean_scope_type, clean_scope_id = _policy_scope_ref(tenant_id=tenant_id, scope_type=scope_type, scope_id=scope_id) + chain = _policy_source_chain(tenant_id=tenant_id, scope_type=clean_scope_type, scope_id=clean_scope_id) + parent_chain = chain[:-1] if chain else [] + sources: list[ConnectorPolicySource] = [] + for source_scope_type, source_scope_id in parent_chain: + row = _connector_policy_row(session, tenant_id=tenant_id, scope_type=source_scope_type, scope_id=source_scope_id) + if row is None: + continue + policy = _normalize_connector_policy(row.policy) + if _policy_is_empty(policy): + continue + sources.append( + ConnectorPolicySource( + scope_type=source_scope_type, + scope_id=source_scope_id, + label=_policy_source_label(source_scope_type), + policy=policy, + ) + ) + return sources + + +def parent_connector_policy( + session: Session, + *, + tenant_id: str, + scope_type: str, + scope_id: str | None = None, +) -> dict[str, Any] | None: + sources = parent_connector_policy_sources(session, tenant_id=tenant_id, scope_type=scope_type, scope_id=scope_id) + if not sources: + return None + return merge_connector_policies(source.policy for source in sources) + + +def validate_connector_policy_allowed_by_parent(parent_policy: Mapping[str, Any] | None, local_policy: Mapping[str, Any] | None) -> None: + if parent_policy is None: + return + _validate_lower_level_limits(_normalize_connector_policy(parent_policy), _normalize_connector_policy(local_policy)) + + +def merge_connector_policies(policies: Iterable[Mapping[str, Any] | None]) -> dict[str, Any]: + result = _empty_connector_policy() + for policy in policies: + normalized = _normalize_connector_policy(policy) + for group in _RULE_GROUPS: + rules = normalized.get(group) or {} + if not isinstance(rules, Mapping): + continue + target = result[group] + for field in CONNECTOR_POLICY_FIELDS: + values = _string_list(rules.get(field)) + if values: + target[field] = _unique([*target.get(field, []), *values]) + lower = normalized.get("allow_lower_level_limits") or {} + if isinstance(lower, Mapping): + for key, value in lower.items(): + if isinstance(value, bool): + result["allow_lower_level_limits"][str(key)] = value + return _compact_connector_policy(result, keep_lower=True) + + +def _connector_policy_row( + session: Session, + *, + tenant_id: str, + scope_type: str, + scope_id: str | None = None, +) -> FileConnectorPolicy | None: + row_tenant_id, row_scope_type, row_scope_id = _policy_scope_key(tenant_id=tenant_id, scope_type=scope_type, scope_id=scope_id) + query = session.query(FileConnectorPolicy).filter(FileConnectorPolicy.scope_type == row_scope_type) + query = query.filter(FileConnectorPolicy.tenant_id.is_(None) if row_tenant_id is None else FileConnectorPolicy.tenant_id == row_tenant_id) + query = query.filter(FileConnectorPolicy.scope_id.is_(None) if row_scope_id is None else FileConnectorPolicy.scope_id == row_scope_id) + return query.order_by(FileConnectorPolicy.created_at.asc()).first() + + +def _policy_scope_key(*, tenant_id: str, scope_type: str, scope_id: str | None) -> tuple[str | None, str, str | None]: + clean_scope_type, clean_scope_id = _policy_scope_ref(tenant_id=tenant_id, scope_type=scope_type, scope_id=scope_id) + if clean_scope_type == "system": + return None, "system", None + return tenant_id, clean_scope_type, clean_scope_id + + +def _policy_scope_ref(*, tenant_id: str, scope_type: str, scope_id: str | None) -> tuple[str, str | None]: + clean_scope_type = normalize_policy_scope_type(scope_type) + clean_scope_id = _clean(scope_id) + if clean_scope_type == "system": + return "system", None + if clean_scope_type == "tenant": + if clean_scope_id and clean_scope_id != tenant_id: + raise FileStorageError("Tenant connector policy scope does not belong to the active tenant") + return "tenant", tenant_id + if clean_scope_type in {"user", "group", "campaign"}: + if not clean_scope_id: + raise FileStorageError(f"{clean_scope_type.capitalize()} connector policy requires scope_id") + return clean_scope_type, clean_scope_id + raise FileStorageError("Unsupported connector policy scope") + + +def _policy_source_chain(*, tenant_id: str, scope_type: str, scope_id: str | None) -> list[tuple[str, str | None]]: + chain: list[tuple[str, str | None]] = [("system", None)] + if scope_type == "system": + return chain + chain.append(("tenant", tenant_id)) + if scope_type == "tenant": + return chain + chain.append((scope_type, scope_id)) + return chain + + +def _normalize_connector_policy(policy: Mapping[str, Any] | None) -> dict[str, Any]: + if policy is None: + return {} + if not isinstance(policy, Mapping): + raise FileStorageError("Connector policy must be a JSON object") + normalized = _empty_connector_policy() + for group in _RULE_GROUPS: + raw = policy.get(group) + if raw is None: + raw = policy.get(f"{group}list") + if raw is None: + raw = policy.get("whitelist" if group == "allow" else "blacklist") + if raw is not None and not isinstance(raw, Mapping): + raise FileStorageError(f"Connector policy {group} rules must be an object") + for field, aliases in _FIELD_ALIASES.items(): + values: list[str] = [] + if isinstance(raw, Mapping): + for alias in aliases: + values.extend(_string_list(raw.get(alias))) + if values: + normalized[group][field] = _unique(values) + lower = policy.get("allow_lower_level_limits") + if lower is not None: + if not isinstance(lower, Mapping): + raise FileStorageError("Connector policy allow_lower_level_limits must be an object") + for key, value in lower.items(): + clean_key = str(key).strip() + if clean_key and isinstance(value, bool): + normalized["allow_lower_level_limits"][clean_key] = value + return _compact_connector_policy(normalized, keep_lower=True) + + +def _empty_connector_policy() -> dict[str, Any]: + return {"allow": {}, "deny": {}, "allow_lower_level_limits": {}} + + +def _compact_connector_policy(policy: dict[str, Any], *, keep_lower: bool = False) -> dict[str, Any]: + compact: dict[str, Any] = {} + for group in _RULE_GROUPS: + rules = {field: _unique(_string_list(values)) for field, values in (policy.get(group) or {}).items()} + rules = {field: values for field, values in rules.items() if values} + if rules: + compact[group] = rules + lower = policy.get("allow_lower_level_limits") or {} + if isinstance(lower, Mapping): + lower_compact = {str(key): bool(value) for key, value in lower.items() if isinstance(value, bool)} + if lower_compact or keep_lower: + compact["allow_lower_level_limits"] = lower_compact + return compact + + +def _policy_is_empty(policy: Mapping[str, Any]) -> bool: + return not connector_policy_applied_fields(policy) and not bool(policy.get("allow_lower_level_limits")) + + +def _source_step(source: ConnectorPolicySource) -> dict[str, Any]: + return source.source_step(connector_policy_applied_fields(source.policy)).to_dict() + + +def _policy_source_label(scope_type: str) -> str: + if scope_type == "system": + return "System connector policy" + if scope_type == "tenant": + return "Tenant connector policy" + return f"{scope_type.capitalize()} connector policy" + + +def _validate_lower_level_limits(parent_policy: Mapping[str, Any], local_policy: Mapping[str, Any]) -> None: + limits = parent_policy.get("allow_lower_level_limits") + if not isinstance(limits, Mapping): + return + for field in connector_policy_applied_fields(local_policy): + allowed = limits.get(field) + if allowed is False: + raise FileStorageError(f"Parent connector policy does not allow lower-level {field} limits") + + +def _string_list(value: object) -> list[str]: + if value is None: + return [] + if isinstance(value, str): + clean = value.strip() + return [clean] if clean else [] + if isinstance(value, Iterable) and not isinstance(value, (str, bytes, bytearray, Mapping)): + return [str(item).strip() for item in value if str(item).strip()] + return [str(value).strip()] if str(value).strip() else [] + + +def _unique(values: Iterable[str]) -> list[str]: + return list(dict.fromkeys(value for value in values if value)) + + +def _clean(value: object) -> str | None: + if value is None: + return None + text = str(value).strip() + return text or None diff --git a/src/govoplan_files/backend/storage/connector_profile_store.py b/src/govoplan_files/backend/storage/connector_profile_store.py new file mode 100644 index 0000000..869f1d0 --- /dev/null +++ b/src/govoplan_files/backend/storage/connector_profile_store.py @@ -0,0 +1,298 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +from sqlalchemy.orm import Session + +from govoplan_core.core.policy import normalize_policy_scope_type +from govoplan_core.security.secrets import decrypt_secret, encrypt_secret +from govoplan_files.backend.db.models import FileConnectorCredential, FileConnectorProfile +from govoplan_files.backend.storage.common import FileStorageError +from govoplan_files.backend.storage.connector_credential_store import connector_credential_from_row, credential_rows_by_id +from govoplan_files.backend.storage.connector_policy import connector_policy_sources_from_payload +from govoplan_files.backend.storage.connector_profiles import ConnectorProfile, supported_connector_providers + + +def list_database_connector_profiles( + session: Session, + *, + tenant_id: str, + include_disabled: bool = False, +) -> list[ConnectorProfile]: + query = session.query(FileConnectorProfile).filter( + (FileConnectorProfile.scope_type == "system") + | (FileConnectorProfile.tenant_id == tenant_id) + ) + if not include_disabled: + query = query.filter(FileConnectorProfile.enabled.is_(True)) + rows = query.order_by(FileConnectorProfile.scope_type.asc(), FileConnectorProfile.label.asc()).all() + credential_ids = {_clean(row.credential_profile_id) for row in rows if _clean(row.credential_profile_id)} + credentials = credential_rows_by_id(session, tenant_id=tenant_id, credential_ids={item for item in credential_ids if item}, include_disabled=include_disabled) + return [connector_profile_from_row(row, credential_row=credentials.get(row.credential_profile_id or "")) for row in rows] + + +def list_connector_profile_rows( + session: Session, + *, + tenant_id: str, + include_disabled: bool = False, +) -> list[FileConnectorProfile]: + query = session.query(FileConnectorProfile).filter( + (FileConnectorProfile.scope_type == "system") + | (FileConnectorProfile.tenant_id == tenant_id) + ) + if not include_disabled: + query = query.filter(FileConnectorProfile.enabled.is_(True)) + return query.order_by(FileConnectorProfile.scope_type.asc(), FileConnectorProfile.label.asc()).all() + + +def connector_profile_from_row(row: FileConnectorProfile, credential_row: FileConnectorCredential | None = None) -> ConnectorProfile: + policy_sources = [] + if row.policy: + policy_sources = connector_policy_sources_from_payload({ + "scope_type": row.scope_type, + "scope_id": row.scope_id, + "label": row.label, + "policy": row.policy, + }) + credential = connector_credential_from_row(credential_row) if credential_row is not None else None + if credential: + policy_sources.extend(credential.policy_sources) + return ConnectorProfile( + id=row.id, + label=row.label, + provider=row.provider, + scope_type=row.scope_type, + scope_id=row.scope_id, + endpoint_url=row.endpoint_url, + base_path=row.base_path, + enabled=row.enabled, + credential_profile_id=row.credential_profile_id, + credential_profile_label=credential.label if credential else None, + credential_mode=credential.credential_mode if credential else row.credential_mode, + username=credential.username if credential else row.username, + password_env=credential.password_env if credential else row.password_env, + token_env=credential.token_env if credential else row.token_env, + secret_ref=credential.secret_ref if credential else row.secret_ref, + password_value=credential.password_value if credential else decrypt_secret(row.password_encrypted), + token_value=credential.token_value if credential else decrypt_secret(row.token_encrypted), + capabilities=tuple(_string_list(row.capabilities)), + policy_sources=tuple(policy_sources), + metadata=dict(row.metadata_ or {}), + source_kind="database", + ) + + +def get_connector_profile_row( + session: Session, + *, + tenant_id: str, + profile_id: str, + include_disabled: bool = False, +) -> FileConnectorProfile: + row = session.get(FileConnectorProfile, profile_id) + if row is None or (row.scope_type != "system" and row.tenant_id != tenant_id): + raise FileStorageError("Connector profile not found") + if not include_disabled and not row.enabled: + raise FileStorageError("Connector profile not found") + return row + + +def create_connector_profile_row( + session: Session, + *, + tenant_id: str, + user_id: str | None, + profile_id: str, + label: str, + provider: str, + scope_type: str, + scope_id: str | None = None, + endpoint_url: str | None = None, + base_path: str | None = None, + enabled: bool = True, + credential_profile_id: str | None = None, + credential_mode: str = "none", + username: str | None = None, + password: str | None = None, + token: str | None = None, + password_env: str | None = None, + token_env: str | None = None, + secret_ref: str | None = None, + capabilities: list[str] | None = None, + policy: Mapping[str, Any] | None = None, + metadata: Mapping[str, Any] | None = None, +) -> FileConnectorProfile: + clean_id = _normalize_profile_id(profile_id) + if session.get(FileConnectorProfile, clean_id) is not None: + raise FileStorageError(f"Connector profile already exists: {clean_id}") + clean_scope_type, clean_scope_id, row_tenant_id = _normalize_scope(tenant_id=tenant_id, scope_type=scope_type, scope_id=scope_id) + row = FileConnectorProfile( + id=clean_id, + tenant_id=row_tenant_id, + scope_type=clean_scope_type, + scope_id=clean_scope_id, + label=_normalize_label(label), + provider=_normalize_provider(provider), + endpoint_url=_clean(endpoint_url), + base_path=_clean(base_path), + enabled=bool(enabled), + credential_profile_id=_clean(credential_profile_id), + credential_mode=_normalize_credential_mode(credential_mode), + username=_clean(username), + password_encrypted=encrypt_secret(_clean(password)), + token_encrypted=encrypt_secret(_clean(token)), + password_env=_clean(password_env), + token_env=_clean(token_env), + secret_ref=_clean(secret_ref), + capabilities=_string_list(capabilities), + policy=dict(policy or {}), + metadata_=dict(metadata or {}), + created_by_user_id=user_id, + updated_by_user_id=user_id, + ) + session.add(row) + session.flush() + return row + + +def update_connector_profile_row( + session: Session, + row: FileConnectorProfile, + *, + user_id: str | None, + label: str | None = None, + provider: str | None = None, + endpoint_url: str | None = None, + base_path: str | None = None, + enabled: bool | None = None, + credential_profile_id: str | None = None, + credential_mode: str | None = None, + username: str | None = None, + password: str | None = None, + token: str | None = None, + password_env: str | None = None, + token_env: str | None = None, + secret_ref: str | None = None, + capabilities: list[str] | None = None, + policy: Mapping[str, Any] | None = None, + metadata: Mapping[str, Any] | None = None, + clear_password: bool = False, + clear_token: bool = False, +) -> FileConnectorProfile: + if label is not None: + row.label = _normalize_label(label) + if provider is not None: + row.provider = _normalize_provider(provider) + if endpoint_url is not None: + row.endpoint_url = _clean(endpoint_url) + if base_path is not None: + row.base_path = _clean(base_path) + if enabled is not None: + row.enabled = bool(enabled) + if credential_profile_id is not None: + row.credential_profile_id = _clean(credential_profile_id) + if credential_mode is not None: + row.credential_mode = _normalize_credential_mode(credential_mode) + if username is not None: + row.username = _clean(username) + if password is not None: + row.password_encrypted = encrypt_secret(_clean(password)) + elif clear_password: + row.password_encrypted = None + if token is not None: + row.token_encrypted = encrypt_secret(_clean(token)) + elif clear_token: + row.token_encrypted = None + if password_env is not None: + row.password_env = _clean(password_env) + if token_env is not None: + row.token_env = _clean(token_env) + if secret_ref is not None: + row.secret_ref = _clean(secret_ref) + if capabilities is not None: + row.capabilities = _string_list(capabilities) + if policy is not None: + row.policy = dict(policy) + if metadata is not None: + row.metadata_ = dict(metadata) + row.updated_by_user_id = user_id + session.add(row) + session.flush() + return row + + +def deactivate_connector_profile_row(session: Session, row: FileConnectorProfile, *, user_id: str | None) -> FileConnectorProfile: + row.enabled = False + row.updated_by_user_id = user_id + session.add(row) + session.flush() + return row + + +def _normalize_scope(*, tenant_id: str, scope_type: str, scope_id: str | None) -> tuple[str, str | None, str | None]: + clean_scope_type = normalize_policy_scope_type(scope_type) + clean_scope_id = _clean(scope_id) + if clean_scope_type == "system": + return "system", None, None + if clean_scope_type == "tenant": + return "tenant", tenant_id, tenant_id + if clean_scope_type in {"user", "group", "campaign"}: + if not clean_scope_id: + raise FileStorageError(f"{clean_scope_type.capitalize()} connector profiles require scope_id") + return clean_scope_type, clean_scope_id, tenant_id + raise FileStorageError("Unsupported connector profile scope") + + +def _normalize_profile_id(value: str) -> str: + clean = _clean(value) + if not clean: + raise FileStorageError("Connector profile id is required") + if len(clean) > 255: + raise FileStorageError("Connector profile id is too long") + if any(char.isspace() for char in clean): + raise FileStorageError("Connector profile id cannot contain whitespace") + return clean + + +def _normalize_label(value: str) -> str: + clean = value.strip() + if not clean: + raise FileStorageError("Connector profile label is required") + if len(clean) > 255: + raise FileStorageError("Connector profile label is too long") + return clean + + +def _normalize_provider(value: str) -> str: + clean = value.strip().casefold() + if clean not in supported_connector_providers(): + raise FileStorageError(f"Unsupported connector provider: {value}") + return clean + + +def _normalize_credential_mode(value: str) -> str: + clean = value.strip().casefold() or "none" + if clean not in {"none", "anonymous", "basic", "token", "secret_ref"}: + raise FileStorageError(f"Unsupported connector credential mode: {value}") + return clean + + +def _string_list(value: object) -> list[str]: + if value is None: + return [] + if isinstance(value, str): + values = value.split(",") + elif isinstance(value, (list, tuple, set)): + values = value + else: + values = [value] + return [str(item).strip() for item in values if str(item).strip()] + + +def _clean(value: object) -> str | None: + if value is None: + return None + text = str(value).strip() + return text or None diff --git a/src/govoplan_files/backend/storage/connector_profiles.py b/src/govoplan_files/backend/storage/connector_profiles.py new file mode 100644 index 0000000..77d3a85 --- /dev/null +++ b/src/govoplan_files/backend/storage/connector_profiles.py @@ -0,0 +1,272 @@ +from __future__ import annotations + +import json +import os +from collections.abc import Iterable, Mapping +from dataclasses import dataclass, field +from typing import Any + +from govoplan_core.core.policy import normalize_policy_scope_type, policy_source_path +from govoplan_files.backend.storage.connector_policy import ConnectorPolicySource, connector_policy_sources_from_payload + + +_ENV_JSON_KEYS = ("GOVOPLAN_FILES_CONNECTOR_PROFILES_JSON", "FILES_CONNECTOR_PROFILES_JSON") +_ENV_FILE_KEYS = ("GOVOPLAN_FILES_CONNECTOR_PROFILES_FILE", "FILES_CONNECTOR_PROFILES_FILE") +_SUPPORTED_PROVIDERS = {"seafile", "nextcloud", "webdav", "smb", "nfs", "dms", "generic"} +_INLINE_SECRET_FIELDS = ("password", "token", "api_key", "access_key", "secret_key") + + +def supported_connector_providers() -> set[str]: + return set(_SUPPORTED_PROVIDERS) + + +@dataclass(frozen=True, slots=True) +class ConnectorProfile: + id: str + label: str + provider: str + scope_type: str = "system" + scope_id: str | None = None + endpoint_url: str | None = None + base_path: str | None = None + enabled: bool = True + credential_profile_id: str | None = None + credential_profile_label: str | None = None + credential_mode: str = "none" + username: str | None = None + password_env: str | None = None + token_env: str | None = None + secret_ref: str | None = None + password_value: str | None = field(default=None, repr=False) + token_value: str | None = field(default=None, repr=False) + has_inline_secret: bool = False + capabilities: tuple[str, ...] = () + policy_sources: tuple[ConnectorPolicySource, ...] = () + metadata: Mapping[str, Any] = field(default_factory=dict) + source_kind: str = "settings" + + @property + def source_path(self) -> str: + return policy_source_path(self.scope_type, self.scope_id) + + @property + def credential_source(self) -> str | None: + if self.credential_profile_id: + return "credential_profile" + if self.secret_ref: + return "secret_ref" + if self.token_value or self.password_value: + return "stored" + if self.token_env: + return "token_env" + if self.password_env: + return "password_env" + if self.has_inline_secret: + return "inline" + return None + + @property + def credentials_configured(self) -> bool: + mode = self.credential_mode.casefold() + if mode in {"", "none", "anonymous"}: + return True + if self.secret_ref or self.has_inline_secret or self.password_value or self.token_value: + return True + if self.token_env: + return bool(os.environ.get(self.token_env)) + if self.password_env: + return bool(os.environ.get(self.password_env)) + return False + + def to_response(self) -> dict[str, Any]: + return { + "id": self.id, + "label": self.label, + "provider": self.provider, + "endpoint_url": self.endpoint_url, + "base_path": self.base_path, + "enabled": self.enabled, + "scope_type": self.scope_type, + "scope_id": self.scope_id, + "source_path": self.source_path, + "credential_profile_id": self.credential_profile_id, + "credential_profile_label": self.credential_profile_label, + "credential_mode": self.credential_mode, + "credential_source": self.credential_source, + "credentials_configured": self.credentials_configured, + "username": self.username, + "capabilities": list(self.capabilities), + "policy_sources": [_policy_source_response(source) for source in self.policy_sources], + "metadata": dict(self.metadata), + "source_kind": self.source_kind, + } + + +def connector_profiles_from_settings(settings: object | None = None) -> list[ConnectorProfile]: + raw = _raw_profiles_payload(settings) + if raw in (None, ""): + return [] + payload = json.loads(raw) if isinstance(raw, str) else raw + return connector_profiles_from_payload(payload) + + +def connector_profiles_from_payload(value: object) -> list[ConnectorProfile]: + if isinstance(value, Mapping): + raw_profiles = value.get("profiles", []) + elif isinstance(value, list): + raw_profiles = value + else: + raise ValueError("Connector profiles must be a JSON object with profiles or a list") + if not isinstance(raw_profiles, list): + raise ValueError("Connector profiles payload requires a profiles list") + return [_profile_from_mapping(item) for item in raw_profiles if isinstance(item, Mapping)] + + +def _profile_from_mapping(value: Mapping[str, Any]) -> ConnectorProfile: + profile_id = _clean(value.get("id") or value.get("connector_id") or value.get("name")) + if not profile_id: + raise ValueError("Connector profiles require id") + provider = (_clean(value.get("provider") or value.get("type")) or "generic").casefold() + if provider not in _SUPPORTED_PROVIDERS: + raise ValueError(f"Unsupported connector provider: {provider}") + scope_type = normalize_policy_scope_type(str(value.get("scope_type") or "system")) + scope_id = _clean(value.get("scope_id")) + if scope_type == "system": + scope_id = None + elif not scope_id: + raise ValueError(f"{scope_type} connector profiles require scope_id") + + credentials = value.get("credentials") + credentials = credentials if isinstance(credentials, Mapping) else {} + mode = _clean(value.get("credential_mode") or value.get("auth_type") or credentials.get("mode") or credentials.get("type")) or "none" + profile = ConnectorProfile( + id=profile_id, + label=_clean(value.get("label") or value.get("name")) or profile_id, + provider=provider, + endpoint_url=_clean(value.get("endpoint_url") or value.get("base_url") or value.get("url")), + base_path=_clean(value.get("base_path") or value.get("path") or value.get("root")), + enabled=_bool(value.get("enabled"), default=True), + scope_type=scope_type, + scope_id=scope_id, + credential_profile_id=_clean(value.get("credential_profile_id") or value.get("credential_id")), + credential_profile_label=_clean(value.get("credential_profile_label") or value.get("credential_label")), + credential_mode=mode, + username=_clean(value.get("username") or credentials.get("username")), + password_env=_clean(value.get("password_env") or credentials.get("password_env")), + token_env=_clean(value.get("token_env") or credentials.get("token_env")), + secret_ref=_clean(value.get("secret_ref") or credentials.get("secret_ref")), + password_value=_clean(value.get("password") or credentials.get("password")), + token_value=_clean(value.get("token") or credentials.get("token")), + has_inline_secret=any(_clean(value.get(field) or credentials.get(field)) for field in _INLINE_SECRET_FIELDS), + capabilities=tuple(_string_list(value.get("capabilities"))), + metadata=_public_metadata(value.get("metadata")), + ) + return ConnectorProfile( + id=profile.id, + label=profile.label, + provider=profile.provider, + scope_type=profile.scope_type, + scope_id=profile.scope_id, + endpoint_url=profile.endpoint_url, + base_path=profile.base_path, + enabled=profile.enabled, + credential_profile_id=profile.credential_profile_id, + credential_profile_label=profile.credential_profile_label, + credential_mode=profile.credential_mode, + username=profile.username, + password_env=profile.password_env, + token_env=profile.token_env, + secret_ref=profile.secret_ref, + password_value=profile.password_value, + token_value=profile.token_value, + has_inline_secret=profile.has_inline_secret, + capabilities=profile.capabilities, + policy_sources=tuple(_policy_sources_from_profile(value, profile)), + metadata=profile.metadata, + source_kind="settings", + ) + + +def _raw_profiles_payload(settings: object | None) -> object: + for key in _ENV_JSON_KEYS: + value = os.environ.get(key) + if value: + return value + for key in _ENV_FILE_KEYS: + path = os.environ.get(key) + if path: + with open(path, encoding="utf-8") as handle: + return handle.read() + if settings is not None: + value = getattr(settings, "files_connector_profiles_json", None) + if value: + return value + value = getattr(settings, "files_connector_profiles", None) + if value: + return value + return None + + +def _policy_sources_from_profile(value: Mapping[str, Any], profile: ConnectorProfile) -> list[ConnectorPolicySource]: + raw_sources: list[Mapping[str, Any]] = [] + policy = value.get("policy") + if isinstance(policy, Mapping): + raw_sources.append({ + "scope_type": profile.scope_type, + "scope_id": profile.scope_id, + "label": profile.label, + "policy": policy, + }) + configured_sources = value.get("policy_sources") or value.get("connector_policy_sources") + if configured_sources is not None: + raw = connector_policy_sources_from_payload(configured_sources) + raw_sources.extend(_policy_source_response(source) for source in raw) + return connector_policy_sources_from_payload(raw_sources) + + +def _policy_source_response(source: ConnectorPolicySource) -> dict[str, Any]: + return { + "scope_type": source.scope_type, + "scope_id": source.scope_id, + "label": source.label, + "policy": dict(source.policy or {}), + } + + +def _public_metadata(value: object) -> Mapping[str, Any]: + if not isinstance(value, Mapping): + return {} + return { + str(key): item + for key, item in value.items() + if str(key).strip().casefold() not in _INLINE_SECRET_FIELDS + } + + +def _string_list(value: object) -> list[str]: + if value is None: + return [] + if isinstance(value, str): + values: Iterable[object] = value.split(",") + elif isinstance(value, Iterable) and not isinstance(value, (bytes, bytearray, Mapping)): + values = value + else: + values = (value,) + return [str(item).strip() for item in values if str(item).strip()] + + +def _bool(value: object, *, default: bool) -> bool: + if value is None: + return default + if isinstance(value, bool): + return value + if isinstance(value, str): + return value.strip().casefold() not in {"0", "false", "no", "off"} + return bool(value) + + +def _clean(value: object) -> str | None: + if value is None: + return None + text = str(value).strip() + return text or None diff --git a/src/govoplan_files/backend/storage/connector_providers.py b/src/govoplan_files/backend/storage/connector_providers.py new file mode 100644 index 0000000..58c101a --- /dev/null +++ b/src/govoplan_files/backend/storage/connector_providers.py @@ -0,0 +1,145 @@ +from __future__ import annotations + +from dataclasses import dataclass +from importlib.util import find_spec + + +CONNECTOR_PROVIDER_CAPABILITY = "files.connector.providers" + + +@dataclass(frozen=True, slots=True) +class ConnectorProviderDescriptor: + provider: str + label: str + protocol: str + implemented: bool + browse_supported: bool + import_supported: bool + optional_dependency: str | None + permission_model: str + sync_strategy: str + conflict_strategy: str + preview_strategy: str + audit_events: tuple[str, ...] + notes: str | None = None + + def to_response(self) -> dict[str, object]: + return { + "provider": self.provider, + "label": self.label, + "protocol": self.protocol, + "implemented": self.implemented, + "installed": self.installed, + "browse_supported": self.browse_supported, + "import_supported": self.import_supported, + "optional_dependency": self.optional_dependency, + "permission_model": self.permission_model, + "sync_strategy": self.sync_strategy, + "conflict_strategy": self.conflict_strategy, + "preview_strategy": self.preview_strategy, + "audit_events": list(self.audit_events), + "notes": self.notes, + } + + @property + def installed(self) -> bool: + if self.optional_dependency is None: + return self.implemented + return find_spec(self.optional_dependency) is not None + + +def connector_provider_descriptors() -> tuple[ConnectorProviderDescriptor, ...]: + freeze_model = "Imported or synced connector files become managed GovOPlaN files with frozen blob/version provenance." + governed_permissions = "GovOPlaN profile scope and connector policy are enforced before browse/import/sync; remote ACLs are still evaluated by the upstream service." + return ( + ConnectorProviderDescriptor( + provider="seafile", + label="Seafile", + protocol="seafile-api", + implemented=True, + browse_supported=True, + import_supported=True, + optional_dependency=None, + permission_model=governed_permissions, + sync_strategy="On-demand browse/import/sync; sync updates the managed file version when source content changes and never mutates the remote source.", + conflict_strategy="Managed import/sync uses existing files conflict_strategy handling: reject, overwrite, or rename.", + preview_strategy="Previews are generated from the frozen managed file after import or sync.", + audit_events=("files.connector.imported", "files.connector.synced", "files.connector.accessed"), + notes="Native account-token API for libraries, directories, file detail, and download-link import; WebDAV opt-in remains available.", + ), + ConnectorProviderDescriptor( + provider="nextcloud", + label="Nextcloud", + protocol="webdav", + implemented=True, + browse_supported=True, + import_supported=True, + optional_dependency=None, + permission_model=governed_permissions, + sync_strategy="On-demand WebDAV browse/import/sync; sync updates the managed file version when source content changes and never mutates the remote source.", + conflict_strategy="Managed import/sync uses existing files conflict_strategy handling: reject, overwrite, or rename.", + preview_strategy="Previews are generated from the frozen managed file after import or sync.", + audit_events=("files.connector.imported", "files.connector.synced", "files.connector.accessed"), + notes="Uses the configured Nextcloud WebDAV endpoint, basic auth, or bearer token profile credentials.", + ), + ConnectorProviderDescriptor( + provider="webdav", + label="WebDAV", + protocol="webdav", + implemented=True, + browse_supported=True, + import_supported=True, + optional_dependency=None, + permission_model=governed_permissions, + sync_strategy="On-demand WebDAV browse/import/sync; sync updates the managed file version when source content changes and never mutates the remote source.", + conflict_strategy="Managed import/sync uses existing files conflict_strategy handling: reject, overwrite, or rename.", + preview_strategy="Previews are generated from the frozen managed file after import or sync.", + audit_events=("files.connector.imported", "files.connector.synced", "files.connector.accessed"), + notes="Generic WebDAV provider for standards-compatible stores.", + ), + ConnectorProviderDescriptor( + provider="smb", + label="SMB", + protocol="smb", + implemented=True, + browse_supported=True, + import_supported=True, + optional_dependency="smbprotocol", + permission_model=governed_permissions, + sync_strategy="On-demand browse/import/sync over administrator-declared shares; sync updates managed versions and never mutates the remote share.", + conflict_strategy="Managed import/sync uses existing files conflict_strategy handling after download.", + preview_strategy="Previews are generated from the frozen managed file after import or sync, not directly from the share.", + audit_events=("files.connector.imported", "files.connector.synced", "files.connector.accessed"), + notes="Requires the optional smbprotocol dependency and an smb://server[:port]/share[/path] profile endpoint.", + ), + ConnectorProviderDescriptor( + provider="nfs", + label="NFS", + protocol="nfs", + implemented=False, + browse_supported=False, + import_supported=False, + optional_dependency=None, + permission_model=governed_permissions, + sync_strategy="Planned optional provider over administrator-mounted paths or a sidecar service.", + conflict_strategy="Will use managed import conflict_strategy handling after download.", + preview_strategy="Will preview from frozen managed file after import, not directly from the mount.", + audit_events=("files.connector.imported", "files.connector.synced", "files.connector.accessed"), + notes="Kept optional because NFS availability is deployment and host-kernel dependent.", + ), + ConnectorProviderDescriptor( + provider="local", + label="Local filesystem", + protocol="file", + implemented=False, + browse_supported=False, + import_supported=False, + optional_dependency=None, + permission_model="Local connector access should be restricted to administrator-declared roots plus GovOPlaN profile and policy checks.", + sync_strategy="Planned optional provider; managed storage local backend already exists separately.", + conflict_strategy=freeze_model, + preview_strategy="Will preview from frozen managed file after import or sync.", + audit_events=("files.connector.imported", "files.connector.synced", "files.connector.accessed"), + notes="Separate from the existing managed-file local storage backend.", + ), + ) diff --git a/src/govoplan_files/backend/storage/connector_spaces.py b/src/govoplan_files/backend/storage/connector_spaces.py new file mode 100644 index 0000000..d36f1a0 --- /dev/null +++ b/src/govoplan_files/backend/storage/connector_spaces.py @@ -0,0 +1,248 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +from sqlalchemy import false, or_ +from sqlalchemy.orm import Session + +from govoplan_files.backend.db.models import FileConnectorSpace +from govoplan_files.backend.storage.access import ensure_owner_access, user_group_ids +from govoplan_files.backend.storage.common import FileStorageError, utcnow +from govoplan_files.backend.storage.connector_browse import normalize_connector_browse_path +from govoplan_files.backend.storage.connector_profiles import ConnectorProfile + + +SYNC_MODES = {"manual"} + + +def connector_space_owner_id(space: FileConnectorSpace) -> str: + return space.owner_user_id if space.owner_type == "user" else space.owner_group_id # type: ignore[return-value] + + +def create_connector_space( + session: Session, + *, + tenant_id: str, + owner_type: str, + owner_id: str, + user_id: str, + label: str, + profile: ConnectorProfile, + library_id: str | None = None, + remote_path: str | None = None, + sync_mode: str = "manual", + read_only: bool = True, + metadata: Mapping[str, Any] | None = None, + is_admin: bool = False, +) -> FileConnectorSpace: + owner_type = owner_type.lower().strip() + ensure_owner_access(session, tenant_id=tenant_id, owner_type=owner_type, owner_id=owner_id, user_id=user_id, is_admin=is_admin) + label = _normalize_label(label) + sync_mode = _normalize_sync_mode(sync_mode) + remote_path = normalize_connector_browse_path(remote_path) + library_id = _clean_optional(library_id) + + existing = _owned_query(session, tenant_id=tenant_id, owner_type=owner_type, owner_id=owner_id).filter( + FileConnectorSpace.label == label, + FileConnectorSpace.deleted_at.is_(None), + ).first() + if existing is not None: + raise FileStorageError(f"Connector space already exists: {label}") + + space = FileConnectorSpace( + tenant_id=tenant_id, + owner_type=owner_type, + owner_user_id=owner_id if owner_type == "user" else None, + owner_group_id=owner_id if owner_type == "group" else None, + label=label, + connector_profile_id=profile.id, + provider=profile.provider, + library_id=library_id, + remote_path=remote_path, + sync_mode=sync_mode, + read_only=read_only, + is_active=True, + created_by_user_id=user_id, + metadata_=dict(metadata or {}), + ) + session.add(space) + session.flush() + return space + + +def list_connector_spaces_for_user( + session: Session, + *, + tenant_id: str, + user_id: str, + owner_type: str | None = None, + owner_id: str | None = None, + include_inactive: bool = False, + is_admin: bool = False, +) -> list[FileConnectorSpace]: + query = session.query(FileConnectorSpace).filter( + FileConnectorSpace.tenant_id == tenant_id, + FileConnectorSpace.deleted_at.is_(None), + ) + if not include_inactive: + query = query.filter(FileConnectorSpace.is_active.is_(True)) + + if owner_type: + if not owner_id: + raise FileStorageError("owner_id is required when owner_type is set") + owner_type = owner_type.lower().strip() + ensure_owner_access(session, tenant_id=tenant_id, owner_type=owner_type, owner_id=owner_id, user_id=user_id, is_admin=is_admin) + query = _owner_filter(query, owner_type, owner_id) + elif not is_admin: + group_ids = user_group_ids(session, tenant_id=tenant_id, user_id=user_id) + query = query.filter( + or_( + FileConnectorSpace.owner_user_id == user_id, + FileConnectorSpace.owner_group_id.in_(group_ids) if group_ids else false(), + ) + ) + + return query.order_by(FileConnectorSpace.owner_type.asc(), FileConnectorSpace.label.asc()).all() + + +def get_connector_space_for_user( + session: Session, + *, + tenant_id: str, + user_id: str, + space_id: str, + include_inactive: bool = False, + is_admin: bool = False, +) -> FileConnectorSpace: + query = session.query(FileConnectorSpace).filter( + FileConnectorSpace.id == space_id, + FileConnectorSpace.tenant_id == tenant_id, + FileConnectorSpace.deleted_at.is_(None), + ) + if not include_inactive: + query = query.filter(FileConnectorSpace.is_active.is_(True)) + space = query.one_or_none() + if space is None: + raise FileStorageError("Connector space not found") + ensure_owner_access( + session, + tenant_id=tenant_id, + owner_type=space.owner_type, + owner_id=connector_space_owner_id(space), + user_id=user_id, + is_admin=is_admin, + ) + return space + + +def update_connector_space( + session: Session, + space: FileConnectorSpace, + *, + user_id: str, + label: str | None = None, + library_id: str | None = None, + remote_path: str | None = None, + sync_mode: str | None = None, + is_active: bool | None = None, + metadata: Mapping[str, Any] | None = None, + is_admin: bool = False, +) -> FileConnectorSpace: + ensure_owner_access( + session, + tenant_id=space.tenant_id, + owner_type=space.owner_type, + owner_id=connector_space_owner_id(space), + user_id=user_id, + is_admin=is_admin, + ) + if label is not None: + new_label = _normalize_label(label) + existing = _owned_query( + session, + tenant_id=space.tenant_id, + owner_type=space.owner_type, + owner_id=connector_space_owner_id(space), + ).filter( + FileConnectorSpace.id != space.id, + FileConnectorSpace.label == new_label, + FileConnectorSpace.deleted_at.is_(None), + ).first() + if existing is not None: + raise FileStorageError(f"Connector space already exists: {new_label}") + space.label = new_label + if library_id is not None: + space.library_id = _clean_optional(library_id) + if remote_path is not None: + space.remote_path = normalize_connector_browse_path(remote_path) + if sync_mode is not None: + space.sync_mode = _normalize_sync_mode(sync_mode) + if is_active is not None: + space.is_active = bool(is_active) + if metadata is not None: + space.metadata_ = dict(metadata) + session.add(space) + session.flush() + return space + + +def soft_delete_connector_space( + session: Session, + space: FileConnectorSpace, + *, + user_id: str, + is_admin: bool = False, +) -> FileConnectorSpace: + ensure_owner_access( + session, + tenant_id=space.tenant_id, + owner_type=space.owner_type, + owner_id=connector_space_owner_id(space), + user_id=user_id, + is_admin=is_admin, + ) + space.deleted_at = utcnow() + space.is_active = False + session.add(space) + session.flush() + return space + + +def _owned_query(session: Session, *, tenant_id: str, owner_type: str, owner_id: str): + query = session.query(FileConnectorSpace).filter( + FileConnectorSpace.tenant_id == tenant_id, + FileConnectorSpace.owner_type == owner_type, + ) + return _owner_filter(query, owner_type, owner_id) + + +def _owner_filter(query, owner_type: str, owner_id: str): + if owner_type == "user": + return query.filter(FileConnectorSpace.owner_user_id == owner_id) + if owner_type == "group": + return query.filter(FileConnectorSpace.owner_group_id == owner_id) + raise FileStorageError("Files must be owned by a user or group") + + +def _normalize_label(value: str) -> str: + label = value.strip() + if not label: + raise FileStorageError("Connector space label is required") + if len(label) > 255: + raise FileStorageError("Connector space label is too long") + return label + + +def _normalize_sync_mode(value: str) -> str: + mode = value.strip().casefold() + if mode not in SYNC_MODES: + raise FileStorageError(f"Unsupported connector space sync mode: {value}") + return mode + + +def _clean_optional(value: object) -> str | None: + if value is None: + return None + text = str(value).strip() + return text or None diff --git a/src/govoplan_files/backend/storage/files.py b/src/govoplan_files/backend/storage/files.py index 7fbf070..d4ea9ff 100644 --- a/src/govoplan_files/backend/storage/files.py +++ b/src/govoplan_files/backend/storage/files.py @@ -6,7 +6,7 @@ from pathlib import PurePosixPath from typing import Any, Iterable from uuid import uuid4 -from sqlalchemy import or_ +from sqlalchemy import and_, or_ from sqlalchemy.orm import Session from govoplan_core.core.campaigns import CAPABILITY_CAMPAIGNS_ACCESS, CampaignAccessProvider @@ -16,6 +16,7 @@ from govoplan_files.backend.storage.access import ensure_owner_access, ensure_sh from govoplan_files.backend.storage.backends import StorageBackendError, get_storage_backend from govoplan_files.backend.storage.common import FileConflictResolution, FileStorageError, UploadedStoredFile, utcnow from govoplan_files.backend.storage.paths import filename_from_path, join_folder_filename, normalize_folder, normalize_logical_path, safe_storage_component +from govoplan_files.backend.storage.provenance import source_provenance_from_metadata def _campaign_access_provider() -> CampaignAccessProvider: @@ -181,6 +182,136 @@ def create_file_asset( return UploadedStoredFile(asset=asset, version=version, blob=blob) +def sync_file_asset_from_source( + session: Session, + *, + tenant_id: str, + owner_type: str, + owner_id: str, + user_id: str, + filename: str, + data: bytes, + metadata: dict[str, Any], + folder: str | None = None, + display_path: str | None = None, + content_type: str | None = None, + campaign_id: str | None = None, + conflict_strategy: str = "rename", + is_admin: bool = False, +) -> tuple[UploadedStoredFile, str, str | None]: + owner_type = owner_type.lower().strip() + ensure_owner_access(session, tenant_id=tenant_id, owner_type=owner_type, owner_id=owner_id, user_id=user_id, is_admin=is_admin) + provenance = source_provenance_from_metadata(metadata) + if not provenance: + raise FileStorageError("Connector sync requires source provenance") + existing = find_asset_by_source( + session, + tenant_id=tenant_id, + owner_type=owner_type, + owner_id=owner_id, + source_provenance=provenance, + ) + if existing is None: + stored = create_file_asset( + session, + tenant_id=tenant_id, + owner_type=owner_type, + owner_id=owner_id, + user_id=user_id, + filename=filename, + data=data, + folder=folder, + display_path=display_path, + content_type=content_type, + metadata=metadata, + campaign_id=campaign_id, + conflict_strategy=conflict_strategy, + is_admin=is_admin, + ) + return stored, "created", None + + previous_version_id = existing.current_version_id + stored, action = update_file_asset_content( + session, + existing, + tenant_id=tenant_id, + user_id=user_id, + filename=filename, + data=data, + content_type=content_type, + metadata=metadata, + ) + if campaign_id: + share_file(session, tenant_id=tenant_id, asset=existing, target_type="campaign", target_id=campaign_id, permission="read", user_id=user_id) + return stored, action, previous_version_id + + +def find_asset_by_source( + session: Session, + *, + tenant_id: str, + owner_type: str, + owner_id: str, + source_provenance: dict[str, Any], +) -> FileAsset | None: + wanted = _source_identity(source_provenance) + if wanted is None: + return None + assets = ( + _asset_query_for_owner(session, tenant_id=tenant_id, owner_type=owner_type, owner_id=owner_id) + .filter(FileAsset.deleted_at.is_(None)) + .order_by(FileAsset.updated_at.desc()) + .all() + ) + for asset in assets: + if _source_identity(source_provenance_from_metadata(asset.metadata_ or {})) == wanted: + return asset + return None + + +def update_file_asset_content( + session: Session, + asset: FileAsset, + *, + tenant_id: str, + user_id: str, + filename: str, + data: bytes, + content_type: str | None, + metadata: dict[str, Any], +) -> tuple[UploadedStoredFile, str]: + if asset.tenant_id != tenant_id or asset.deleted_at is not None: + raise FileStorageError("File not found") + safe_filename = filename_from_path(normalize_logical_path(filename, fallback_filename="file")) + if not content_type: + content_type = mimetypes.guess_type(safe_filename)[0] or "application/octet-stream" + current_version, current_blob = current_version_and_blob(session, asset) + checksum = hashlib.sha256(data).hexdigest() + asset.metadata_ = metadata + session.add(asset) + if current_blob.checksum_sha256 == checksum and current_blob.size_bytes == len(data): + return UploadedStoredFile(asset=asset, version=current_version, blob=current_blob), "unchanged" + + blob = _get_or_create_blob(session, tenant_id=tenant_id, data=data, filename=safe_filename, content_type=content_type) + version = FileVersion( + tenant_id=tenant_id, + file_asset_id=asset.id, + blob_id=blob.id, + version_number=_next_version_number(session, asset.id), + filename_at_upload=safe_filename, + display_path_at_upload=asset.display_path, + content_type=content_type, + size_bytes=blob.size_bytes, + checksum_sha256=blob.checksum_sha256, + created_by_user_id=user_id, + ) + session.add(version) + session.flush() + asset.current_version_id = version.id + session.add(asset) + return UploadedStoredFile(asset=asset, version=version, blob=blob), "updated" + + def get_asset_for_user(session: Session, *, tenant_id: str, user_id: str, asset_id: str, require_write: bool = False, is_admin: bool = False) -> FileAsset: asset = session.get(FileAsset, asset_id) if not asset or asset.tenant_id != tenant_id or asset.deleted_at is not None: @@ -224,6 +355,71 @@ def list_assets_for_user( include_deleted: bool = False, is_admin: bool = False, ) -> list[FileAsset]: + query = _asset_visibility_query_for_user( + session, + tenant_id=tenant_id, + user_id=user_id, + owner_type=owner_type, + owner_id=owner_id, + campaign_id=campaign_id, + path_prefix=path_prefix, + include_deleted=include_deleted, + is_admin=is_admin, + ) + return query.order_by(FileAsset.display_path.asc(), FileAsset.updated_at.desc(), FileAsset.id.asc()).all() + + +def list_assets_for_user_window( + session: Session, + *, + tenant_id: str, + user_id: str, + owner_type: str | None = None, + owner_id: str | None = None, + campaign_id: str | None = None, + path_prefix: str | None = None, + include_deleted: bool = False, + is_admin: bool = False, + page_size: int, + after_display_path: str | None = None, + after_updated_at=None, + after_id: str | None = None, +) -> tuple[list[FileAsset], bool]: + query = _asset_visibility_query_for_user( + session, + tenant_id=tenant_id, + user_id=user_id, + owner_type=owner_type, + owner_id=owner_id, + campaign_id=campaign_id, + path_prefix=path_prefix, + include_deleted=include_deleted, + is_admin=is_admin, + ) + if after_display_path is not None and after_updated_at is not None and after_id: + query = query.filter( + or_( + FileAsset.display_path > after_display_path, + and_(FileAsset.display_path == after_display_path, FileAsset.updated_at < after_updated_at), + and_(FileAsset.display_path == after_display_path, FileAsset.updated_at == after_updated_at, FileAsset.id > after_id), + ) + ) + rows = query.order_by(FileAsset.display_path.asc(), FileAsset.updated_at.desc(), FileAsset.id.asc()).limit(page_size + 1).all() + return rows[:page_size], len(rows) > page_size + + +def _asset_visibility_query_for_user( + session: Session, + *, + tenant_id: str, + user_id: str, + owner_type: str | None = None, + owner_id: str | None = None, + campaign_id: str | None = None, + path_prefix: str | None = None, + include_deleted: bool = False, + is_admin: bool = False, +): query = session.query(FileAsset).filter(FileAsset.tenant_id == tenant_id) if not include_deleted: query = query.filter(FileAsset.deleted_at.is_(None)) @@ -255,7 +451,7 @@ def list_assets_for_user( prefix = normalize_folder(path_prefix) if prefix: query = query.filter(FileAsset.display_path.like(f"{prefix}/%")) - return query.order_by(FileAsset.display_path.asc(), FileAsset.updated_at.desc()).all() + return query def current_version_and_blob(session: Session, asset: FileAsset) -> tuple[FileVersion, FileBlob]: @@ -553,6 +749,39 @@ def _normalize_conflict_strategy(strategy: str | None) -> str: return normalized +def _next_version_number(session: Session, asset_id: str) -> int: + row = ( + session.query(FileVersion.version_number) + .filter(FileVersion.file_asset_id == asset_id) + .order_by(FileVersion.version_number.desc()) + .first() + ) + return (int(row[0]) if row else 0) + 1 + + +def _source_identity(provenance: dict[str, Any] | None) -> tuple[object, ...] | None: + if not provenance: + return None + connector_id = _clean_identity(provenance.get("connector_id")) + provider = _clean_identity(provenance.get("provider")) + external_id = _clean_identity(provenance.get("external_id")) + if connector_id and external_id: + return ("external_id", connector_id, provider, external_id) + external_path = _clean_identity(provenance.get("external_path")) + metadata = provenance.get("metadata") if isinstance(provenance.get("metadata"), dict) else {} + library_id = _clean_identity(metadata.get("library_id") or metadata.get("profile_id") or metadata.get("share")) + if connector_id and external_path: + return ("external_path", connector_id, provider, library_id, external_path) + return None + + +def _clean_identity(value: object) -> str | None: + if value is None: + return None + text = str(value).strip() + return text or None + + def _copy_asset_to_path( session: Session, asset: FileAsset, diff --git a/src/govoplan_files/backend/storage/folders.py b/src/govoplan_files/backend/storage/folders.py index fa91e84..35fe8cd 100644 --- a/src/govoplan_files/backend/storage/folders.py +++ b/src/govoplan_files/backend/storage/folders.py @@ -68,13 +68,63 @@ def list_folders_for_user( include_deleted: bool = False, is_admin: bool = False, ) -> list[FileFolder]: + query = _folder_visibility_query_for_user( + session, + tenant_id=tenant_id, + user_id=user_id, + owner_type=owner_type, + owner_id=owner_id, + include_deleted=include_deleted, + is_admin=is_admin, + ) + return query.order_by(FileFolder.path.asc(), FileFolder.id.asc()).all() + + +def list_folders_for_user_window( + session: Session, + *, + tenant_id: str, + user_id: str, + owner_type: str, + owner_id: str, + include_deleted: bool = False, + is_admin: bool = False, + page_size: int, + after_path: str | None = None, + after_id: str | None = None, +) -> tuple[list[FileFolder], bool]: + query = _folder_visibility_query_for_user( + session, + tenant_id=tenant_id, + user_id=user_id, + owner_type=owner_type, + owner_id=owner_id, + include_deleted=include_deleted, + is_admin=is_admin, + ) + if after_path is not None and after_id: + query = query.filter(or_(FileFolder.path > after_path, (FileFolder.path == after_path) & (FileFolder.id > after_id))) + rows = query.order_by(FileFolder.path.asc(), FileFolder.id.asc()).limit(page_size + 1).all() + return rows[:page_size], len(rows) > page_size + + +def _folder_visibility_query_for_user( + session: Session, + *, + tenant_id: str, + user_id: str, + owner_type: str, + owner_id: str, + include_deleted: bool = False, + is_admin: bool = False, +): owner_type = owner_type.lower().strip() ensure_owner_access(session, tenant_id=tenant_id, owner_type=owner_type, owner_id=owner_id, user_id=user_id, is_admin=is_admin) query = session.query(FileFolder).filter(FileFolder.tenant_id == tenant_id, FileFolder.owner_type == owner_type) query = _owner_filter(query, owner_type, owner_id) if not include_deleted: query = query.filter(FileFolder.deleted_at.is_(None)) - return query.order_by(FileFolder.path.asc()).all() + return query def soft_delete_folder( diff --git a/src/govoplan_files/backend/storage/provenance.py b/src/govoplan_files/backend/storage/provenance.py new file mode 100644 index 0000000..1ba5757 --- /dev/null +++ b/src/govoplan_files/backend/storage/provenance.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + + +SOURCE_PROVENANCE_METADATA_KEY = "source_provenance" +SOURCE_REVISION_METADATA_KEY = "source_revision" + +_PROVENANCE_STRING_FIELDS = { + "source_type", + "connector_id", + "provider", + "external_id", + "external_path", + "external_url", + "revision", + "revision_label", + "observed_at", + "imported_at", +} + + +def normalize_source_revision(value: object) -> str | None: + if value is None: + return None + text = str(value).strip() + return text or None + + +def normalize_source_provenance(value: object, *, source_revision: object = None) -> dict[str, Any] | None: + if value is None: + revision = normalize_source_revision(source_revision) + return {"revision": revision} if revision else None + if not isinstance(value, Mapping): + raise ValueError("source_provenance must be a JSON object") + + payload: dict[str, Any] = {} + for field in _PROVENANCE_STRING_FIELDS: + normalized = normalize_source_revision(value.get(field)) + if normalized: + payload[field] = normalized + + extra = value.get("metadata") + if isinstance(extra, Mapping) and extra: + payload["metadata"] = dict(extra) + elif extra is not None and extra != "": + raise ValueError("source_provenance.metadata must be a JSON object") + + revision = normalize_source_revision(source_revision) + if revision: + payload["revision"] = revision + return payload or None + + +def source_metadata( + *, + existing: Mapping[str, Any] | None = None, + source_provenance: object = None, + source_revision: object = None, +) -> dict[str, Any] | None: + payload = dict(existing or {}) + normalized_provenance = normalize_source_provenance(source_provenance, source_revision=source_revision) + normalized_revision = normalize_source_revision(source_revision) + if normalized_provenance: + payload[SOURCE_PROVENANCE_METADATA_KEY] = normalized_provenance + if normalized_revision: + payload[SOURCE_REVISION_METADATA_KEY] = normalized_revision + elif normalized_provenance and normalized_provenance.get("revision"): + payload[SOURCE_REVISION_METADATA_KEY] = str(normalized_provenance["revision"]) + return payload or None + + +def source_provenance_from_metadata(metadata: Mapping[str, Any] | None) -> dict[str, Any] | None: + if not isinstance(metadata, Mapping): + return None + value = metadata.get(SOURCE_PROVENANCE_METADATA_KEY) + if isinstance(value, Mapping): + return dict(value) + return None + + +def source_revision_from_metadata(metadata: Mapping[str, Any] | None) -> str | None: + if not isinstance(metadata, Mapping): + return None + revision = normalize_source_revision(metadata.get(SOURCE_REVISION_METADATA_KEY)) + if revision: + return revision + provenance = source_provenance_from_metadata(metadata) + return normalize_source_revision(provenance.get("revision") if provenance else None) diff --git a/src/govoplan_files/backend/storage/services.py b/src/govoplan_files/backend/storage/services.py index 62efb23..080157f 100644 --- a/src/govoplan_files/backend/storage/services.py +++ b/src/govoplan_files/backend/storage/services.py @@ -51,7 +51,7 @@ from govoplan_files.backend.storage.folders import ( soft_delete_folder, ) from govoplan_files.backend.storage.search import match_assets, resolve_patterns -from govoplan_files.backend.storage.transfers import build_rename_preview, rename_selection, transfer_selection +from govoplan_files.backend.storage.transfers import build_rename_preview, legacy_rename_selection_without_owner_context, rename_selection, transfer_selection __all__ = [ "FileConflictResolution", @@ -76,6 +76,7 @@ __all__ = [ "match_assets", "read_asset_bytes", "record_campaign_attachment_uses_for_job", + "legacy_rename_selection_without_owner_context", "rename_asset", "rename_selection", "resolve_patterns", diff --git a/src/govoplan_files/backend/storage/transfers.py b/src/govoplan_files/backend/storage/transfers.py index 81d258c..77e285f 100644 --- a/src/govoplan_files/backend/storage/transfers.py +++ b/src/govoplan_files/backend/storage/transfers.py @@ -341,7 +341,7 @@ def _file_new_path_for_root(path: str, root: str, new_root: str, *, recursive: b return normalize_logical_path(f"{new_root}/{relative}") -def rename_selection( +def _rename_selection( session: Session, *, tenant_id: str, @@ -359,6 +359,7 @@ def rename_selection( recursive: bool = False, dry_run: bool = True, is_admin: bool = False, + allow_legacy_without_owner_context: bool = False, ) -> list[RenamePlanItem]: mode = mode.lower().strip() if mode not in {"direct", "prefix", "suffix", "replace"}: @@ -387,10 +388,12 @@ def rename_selection( file_ids=selected_file_ids, ) ) - else: + elif allow_legacy_without_owner_context: for file_id in selected_file_ids: asset = get_asset_for_user(session, tenant_id=tenant_id, user_id=user_id, asset_id=file_id, require_write=True, is_admin=is_admin) assets[asset.id] = asset + else: + raise FileStorageError("Bulk rename requires an owner file space") folder_rows_by_path: dict[str, FileFolder] = {} affected_folder_paths: set[str] = set() @@ -480,3 +483,80 @@ def rename_selection( rename_asset(assets[asset_id], new_path=new_path) session.add(assets[asset_id]) return plan + + +def rename_selection( + session: Session, + *, + tenant_id: str, + user_id: str, + file_ids: list[str], + folder_paths: list[str], + owner_type: str, + owner_id: str, + mode: str, + new_name: str | None = None, + find: str | None = None, + replacement: str = "", + prefix: str = "", + suffix: str = "", + recursive: bool = False, + dry_run: bool = True, + is_admin: bool = False, +) -> list[RenamePlanItem]: + return _rename_selection( + session, + tenant_id=tenant_id, + user_id=user_id, + file_ids=file_ids, + folder_paths=folder_paths, + owner_type=owner_type, + owner_id=owner_id, + mode=mode, + new_name=new_name, + find=find, + replacement=replacement, + prefix=prefix, + suffix=suffix, + recursive=recursive, + dry_run=dry_run, + is_admin=is_admin, + ) + + +def legacy_rename_selection_without_owner_context( + session: Session, + *, + tenant_id: str, + user_id: str, + file_ids: list[str], + mode: str, + new_name: str | None = None, + find: str | None = None, + replacement: str = "", + prefix: str = "", + suffix: str = "", + dry_run: bool = True, + is_admin: bool = False, +) -> list[RenamePlanItem]: + """Compatibility path for historical file-only callers that lack owner scope.""" + + return _rename_selection( + session, + tenant_id=tenant_id, + user_id=user_id, + file_ids=file_ids, + folder_paths=[], + owner_type=None, + owner_id=None, + mode=mode, + new_name=new_name, + find=find, + replacement=replacement, + prefix=prefix, + suffix=suffix, + recursive=False, + dry_run=dry_run, + is_admin=is_admin, + allow_legacy_without_owner_context=True, + ) diff --git a/webui/package.json b/webui/package.json index 50ea54e..648b9b6 100644 --- a/webui/package.json +++ b/webui/package.json @@ -20,7 +20,7 @@ "react": "^19.0.0", "react-dom": "^19.0.0", "react-router-dom": "^7.1.1", - "lucide-react": "^0.555.0", + "lucide-react": "^1.23.0", "@govoplan/core-webui": "^0.1.6" }, "peerDependenciesMeta": { diff --git a/webui/src/api/files.ts b/webui/src/api/files.ts index 5bcba56..18062cd 100644 --- a/webui/src/api/files.ts +++ b/webui/src/api/files.ts @@ -1,4 +1,4 @@ -import { ApiError, apiFetch, apiUrl, authHeaders, csrfToken, type ApiSettings, type FilesManagedFileLinkTarget } from "@govoplan/core-webui"; +import { ApiError, apiFetch, apiUrl, authHeaders, csrfToken, type ApiSettings, type DeltaDeletedItem, type FilesManagedFileLinkTarget } from "@govoplan/core-webui"; export type FileSpace = { id: string; @@ -6,6 +6,14 @@ export type FileSpace = { owner_type: "user" | "group"; owner_id: string; description?: string | null; + space_type?: "managed" | "connector"; + connector_space_id?: string | null; + connector_profile_id?: string | null; + provider?: string | null; + library_id?: string | null; + remote_path?: string | null; + sync_mode?: string | null; + read_only?: boolean; }; export type FileShare = { @@ -17,6 +25,250 @@ export type FileShare = { revoked_at?: string | null; }; +export type FileSourceProvenance = { + source_type?: string | null; + connector_id?: string | null; + provider?: string | null; + external_id?: string | null; + external_path?: string | null; + external_url?: string | null; + revision?: string | null; + revision_label?: string | null; + observed_at?: string | null; + imported_at?: string | null; + metadata?: Record; +}; + +export type FileConnectorPolicySource = { + scope_type: "system" | "tenant" | "user" | "group" | "campaign"; + scope_id?: string | null; + label?: string | null; + policy: Record; +}; + +export type FileConnectorPolicyDecision = { + allowed: boolean; + reason?: string | null; + source_path: unknown[]; + requirements: string[]; + details: Record; +}; + +export type FileConnectorPolicyStep = { + scope_type: "system" | "tenant" | "user" | "group" | "campaign"; + scope_id?: string | null; + path: string; + label: string; + applied_fields: string[]; + policy: Record; +}; + +export type FileConnectorPolicyResponse = { + scope_type: "system" | "tenant" | "user" | "group" | "campaign"; + scope_id?: string | null; + policy: Record; + effective_policy?: Record | null; + parent_policy?: Record | null; + effective_policy_sources?: FileConnectorPolicyStep[]; + parent_policy_sources?: FileConnectorPolicyStep[]; +}; + +export type FileConnectorProfile = { + id: string; + label: string; + provider: string; + endpoint_url?: string | null; + base_path?: string | null; + enabled: boolean; + scope_type: "system" | "tenant" | "user" | "group" | "campaign"; + scope_id?: string | null; + source_path: string; + credential_profile_id?: string | null; + credential_profile_label?: string | null; + credential_mode: string; + credential_source?: string | null; + credentials_configured: boolean; + username?: string | null; + capabilities: string[]; + policy_sources: FileConnectorPolicySource[]; + metadata: Record; + source_kind?: string; +}; + +export type FileConnectorCredential = { + id: string; + label: string; + provider?: string | null; + enabled: boolean; + scope_type: "system" | "tenant" | "user" | "group" | "campaign"; + scope_id?: string | null; + source_path: string; + credential_mode: string; + credential_secret_source?: string | null; + credentials_configured: boolean; + username?: string | null; + policy_sources: FileConnectorPolicySource[]; + metadata: Record; + source_kind?: string; +}; + +export type FileConnectorCredentialsPayload = { + username?: string | null; + password?: string | null; + token?: string | null; + password_env?: string | null; + token_env?: string | null; + secret_ref?: string | null; +}; + +export type FileConnectorDiscoveryCandidate = { + endpoint_url: string; + status: "failed" | "usable" | "found" | "credentials_rejected"; + message: string; +}; + +export type FileConnectorDiscoveryPayload = { + provider: FileConnectorProfilePayload["provider"]; + endpoint_url: string; + base_path?: string | null; + credential_mode?: FileConnectorProfilePayload["credential_mode"]; + credentials?: FileConnectorCredentialsPayload; + metadata?: Record; + require_valid_credentials?: boolean; +}; + +export type FileConnectorDiscoveryResponse = { + provider: string; + endpoint_url?: string | null; + base_path?: string | null; + status: "usable" | "found" | "credentials_rejected" | "not_found" | "unsupported"; + message: string; + candidates: FileConnectorDiscoveryCandidate[]; + metadata: Record; +}; + +export type FileConnectorProfilePayload = { + id: string; + label: string; + provider: "seafile" | "nextcloud" | "webdav" | "smb" | "nfs" | "dms" | "generic"; + endpoint_url?: string | null; + base_path?: string | null; + enabled?: boolean; + scope_type?: "system" | "tenant" | "user" | "group" | "campaign"; + scope_id?: string | null; + credential_profile_id?: string | null; + credential_mode?: "none" | "anonymous" | "basic" | "token" | "secret_ref"; + credentials?: FileConnectorCredentialsPayload; + capabilities?: string[]; + policy?: Record; + metadata?: Record; +}; + +export type FileConnectorProfileUpdatePayload = Partial> & { + clear_password?: boolean; + clear_token?: boolean; +}; + +export type FileConnectorCredentialPayload = { + id: string; + label: string; + provider?: "seafile" | "nextcloud" | "webdav" | "smb" | "nfs" | "dms" | "generic" | null; + enabled?: boolean; + scope_type?: "system" | "tenant" | "user" | "group" | "campaign"; + scope_id?: string | null; + credential_mode?: "none" | "anonymous" | "basic" | "token" | "secret_ref"; + credentials?: FileConnectorCredentialsPayload; + policy?: Record; + metadata?: Record; +}; + +export type FileConnectorCredentialUpdatePayload = Partial> & { + clear_password?: boolean; + clear_token?: boolean; +}; + +export type FileConnectorBrowseItem = { + kind: "library" | "folder" | "file"; + name: string; + path: string; + external_id?: string | null; + external_url?: string | null; + size_bytes?: number | null; + content_type?: string | null; + modified_at?: string | null; + etag?: string | null; + metadata: Record; +}; + +export type FileConnectorBrowseResponse = { + profile_id: string; + provider: string; + path: string; + library_id?: string | null; + read_only: boolean; + decision: FileConnectorPolicyDecision; + items: FileConnectorBrowseItem[]; +}; + +export type FileConnectorProvider = { + provider: string; + label: string; + protocol: string; + implemented: boolean; + installed: boolean; + browse_supported: boolean; + import_supported: boolean; + optional_dependency?: string | null; + permission_model: string; + sync_strategy: string; + conflict_strategy: string; + preview_strategy: string; + audit_events: string[]; + notes?: string | null; +}; + +export type FileConnectorSpace = { + id: string; + tenant_id: string; + owner_type: "user" | "group"; + owner_id: string; + label: string; + connector_profile_id: string; + provider: string; + library_id?: string | null; + remote_path: string; + sync_mode: string; + read_only: boolean; + is_active: boolean; + created_at: string; + updated_at: string; + deleted_at?: string | null; + metadata: Record; +}; + +export type FileConnectorSettingsDeltaResponse = { + profiles: FileConnectorProfile[]; + credentials: FileConnectorCredential[]; + spaces: FileConnectorSpace[]; + policy?: FileConnectorPolicyResponse | null; + changed_sections?: string[]; + deleted: DeltaDeletedItem[]; + watermark?: string | null; + has_more: boolean; + full: boolean; +}; + +export type FileConnectorSpacePayload = { + owner_type?: "user" | "group"; + owner_id?: string | null; + label: string; + connector_profile_id: string; + library_id?: string | null; + remote_path?: string; + sync_mode?: "manual"; + metadata?: Record; +}; + export type ManagedFile = { id: string; tenant_id: string; @@ -34,13 +286,43 @@ export type ManagedFile = { deleted_at?: string | null; audit_relevant: boolean; metadata?: Record | null; + source_provenance?: FileSourceProvenance | null; + source_revision?: string | null; shares?: FileShare[]; }; -export type FileListResponse = { files: ManagedFile[] }; -export type FileSpacesResponse = { spaces: FileSpace[] }; -export type FileUploadResponse = { files: ManagedFile[] }; -export type FileUploadProgress = { loaded: number; total?: number; percentage: number | null }; +export type FileListResponse = {files: ManagedFile[];cursor?: string | null;next_cursor?: string | null;watermark?: string | null;}; +export type FileDeltaDeletedItem = {id: string;resource_type?: string | null;revision?: string | null;deleted_at?: string | null;}; +export type FileDeltaResponse = { + files: ManagedFile[]; + folders: FileFolder[]; + deleted: FileDeltaDeletedItem[]; + watermark?: string | null; + has_more: boolean; + full: boolean; +}; +export type ManagedFileSnapshotResponse = {files: ManagedFile[];folders: FileFolder[];watermark?: string | null;}; +export type FileSpacesResponse = {spaces: FileSpace[];}; +export type FileUploadResponse = {files: ManagedFile[];}; +export type FileUploadProgress = {loaded: number;total?: number;percentage: number | null;}; +export type FileConnectorFilePayload = { + library_id: string; + path: string; + owner_type: "user" | "group"; + owner_id?: string | null; + target_folder?: string | null; + target_path?: string | null; + campaign_id?: string | null; + conflict_strategy?: ConflictStrategy; + source_revision?: string | null; + metadata?: Record; +}; +export type FileConnectorSyncResponse = { + file: ManagedFile; + action: "created" | "updated" | "unchanged"; + previous_version_id?: string | null; + current_version_id: string; +}; export type FileFolder = { id: string; tenant_id: string; @@ -51,16 +333,18 @@ export type FileFolder = { updated_at: string; deleted_at?: string | null; }; -export type FileFoldersResponse = { folders: FileFolder[] }; -export type FolderDeleteResponse = { deleted_folders: number; deleted_files: number }; -export type BulkDeleteResponse = { deleted_count: number }; -export type RenameResponse = { dry_run: boolean; items: { kind: "file" | "folder"; id: string; file_id?: string | null; folder_path?: string | null; old_path: string; new_path: string }[] }; -export type TransferResponse = { operation: "move" | "copy"; files: number; folders: number }; +export type FileFoldersResponse = {folders: FileFolder[];cursor?: string | null;next_cursor?: string | null;watermark?: string | null;}; + +const DEFAULT_MANAGED_FILE_WINDOW_SIZE = 500; +export type FolderDeleteResponse = {deleted_folders: number;deleted_files: number;}; +export type BulkDeleteResponse = {deleted_count: number;}; +export type RenameResponse = {dry_run: boolean;items: {kind: "file" | "folder";id: string;file_id?: string | null;folder_path?: string | null;old_path: string;new_path: string;}[];}; +export type TransferResponse = {operation: "move" | "copy";files: number;folders: number;}; export type ConflictAction = "overwrite" | "rename" | "skip"; export type ConflictStrategy = "reject" | "overwrite" | "rename"; -export type ConflictResolution = { target_path: string; action: ConflictAction; new_path?: string }; +export type ConflictResolution = {target_path: string;action: ConflictAction;new_path?: string;}; export type PatternResolveResponse = { - patterns: { pattern: string; matches: ManagedFile[] }[]; + patterns: {pattern: string;matches: ManagedFile[];}[]; unmatched: ManagedFile[]; }; @@ -70,50 +354,141 @@ export function listFileSpaces(settings: ApiSettings): Promise { +export function listFolders(settings: ApiSettings, params: {owner_type: "user" | "group";owner_id: string;page_size?: number;cursor?: string | null;}): Promise { const search = new URLSearchParams(); search.set("owner_type", params.owner_type); search.set("owner_id", params.owner_id); + if (params.page_size) search.set("page_size", String(params.page_size)); + if (params.cursor) search.set("cursor", params.cursor); return apiFetch(settings, `/api/v1/files/folders?${search.toString()}`); } export function createFolder( - settings: ApiSettings, - payload: { owner_type: "user" | "group"; owner_id: string; path: string } -): Promise { +settings: ApiSettings, +payload: {owner_type: "user" | "group";owner_id: string;path: string;}) +: Promise { return apiFetch(settings, "/api/v1/files/folders", { method: "POST", body: JSON.stringify(payload) }); } export function deleteFolder( - settings: ApiSettings, - payload: { owner_type: "user" | "group"; owner_id: string; path: string; recursive?: boolean } -): Promise { +settings: ApiSettings, +payload: {owner_type: "user" | "group";owner_id: string;path: string;recursive?: boolean;}) +: Promise { return apiFetch(settings, "/api/v1/files/folders/delete", { method: "POST", body: JSON.stringify({ recursive: true, ...payload }) }); } -export function listFiles(settings: ApiSettings, params: { owner_type?: string; owner_id?: string; campaign_id?: string; path_prefix?: string } = {}): Promise { +export function listFiles(settings: ApiSettings, params: {owner_type?: string;owner_id?: string;campaign_id?: string;path_prefix?: string;page_size?: number;cursor?: string | null;} = {}): Promise { const search = new URLSearchParams(); for (const [key, value] of Object.entries(params)) { - if (value) search.set(key, value); + if (value) search.set(key, String(value)); } const suffix = search.toString() ? `?${search.toString()}` : ""; return apiFetch(settings, `/api/v1/files${suffix}`); } -export async function uploadFiles( - settings: ApiSettings, - files: File[], - options: { - owner_type: "user" | "group"; - owner_id: string; - path?: string; - campaign_id?: string; - unpack_zip?: boolean; - conflict_strategy?: ConflictStrategy; - conflict_resolutions?: ConflictResolution[]; - onProgress?: (progress: FileUploadProgress) => void; +export function listFilesDelta( +settings: ApiSettings, +params: {owner_type?: string;owner_id?: string;campaign_id?: string;path_prefix?: string;since?: string;limit?: number;} = {}) +: Promise { + const search = new URLSearchParams(); + for (const [key, value] of Object.entries(params)) { + if (value !== undefined && value !== null && value !== "") search.set(key, String(value)); } -): Promise { + const suffix = search.toString() ? `?${search.toString()}` : ""; + return apiFetch(settings, `/api/v1/files/delta${suffix}`); +} + +function applyDeltaToSnapshot( +files: ManagedFile[], +folders: FileFolder[], +response: FileDeltaResponse) +: {files: ManagedFile[];folders: FileFolder[];} { + if (response.full) return { files: response.files, folders: response.folders }; + const deletedFileIds = new Set(response.deleted.filter((item) => !item.resource_type || item.resource_type === "file").map((item) => item.id)); + const deletedFolderIds = new Set(response.deleted.filter((item) => item.resource_type === "folder").map((item) => item.id)); + const filesById = new Map(files.map((file) => [file.id, file])); + const foldersById = new Map(folders.map((folder) => [folder.id, folder])); + deletedFileIds.forEach((id) => filesById.delete(id)); + deletedFolderIds.forEach((id) => foldersById.delete(id)); + response.files.forEach((file) => filesById.set(file.id, file)); + response.folders.forEach((folder) => foldersById.set(folder.id, folder)); + return { files: Array.from(filesById.values()), folders: Array.from(foldersById.values()) }; +} + +export async function listManagedFileSnapshot( +settings: ApiSettings, +params: {owner_type: "user" | "group";owner_id: string;path_prefix?: string;page_size?: number;}) +: Promise { + const pageSize = params.page_size ?? DEFAULT_MANAGED_FILE_WINDOW_SIZE; + let watermark: string | null | undefined = null; + let folderCursor: string | null | undefined = null; + let folders: FileFolder[] = []; + do { + const response = await listFolders(settings, { + owner_type: params.owner_type, + owner_id: params.owner_id, + page_size: pageSize, + cursor: folderCursor + }); + watermark = watermark || response.watermark; + folders = folders.concat(response.folders); + folderCursor = response.next_cursor; + } while (folderCursor); + + let fileCursor: string | null | undefined = null; + let files: ManagedFile[] = []; + do { + const response = await listFiles(settings, { + owner_type: params.owner_type, + owner_id: params.owner_id, + path_prefix: params.path_prefix, + page_size: pageSize, + cursor: fileCursor + }); + watermark = watermark || response.watermark; + files = files.concat(response.files); + fileCursor = response.next_cursor; + } while (fileCursor); + + if (watermark) { + let since = watermark; + let response: FileDeltaResponse | null = null; + do { + response = await listFilesDelta(settings, { + owner_type: params.owner_type, + owner_id: params.owner_id, + path_prefix: params.path_prefix, + since, + limit: pageSize + }); + const snapshot = applyDeltaToSnapshot(files, folders, response); + files = snapshot.files; + folders = snapshot.folders; + since = response.watermark || ""; + } while (response.has_more && response.watermark); + watermark = since || watermark; + } + + return { files, folders, watermark }; +} + +export async function uploadFiles( +settings: ApiSettings, +files: File[], +options: { + owner_type: "user" | "group"; + owner_id: string; + path?: string; + campaign_id?: string; + unpack_zip?: boolean; + conflict_strategy?: ConflictStrategy; + conflict_resolutions?: ConflictResolution[]; + source_provenance?: FileSourceProvenance; + source_revision?: string; + connector_policy_sources?: FileConnectorPolicySource[]; + onProgress?: (progress: FileUploadProgress) => void; +}) +: Promise { const form = new FormData(); files.forEach((file) => form.append("files", file)); form.append("owner_type", options.owner_type); @@ -123,15 +498,18 @@ export async function uploadFiles( if (options.unpack_zip) form.append("unpack_zip", "true"); if (options.conflict_strategy) form.append("conflict_strategy", options.conflict_strategy); if (options.conflict_resolutions?.length) form.append("conflict_resolutions_json", JSON.stringify(options.conflict_resolutions)); + if (options.source_provenance) form.append("source_provenance_json", JSON.stringify(options.source_provenance)); + if (options.source_revision) form.append("source_revision", options.source_revision); + if (options.connector_policy_sources?.length) form.append("connector_policy_json", JSON.stringify({ sources: options.connector_policy_sources })); if (options.onProgress) return uploadFilesWithProgress(settings, form, options.onProgress); return apiFetch(settings, "/api/v1/files/upload", { method: "POST", body: form }); } function uploadFilesWithProgress( - settings: ApiSettings, - form: FormData, - onProgress: (progress: FileUploadProgress) => void -): Promise { +settings: ApiSettings, +form: FormData, +onProgress: (progress: FileUploadProgress) => void) +: Promise { return new Promise((resolve, reject) => { const xhr = new XMLHttpRequest(); xhr.open("POST", apiUrl(settings, "/api/v1/files/upload")); @@ -145,12 +523,12 @@ function uploadFilesWithProgress( onProgress({ loaded: event.loaded, total, - percentage: total && total > 0 ? Math.round((event.loaded / total) * 100) : null + percentage: total && total > 0 ? Math.round(event.loaded / total * 100) : null }); }; - xhr.onerror = () => reject(new Error("Upload failed because the network request could not be completed.")); - xhr.onabort = () => reject(new Error("Upload was cancelled.")); + xhr.onerror = () => reject(new Error("i18n:govoplan-files.upload_failed_because_the_network_request_could_.360a5ab3")); + xhr.onabort = () => reject(new Error("i18n:govoplan-files.upload_was_cancelled.5bab0f9b")); xhr.onload = () => { const responseText = xhr.responseText || ""; if (xhr.status < 200 || xhr.status >= 300) { @@ -179,7 +557,7 @@ export function bulkDeleteFiles(settings: ApiSettings, fileIds: string[]): Promi return apiFetch(settings, "/api/v1/files/bulk-delete", { method: "POST", body: JSON.stringify({ file_ids: fileIds }) }); } -export type FileBulkShareResponse = { shares: FileShare[]; shared_count: number }; +export type FileBulkShareResponse = {shares: FileShare[];shared_count: number;}; export function shareFilesWithTarget(settings: ApiSettings, fileIds: string[], target: FilesManagedFileLinkTarget): Promise { return apiFetch(settings, "/api/v1/files/bulk-shares", { @@ -188,6 +566,231 @@ export function shareFilesWithTarget(settings: ApiSettings, fileIds: string[], t }); } +export function evaluateFileConnectorPolicy( +settings: ApiSettings, +payload: {source_provenance: FileSourceProvenance;operation?: string;policy_sources?: FileConnectorPolicySource[];}) +: Promise<{decision: FileConnectorPolicyDecision;}> { + return apiFetch<{decision: FileConnectorPolicyDecision;}>(settings, "/api/v1/files/connector-policy/evaluate", { + method: "POST", + body: JSON.stringify({ operation: "access", policy_sources: [], ...payload }) + }); +} + +export function getFileConnectorPolicy( +settings: ApiSettings, +scopeType: "system" | "tenant" | "user" | "group" | "campaign", +scopeId?: string | null) +: Promise { + const search = new URLSearchParams(); + if (scopeId) search.set("scope_id", scopeId); + const suffix = search.toString() ? `?${search.toString()}` : ""; + return apiFetch(settings, `/api/v1/files/connectors/policies/${encodeURIComponent(scopeType)}${suffix}`); +} + +export function fetchFileConnectorSettingsDelta( +settings: ApiSettings, +params: { + scope_type?: "system" | "tenant" | "user" | "group" | "campaign"; + scope_id?: string | null; + provider?: string; + campaign_id?: string | null; + include_disabled?: boolean; + include_inactive?: boolean; + owner_type?: "user" | "group"; + owner_id?: string; + since?: string | null; + limit?: number; +} = {}) +: Promise { + const search = new URLSearchParams(); + if (params.scope_type) search.set("scope_type", params.scope_type); + if (params.scope_id) search.set("scope_id", params.scope_id); + if (params.provider) search.set("provider", params.provider); + if (params.campaign_id) search.set("campaign_id", params.campaign_id); + if (params.include_disabled) search.set("include_disabled", "true"); + if (params.include_inactive) search.set("include_inactive", "true"); + if (params.owner_type) search.set("owner_type", params.owner_type); + if (params.owner_id) search.set("owner_id", params.owner_id); + if (params.since) search.set("since", params.since); + if (params.limit) search.set("limit", String(params.limit)); + const suffix = search.toString() ? `?${search.toString()}` : ""; + return apiFetch(settings, `/api/v1/files/connectors/settings/delta${suffix}`); +} + +export function updateFileConnectorPolicy( +settings: ApiSettings, +scopeType: "system" | "tenant" | "user" | "group" | "campaign", +policy: Record, +scopeId?: string | null) +: Promise { + const search = new URLSearchParams(); + if (scopeId) search.set("scope_id", scopeId); + const suffix = search.toString() ? `?${search.toString()}` : ""; + return apiFetch(settings, `/api/v1/files/connectors/policies/${encodeURIComponent(scopeType)}${suffix}`, { + method: "PUT", + body: JSON.stringify({ policy }) + }); +} + +export function listFileConnectorProfiles( +settings: ApiSettings, +params: {provider?: string;campaign_id?: string;include_disabled?: boolean;} = {}) +: Promise<{profiles: FileConnectorProfile[];}> { + const search = new URLSearchParams(); + if (params.provider) search.set("provider", params.provider); + if (params.campaign_id) search.set("campaign_id", params.campaign_id); + if (params.include_disabled) search.set("include_disabled", "true"); + const suffix = search.toString() ? `?${search.toString()}` : ""; + return apiFetch<{profiles: FileConnectorProfile[];}>(settings, `/api/v1/files/connectors/profiles${suffix}`); +} + +export function listFileConnectorProviders(settings: ApiSettings): Promise<{providers: FileConnectorProvider[];}> { + return apiFetch<{providers: FileConnectorProvider[];}>(settings, "/api/v1/files/connectors/providers"); +} + +export function discoverFileConnectorEndpoint(settings: ApiSettings, payload: FileConnectorDiscoveryPayload): Promise { + return apiFetch(settings, "/api/v1/files/connectors/discover", { + method: "POST", + body: JSON.stringify(payload) + }); +} + +export function listFileConnectorCredentials( +settings: ApiSettings, +params: {provider?: string;include_disabled?: boolean;} = {}) +: Promise<{credentials: FileConnectorCredential[];}> { + const search = new URLSearchParams(); + if (params.provider) search.set("provider", params.provider); + if (params.include_disabled) search.set("include_disabled", "true"); + const suffix = search.toString() ? `?${search.toString()}` : ""; + return apiFetch<{credentials: FileConnectorCredential[];}>(settings, `/api/v1/files/connectors/credentials${suffix}`); +} + +export function createFileConnectorCredential(settings: ApiSettings, payload: FileConnectorCredentialPayload): Promise { + return apiFetch(settings, "/api/v1/files/connectors/credentials", { + method: "POST", + body: JSON.stringify(payload) + }); +} + +export function updateFileConnectorCredential( +settings: ApiSettings, +credentialId: string, +payload: FileConnectorCredentialUpdatePayload) +: Promise { + return apiFetch(settings, `/api/v1/files/connectors/credentials/${encodeURIComponent(credentialId)}`, { + method: "PATCH", + body: JSON.stringify(payload) + }); +} + +export function deactivateFileConnectorCredential(settings: ApiSettings, credentialId: string): Promise { + return apiFetch(settings, `/api/v1/files/connectors/credentials/${encodeURIComponent(credentialId)}`, { method: "DELETE" }); +} + +export function listFileConnectorSpaces( +settings: ApiSettings, +params: {owner_type?: "user" | "group";owner_id?: string;include_inactive?: boolean;} = {}) +: Promise<{spaces: FileConnectorSpace[];}> { + const search = new URLSearchParams(); + if (params.owner_type) search.set("owner_type", params.owner_type); + if (params.owner_id) search.set("owner_id", params.owner_id); + if (params.include_inactive) search.set("include_inactive", "true"); + const suffix = search.toString() ? `?${search.toString()}` : ""; + return apiFetch<{spaces: FileConnectorSpace[];}>(settings, `/api/v1/files/connector-spaces${suffix}`); +} + +export function createFileConnectorSpace(settings: ApiSettings, payload: FileConnectorSpacePayload): Promise { + return apiFetch(settings, "/api/v1/files/connector-spaces", { + method: "POST", + body: JSON.stringify({ owner_type: "user", remote_path: "", sync_mode: "manual", metadata: {}, ...payload }) + }); +} + +export function updateFileConnectorSpace( +settings: ApiSettings, +spaceId: string, +payload: Partial> & {is_active?: boolean;}) +: Promise { + return apiFetch(settings, `/api/v1/files/connector-spaces/${encodeURIComponent(spaceId)}`, { + method: "PATCH", + body: JSON.stringify(payload) + }); +} + +export function deleteFileConnectorSpace(settings: ApiSettings, spaceId: string): Promise { + return apiFetch(settings, `/api/v1/files/connector-spaces/${encodeURIComponent(spaceId)}`, { method: "DELETE" }); +} + +export function getFileConnectorProfile( +settings: ApiSettings, +profileId: string, +params: {campaign_id?: string;include_disabled?: boolean;} = {}) +: Promise { + const search = new URLSearchParams(); + if (params.campaign_id) search.set("campaign_id", params.campaign_id); + if (params.include_disabled) search.set("include_disabled", "true"); + const suffix = search.toString() ? `?${search.toString()}` : ""; + return apiFetch(settings, `/api/v1/files/connectors/profiles/${encodeURIComponent(profileId)}${suffix}`); +} + +export function createFileConnectorProfile(settings: ApiSettings, payload: FileConnectorProfilePayload): Promise { + return apiFetch(settings, "/api/v1/files/connectors/profiles", { + method: "POST", + body: JSON.stringify(payload) + }); +} + +export function updateFileConnectorProfile( +settings: ApiSettings, +profileId: string, +payload: FileConnectorProfileUpdatePayload) +: Promise { + return apiFetch(settings, `/api/v1/files/connectors/profiles/${encodeURIComponent(profileId)}`, { + method: "PATCH", + body: JSON.stringify(payload) + }); +} + +export function deactivateFileConnectorProfile(settings: ApiSettings, profileId: string): Promise { + return apiFetch(settings, `/api/v1/files/connectors/profiles/${encodeURIComponent(profileId)}`, { method: "DELETE" }); +} + +export function browseFileConnectorProfile( +settings: ApiSettings, +profileId: string, +params: {path?: string;library_id?: string;campaign_id?: string;} = {}) +: Promise { + const search = new URLSearchParams(); + if (params.path) search.set("path", params.path); + if (params.library_id) search.set("library_id", params.library_id); + if (params.campaign_id) search.set("campaign_id", params.campaign_id); + const suffix = search.toString() ? `?${search.toString()}` : ""; + return apiFetch(settings, `/api/v1/files/connectors/profiles/${encodeURIComponent(profileId)}/browse${suffix}`); +} + +export function importFileConnectorFile( +settings: ApiSettings, +profileId: string, +payload: FileConnectorFilePayload) +: Promise { + return apiFetch(settings, `/api/v1/files/connectors/profiles/${encodeURIComponent(profileId)}/import`, { + method: "POST", + body: JSON.stringify({ conflict_strategy: "reject", metadata: {}, ...payload }) + }); +} + +export function syncFileConnectorFile( +settings: ApiSettings, +profileId: string, +payload: FileConnectorFilePayload) +: Promise { + return apiFetch(settings, `/api/v1/files/connectors/profiles/${encodeURIComponent(profileId)}/sync`, { + method: "POST", + body: JSON.stringify({ conflict_strategy: "rename", metadata: {}, ...payload }) + }); +} + export function shareFilesWithCampaign(settings: ApiSettings, fileIds: string[], campaignId: string): Promise { return shareFilesWithTarget(settings, fileIds, { type: "campaign", id: campaignId, label: "campaign" }); } @@ -195,14 +798,14 @@ export function shareFilesWithCampaign(settings: ApiSettings, fileIds: string[], export async function shareFileWithCampaign(settings: ApiSettings, fileId: string, campaignId: string): Promise { const response = await shareFilesWithCampaign(settings, [fileId], campaignId); const share = response.shares[0]; - if (!share) throw new Error("File share was not created."); + if (!share) throw new Error("i18n:govoplan-files.file_share_was_not_created.033ac476"); return share; } export function bulkRenameFiles( - settings: ApiSettings, - payload: { file_ids: string[]; folder_paths?: string[]; owner_type?: "user" | "group"; owner_id?: string; mode: "direct" | "prefix" | "suffix" | "replace"; new_name?: string; find?: string; replacement?: string; prefix?: string; suffix?: string; recursive?: boolean; dry_run?: boolean } -): Promise { +settings: ApiSettings, +payload: {file_ids: string[];folder_paths?: string[];owner_type: "user" | "group";owner_id: string;mode: "direct" | "prefix" | "suffix" | "replace";new_name?: string;find?: string;replacement?: string;prefix?: string;suffix?: string;recursive?: boolean;dry_run?: boolean;}) +: Promise { return apiFetch(settings, "/api/v1/files/bulk-rename", { method: "POST", body: JSON.stringify({ replacement: "", prefix: "", suffix: "", dry_run: true, ...payload }) @@ -210,27 +813,27 @@ export function bulkRenameFiles( } export function resolveFilePatterns( - settings: ApiSettings, - payload: { patterns: string[]; owner_type?: "user" | "group"; owner_id?: string; campaign_id?: string; path_prefix?: string; include_unmatched?: boolean; case_sensitive?: boolean } -): Promise { +settings: ApiSettings, +payload: {patterns: string[];owner_type?: "user" | "group";owner_id?: string;campaign_id?: string;path_prefix?: string;include_unmatched?: boolean;case_sensitive?: boolean;}) +: Promise { return apiFetch(settings, "/api/v1/files/resolve-patterns", { method: "POST", body: JSON.stringify(payload) }); } export function transferFiles( - settings: ApiSettings, - payload: { - operation: "move" | "copy"; - file_ids: string[]; - folder_paths: string[]; - source_owner_type: "user" | "group"; - source_owner_id: string; - target_owner_type: "user" | "group"; - target_owner_id: string; - target_folder: string; - conflict_strategy?: ConflictStrategy; - conflict_resolutions?: ConflictResolution[]; - } -): Promise { +settings: ApiSettings, +payload: { + operation: "move" | "copy"; + file_ids: string[]; + folder_paths: string[]; + source_owner_type: "user" | "group"; + source_owner_id: string; + target_owner_type: "user" | "group"; + target_owner_id: string; + target_folder: string; + conflict_strategy?: ConflictStrategy; + conflict_resolutions?: ConflictResolution[]; +}) +: Promise { return apiFetch(settings, "/api/v1/files/transfer", { method: "POST", body: JSON.stringify(payload) }); } diff --git a/webui/src/features/files/FileConnectorSettingsPanel.tsx b/webui/src/features/files/FileConnectorSettingsPanel.tsx new file mode 100644 index 0000000..fb55d79 --- /dev/null +++ b/webui/src/features/files/FileConnectorSettingsPanel.tsx @@ -0,0 +1,1717 @@ +import { useEffect, useMemo, useState } from "react"; +import { Edit3, Plus, RefreshCw, Save, ShieldCheck, Trash2, UserRoundKey } from "lucide-react"; +import { + Button, + Card, + ConnectionTree, + ConfirmDialog, + DismissibleAlert, + Dialog, + FormField, + ActionBlockerHint, + AdvancedOptionsPanel, + LoadingFrame, + mergeDeltaRows, + SegmentedControl, + ToggleSwitch, + useDeltaWatermarks, + useUnsavedDraftGuard, + type ApiSettings, + type ConnectionTreeColumn, + type FileConnectorScope, + type FileConnectorTargetOption, + i18nMessage } from +"@govoplan/core-webui"; +import { + browseFileConnectorProfile, + createFileConnectorCredential, + createFileConnectorProfile, + deactivateFileConnectorCredential, + deactivateFileConnectorProfile, + discoverFileConnectorEndpoint, + fetchFileConnectorSettingsDelta, + updateFileConnectorPolicy, + updateFileConnectorCredential, + updateFileConnectorProfile, + type FileConnectorCredential, + type FileConnectorCredentialPayload, + type FileConnectorCredentialUpdatePayload, + type FileConnectorPolicyResponse, + type FileConnectorProfile, + type FileConnectorProfilePayload, + type FileConnectorProfileUpdatePayload } from +"../../api/files"; + +type ConnectorScope = FileConnectorScope; +type Provider = FileConnectorProfilePayload["provider"]; +type CredentialMode = NonNullable; +type PolicyMode = "allow-provider" | "allowed-paths" | "deny-provider"; + +type ConnectorTreeRow = +{kind: "profile";id: string;profile: FileConnectorProfile;} | +{kind: "credential";id: string;credential: FileConnectorCredential;profile: FileConnectorProfile;}; + +type ConnectorProfileDraft = { + id: string; + label: string; + provider: Provider; + endpointUrl: string; + basePath: string; + enabled: boolean; + credentialProfileId: string; + credentialMode: CredentialMode; + username: string; + password: string; + token: string; + passwordEnv: string; + tokenEnv: string; + secretRef: string; + canBrowse: boolean; + canImport: boolean; + canSync: boolean; + policyMode: PolicyMode; + allowedPaths: string; + deniedPaths: string; + requireSigning: boolean; + authProtocol: string; + browseProtocol: string; + metadataJson: string; + clearPassword: boolean; + clearToken: boolean; +}; + +type ConnectorCredentialDraft = { + id: string; + label: string; + provider: Provider | ""; + enabled: boolean; + credentialMode: CredentialMode; + username: string; + password: string; + token: string; + passwordEnv: string; + tokenEnv: string; + secretRef: string; + policyMode: PolicyMode; + allowedPaths: string; + deniedPaths: string; + metadataJson: string; + clearPassword: boolean; + clearToken: boolean; +}; + +type CentralConnectorPolicyDraft = { + allowedConnectors: string; + allowedCredentials: string; + allowedProviders: string; + allowedPaths: string; + allowedUrls: string; + deniedConnectors: string; + deniedCredentials: string; + deniedProviders: string; + deniedPaths: string; + deniedUrls: string; + allowLowerConnectionLimits: boolean; + allowLowerCredentialLimits: boolean; + allowLowerProviderLimits: boolean; + allowLowerPathLimits: boolean; + allowLowerUrlLimits: boolean; +}; + +const PROVIDERS: Array<{id: Provider;label: string;}> = [ +{ id: "webdav", label: "i18n:govoplan-files.webdav.521b4c8b" }, +{ id: "nextcloud", label: "i18n:govoplan-files.nextcloud.aab73a3d" }, +{ id: "seafile", label: "i18n:govoplan-files.seafile.600192cc" }, +{ id: "smb", label: "i18n:govoplan-files.smb.515d2f88" }, +{ id: "nfs", label: "i18n:govoplan-files.nfs.db121865" }, +{ id: "dms", label: "i18n:govoplan-files.dms.477e5652" }, +{ id: "generic", label: "i18n:govoplan-files.generic.ff7613e5" }]; + + +const ENDPOINT_PLACEHOLDERS: Record = { + webdav: "http://127.0.0.1:9083/", + nextcloud: "http://127.0.0.1:9081/remote.php/dav/files/admin/", + seafile: "http://127.0.0.1:9082/", + smb: "smb://127.0.0.1:1445/files", + nfs: "nfs://127.0.0.1/export", + dms: "https://dms.example.test", + generic: "https://files.example.test" +}; + +export default function FileConnectorSettingsPanel({ + settings, + scopeType, + scopeId = null, + targetOptions = [], + targetLabel = "i18n:govoplan-files.target.61ad50a9", + title, + canWrite + + + + + + + + +}: {settings: ApiSettings;scopeType: ConnectorScope;scopeId?: string | null;targetOptions?: FileConnectorTargetOption[];targetLabel?: string;title?: string;canWrite: boolean;}) { + const [profiles, setProfiles] = useState([]); + const [credentials, setCredentials] = useState([]); + const [selectedTargetId, setSelectedTargetId] = useState(scopeId || targetOptions[0]?.id || ""); + const [policyResponse, setPolicyResponse] = useState(null); + const [loading, setLoading] = useState(false); + const [saving, setSaving] = useState(false); + const [draft, setDraft] = useState(null); + const [savedDraftKey, setSavedDraftKey] = useState(""); + const [credentialDraft, setCredentialDraft] = useState(null); + const [savedCredentialDraftKey, setSavedCredentialDraftKey] = useState(""); + const [credentialAttachProfileId, setCredentialAttachProfileId] = useState(null); + const [policyDraft, setPolicyDraft] = useState(emptyCentralPolicyDraft()); + const [savedPolicyDraftKey, setSavedPolicyDraftKey] = useState(centralPolicyDraftKey(emptyCentralPolicyDraft())); + const [editingProfileId, setEditingProfileId] = useState(null); + const [editingCredentialId, setEditingCredentialId] = useState(null); + const [pendingDeactivate, setPendingDeactivate] = useState(null); + const [pendingCredentialDeactivate, setPendingCredentialDeactivate] = useState(null); + const [testingProfileId, setTestingProfileId] = useState(null); + const [testResults, setTestResults] = useState>({}); + const [discoveringProfile, setDiscoveringProfile] = useState(false); + const [profileDiscoveryResult, setProfileDiscoveryResult] = useState<{ok: boolean;message: string;} | null>(null); + const [testingCredentialLogin, setTestingCredentialLogin] = useState(false); + const [credentialLoginResult, setCredentialLoginResult] = useState<{ok: boolean;message: string;} | null>(null); + const [message, setMessage] = useState(""); + const [error, setError] = useState(""); + const { getDeltaWatermark, setDeltaWatermark, resetDeltaWatermark } = useDeltaWatermarks(); + + const requiresTarget = scopeType === "user" || scopeType === "group"; + const targetSelectionRequired = requiresTarget && !scopeId; + const hasSelectableTarget = targetOptions.length > 0; + const activeScopeId = scopeId || selectedTargetId || null; + const deltaKey = `files:connectors:${scopeType}:${activeScopeId ?? ""}`; + const scopeReady = !requiresTarget || Boolean(activeScopeId); + const targetEmptyText = i18nMessage("i18n:govoplan-files.no_value_available", { value0: targetPluralLabel(scopeType, targetLabel) }); + const profileDirty = Boolean(draft) && connectorProfileDraftKey(draft) !== savedDraftKey; + const credentialDirty = Boolean(credentialDraft) && connectorCredentialDraftKey(credentialDraft) !== savedCredentialDraftKey; + const policyDirty = scopeReady && centralPolicyDraftKey(policyDraft) !== savedPolicyDraftKey; + + useUnsavedDraftGuard({ + dirty: profileDirty || credentialDirty || policyDirty, + onSave: saveDirtyDrafts, + onDiscard: discardDirtyDrafts + }); + + useEffect(() => { + if (scopeId) { + setSelectedTargetId(scopeId); + return; + } + if (targetOptions.length > 0 && !targetOptions.some((option) => option.id === selectedTargetId)) { + setSelectedTargetId(targetOptions[0].id); + return; + } + if (targetOptions.length === 0 && selectedTargetId) setSelectedTargetId(""); + }, [scopeId, selectedTargetId, targetOptions]); + + const scopedProfiles = useMemo( + () => profiles.filter((profile) => connectorBelongsToEditableScope(profile, scopeType, activeScopeId)), + [activeScopeId, profiles, scopeType] + ); + const scopedCredentials = useMemo( + () => credentials.filter((credential) => connectorBelongsToEditableScope(credential, scopeType, activeScopeId)), + [activeScopeId, credentials, scopeType] + ); + + useEffect(() => { + resetDeltaWatermark(deltaKey); + void loadProfiles(true); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [settings.apiBaseUrl, settings.apiKey, settings.accessToken, scopeType, activeScopeId, scopeReady, deltaKey, resetDeltaWatermark]); + + async function loadProfiles(forceFull = false) { + setLoading(true); + setError(""); + if (!scopeReady) { + setProfiles([]); + setCredentials([]); + setPolicyResponse(null); + setPolicyDraft(emptyCentralPolicyDraft()); + setSavedPolicyDraftKey(centralPolicyDraftKey(emptyCentralPolicyDraft())); + resetDeltaWatermark(deltaKey); + setLoading(false); + return; + } + try { + let nextProfiles = forceFull ? [] : profiles; + let nextCredentials = forceFull ? [] : credentials; + let nextPolicyResponse = forceFull ? null : policyResponse; + let nextWatermark = forceFull ? null : getDeltaWatermark(deltaKey); + let hasMore = false; + do { + const response = await fetchFileConnectorSettingsDelta(settings, { + scope_type: scopeType, + scope_id: activeScopeId, + include_disabled: true, + since: nextWatermark + }); + nextProfiles = response.full ? + response.profiles : + mergeDeltaRows(nextProfiles, response.profiles, response.deleted, (profile) => profile.id, { + deletedResourceType: "file_connector_profile", + sort: sortConnectorProfiles + }); + nextCredentials = response.full ? + response.credentials : + mergeDeltaRows(nextCredentials, response.credentials, response.deleted, (credential) => credential.id, { + deletedResourceType: "file_connector_credential", + sort: sortConnectorCredentials + }); + if (response.policy) nextPolicyResponse = response.policy; + nextWatermark = response.watermark ?? null; + hasMore = response.has_more; + } while (hasMore); + setProfiles(nextProfiles); + setCredentials(nextCredentials); + setPolicyResponse(nextPolicyResponse); + if (nextPolicyResponse) { + const nextPolicyDraft = centralPolicyDraftFromPolicy(nextPolicyResponse.policy); + setPolicyDraft(nextPolicyDraft); + setSavedPolicyDraftKey(centralPolicyDraftKey(nextPolicyDraft)); + } + setDeltaWatermark(deltaKey, nextWatermark); + } catch (err) { + resetDeltaWatermark(deltaKey); + setError(errorMessage(err)); + } finally { + setLoading(false); + } + } + + function startCreate() { + setEditingProfileId(null); + const nextDraft = emptyDraft(scopeType); + setDraft(nextDraft); + setSavedDraftKey(connectorProfileDraftKey(nextDraft)); + setProfileDiscoveryResult(null); + setMessage(""); + setError(""); + } + + function startCredentialCreate(profile: FileConnectorProfile) { + setEditingCredentialId(null); + setCredentialAttachProfileId(profile.id); + const next = emptyCredentialDraft(scopeType); + next.id = `${profile.id}-credentials`; + next.label = `${profile.label} credentials`; + next.provider = knownProvider(profile.provider); + next.credentialMode = knownCredentialMode(profile.credential_mode); + setCredentialDraft(next); + setSavedCredentialDraftKey(connectorCredentialDraftKey(next)); + setCredentialLoginResult(null); + setMessage(""); + setError(""); + } + + function startEdit(profile: FileConnectorProfile) { + setEditingProfileId(profile.id); + const nextDraft = draftFromProfile(profile, scopeType); + setDraft(nextDraft); + setSavedDraftKey(connectorProfileDraftKey(nextDraft)); + setProfileDiscoveryResult(null); + setMessage(""); + setError(""); + } + + function startCredentialEdit(credential: FileConnectorCredential) { + setEditingCredentialId(credential.id); + setCredentialAttachProfileId(null); + const nextDraft = credentialDraftFromCredential(credential, scopeType); + setCredentialDraft(nextDraft); + setSavedCredentialDraftKey(connectorCredentialDraftKey(nextDraft)); + setCredentialLoginResult(null); + setMessage(""); + setError(""); + } + + function patchDraft(patch: Partial) { + setDraft((current) => current ? { ...current, ...patch } : current); + } + + function patchCredentialDraft(patch: Partial) { + setCredentialLoginResult(null); + setCredentialDraft((current) => current ? { ...current, ...patch } : current); + } + + function patchPolicyDraft(patch: Partial) { + setPolicyDraft((current) => ({ ...current, ...patch })); + } + + function closeProfileDialog() { + if (saving) return; + setDraft(null); + setEditingProfileId(null); + setProfileDiscoveryResult(null); + } + + function closeCredentialDialog() { + if (saving) return; + setCredentialDraft(null); + setEditingCredentialId(null); + setCredentialAttachProfileId(null); + setCredentialLoginResult(null); + } + + async function savePolicyDraft(): Promise { + if (!canWrite || !scopeReady) return false; + setSaving(true); + setError(""); + try { + const response = await updateFileConnectorPolicy(settings, scopeType, centralPolicyFromDraft(policyDraft), activeScopeId); + setPolicyResponse(response); + const savedPolicyDraft = centralPolicyDraftFromPolicy(response.policy); + setPolicyDraft(savedPolicyDraft); + setSavedPolicyDraftKey(centralPolicyDraftKey(savedPolicyDraft)); + setMessage(i18nMessage("i18n:govoplan-files.value_connector_policy_saved.430d4eca", { value0: scopeLabel(scopeType) })); + return true; + } catch (err) { + setError(errorMessage(err)); + return false; + } finally { + setSaving(false); + } + } + + async function saveDraft(): Promise { + if (!draft || !canWrite || !scopeReady) return false; + const label = draft.label.trim(); + const id = draft.id.trim(); + if (!label || !editingProfileId && !id) { + setError("i18n:govoplan-files.profile_id_and_label_are_required.6ad85df1"); + return false; + } + + let metadata: Record; + try { + metadata = metadataFromDraft(draft); + } catch (err) { + setError(errorMessage(err)); + return false; + } + + const capabilities = [ + draft.canBrowse ? "browse" : "", + draft.canImport ? "import" : "", + draft.canSync ? "sync" : ""]. + filter(Boolean); + const sharedPayload = { + label, + provider: draft.provider, + endpoint_url: cleanOrNull(draft.endpointUrl), + base_path: cleanOrNull(draft.basePath), + enabled: draft.enabled, + credential_profile_id: cleanOrNull(credentialProfileIdForDraft(draft, scopedCredentials)), + credential_mode: draft.credentialMode, + capabilities, + policy: policyFromDraft(draft), + metadata + }; + + setSaving(true); + setError(""); + try { + if (editingProfileId) { + const payload: FileConnectorProfileUpdatePayload = { + ...sharedPayload, + clear_password: draft.clearPassword, + clear_token: draft.clearToken + }; + await updateFileConnectorProfile(settings, editingProfileId, payload); + } else { + const payload: FileConnectorProfilePayload = { + id, + scope_type: scopeType, + scope_id: activeScopeId, + ...sharedPayload + }; + await createFileConnectorProfile(settings, payload); + } + setMessage(editingProfileId ? "i18n:govoplan-files.file_connection_saved.68c16ee9" : "i18n:govoplan-files.file_connection_created.bdf6e1f6"); + setDraft(null); + setSavedDraftKey(""); + setEditingProfileId(null); + await loadProfiles(); + return true; + } catch (err) { + setError(errorMessage(err)); + return false; + } finally { + setSaving(false); + } + } + + async function saveCredentialDraft(): Promise { + if (!credentialDraft || !canWrite || !scopeReady) return false; + if (!editingCredentialId && !credentialAttachProfileId) { + setError("Create credentials from a connection row so the credential is attached to a file connection."); + return false; + } + const label = credentialDraft.label.trim(); + const id = credentialDraft.id.trim(); + if (!label || !editingCredentialId && !id) { + setError("i18n:govoplan-files.credential_id_and_label_are_required.4cfd1ffd"); + return false; + } + + let metadata: Record; + try { + metadata = metadataFromJson(credentialDraft.metadataJson); + } catch (err) { + setError(errorMessage(err)); + return false; + } + + const credentials = { + username: cleanOrNull(credentialDraft.username), + password: cleanOrUndefined(credentialDraft.password), + token: cleanOrUndefined(credentialDraft.token), + password_env: cleanOrNull(credentialDraft.passwordEnv), + token_env: cleanOrNull(credentialDraft.tokenEnv), + secret_ref: cleanOrNull(credentialDraft.secretRef) + }; + const sharedPayload = { + label, + provider: credentialDraft.provider || null, + enabled: credentialDraft.enabled, + credential_mode: credentialDraft.credentialMode, + credentials, + policy: credentialPolicyFromDraft(credentialDraft), + metadata + }; + + setSaving(true); + setError(""); + try { + if (editingCredentialId) { + const payload: FileConnectorCredentialUpdatePayload = { + ...sharedPayload, + clear_password: credentialDraft.clearPassword, + clear_token: credentialDraft.clearToken + }; + await updateFileConnectorCredential(settings, editingCredentialId, payload); + } else { + const payload: FileConnectorCredentialPayload = { + id, + scope_type: scopeType, + scope_id: activeScopeId, + ...sharedPayload + }; + await createFileConnectorCredential(settings, payload); + if (credentialAttachProfileId) { + await updateFileConnectorProfile(settings, credentialAttachProfileId, { + credential_profile_id: id, + credential_mode: credentialDraft.credentialMode + }); + } + } + setMessage(editingCredentialId ? "i18n:govoplan-files.file_credential_saved.d6a1eef8" : "i18n:govoplan-files.file_credential_created.87c31882"); + setCredentialDraft(null); + setSavedCredentialDraftKey(""); + setEditingCredentialId(null); + setCredentialAttachProfileId(null); + await loadProfiles(); + return true; + } catch (err) { + setError(errorMessage(err)); + return false; + } finally { + setSaving(false); + } + } + + async function saveDirtyDrafts(): Promise { + let ok = true; + if (policyDirty) ok = await savePolicyDraft() && ok; + if (credentialDirty) ok = await saveCredentialDraft() && ok; + if (profileDirty) ok = await saveDraft() && ok; + return ok; + } + + function discardDirtyDrafts() { + setDraft(null); + setSavedDraftKey(""); + setEditingProfileId(null); + setCredentialDraft(null); + setSavedCredentialDraftKey(""); + setEditingCredentialId(null); + setCredentialAttachProfileId(null); + setPolicyDraft(JSON.parse(savedPolicyDraftKey) as CentralConnectorPolicyDraft); + } + + async function confirmDeactivate() { + if (!pendingDeactivate) return; + const profile = pendingDeactivate; + setSaving(true); + setError(""); + try { + await deactivateFileConnectorProfile(settings, profile.id); + setMessage(i18nMessage("i18n:govoplan-files.file_connection_disabled_value.059a7de9", { value0: profile.label })); + setPendingDeactivate(null); + await loadProfiles(); + } catch (err) { + setError(errorMessage(err)); + } finally { + setSaving(false); + } + } + + async function confirmCredentialDeactivate() { + if (!pendingCredentialDeactivate) return; + const credential = pendingCredentialDeactivate; + setSaving(true); + setError(""); + try { + await deactivateFileConnectorCredential(settings, credential.id); + setMessage(i18nMessage("i18n:govoplan-files.file_credential_disabled_value.947538fd", { value0: credential.label })); + setPendingCredentialDeactivate(null); + await loadProfiles(); + } catch (err) { + setError(errorMessage(err)); + } finally { + setSaving(false); + } + } + + async function testBrowse(profile: FileConnectorProfile) { + setTestingProfileId(profile.id); + setTestResults((current) => ({ ...current, [profile.id]: { ok: true, message: "i18n:govoplan-files.testing.15ccc832" } })); + try { + const response = await browseFileConnectorProfile(settings, profile.id, { path: "" }); + setTestResults((current) => ({ + ...current, + [profile.id]: { ok: true, message: i18nMessage("i18n:govoplan-files.browse_ok_value_item_value.921906fd", { value0: response.items.length, value1: response.items.length === 1 ? "" : "s" }) } + })); + } catch (err) { + setTestResults((current) => ({ ...current, [profile.id]: { ok: false, message: errorMessage(err) } })); + } finally { + setTestingProfileId(null); + } + } + + async function discoverProfileEndpoint() { + if (!draft) return; + const endpointUrl = draft.endpointUrl.trim(); + if (!endpointUrl) { + setProfileDiscoveryResult({ ok: false, message: "Enter a server URL before running discovery." }); + return; + } + setDiscoveringProfile(true); + setProfileDiscoveryResult(null); + try { + const response = await discoverFileConnectorEndpoint(settings, { + provider: draft.provider, + endpoint_url: endpointUrl, + base_path: cleanOrNull(draft.basePath), + credential_mode: draft.credentialMode, + credentials: { + username: cleanOrNull(draft.username), + password: cleanOrUndefined(draft.password), + token: cleanOrUndefined(draft.token), + password_env: cleanOrNull(draft.passwordEnv), + token_env: cleanOrNull(draft.tokenEnv), + secret_ref: cleanOrNull(draft.secretRef) + }, + metadata: metadataFromDraft(draft) + }); + if (response.endpoint_url) { + patchDraft({ + endpointUrl: response.endpoint_url, + basePath: response.base_path ?? draft.basePath, + metadataJson: JSON.stringify(response.metadata ?? metadataFromDraft(draft), null, 2) + }); + } + setProfileDiscoveryResult({ + ok: response.status === "usable" || response.status === "found", + message: response.message + }); + } catch (err) { + setProfileDiscoveryResult({ ok: false, message: errorMessage(err) }); + } finally { + setDiscoveringProfile(false); + } + } + + async function testCredentialLogin() { + if (!credentialDraft) return; + const profile = credentialTestProfile; + if (!profile) { + setCredentialLoginResult({ ok: false, message: "Create or select a file connection before testing this credential." }); + return; + } + if (profile.provider !== "webdav" && profile.provider !== "nextcloud") { + setCredentialLoginResult({ ok: false, message: "Credential login testing is currently available for WebDAV and Nextcloud connections." }); + return; + } + setTestingCredentialLogin(true); + setCredentialLoginResult(null); + try { + const response = await discoverFileConnectorEndpoint(settings, { + provider: knownProvider(profile.provider), + endpoint_url: profile.endpoint_url || "", + base_path: profile.base_path || "", + credential_mode: credentialDraft.credentialMode, + credentials: { + username: cleanOrNull(credentialDraft.username), + password: cleanOrUndefined(credentialDraft.password), + token: cleanOrUndefined(credentialDraft.token), + password_env: cleanOrNull(credentialDraft.passwordEnv), + token_env: cleanOrNull(credentialDraft.tokenEnv), + secret_ref: cleanOrNull(credentialDraft.secretRef) + }, + metadata: profile.metadata ?? {}, + require_valid_credentials: true + }); + const ok = response.status === "usable" || response.status === "found"; + setCredentialLoginResult({ + ok, + message: ok ? "Login successful. The credentials can browse the selected connection." : response.message + }); + } catch (err) { + setCredentialLoginResult({ ok: false, message: errorMessage(err) }); + } finally { + setTestingCredentialLogin(false); + } + } + + const panelTitle = title ?? i18nMessage("i18n:govoplan-files.value_file_connections", { value0: scopeLabel(scopeType) }); + const selectedCredentialIds = new Set(scopedProfiles.map((profile) => profile.credential_profile_id).filter((value): value is string => Boolean(value))); + const firstCompatibleProfileByCredential = new Map(); + for (const credential of scopedCredentials) { + const selectedProfile = scopedProfiles.find((profile) => profile.credential_profile_id === credential.id); + const fallbackProfile = selectedProfile ?? scopedProfiles.find((profile) => !selectedCredentialIds.has(credential.id) && credentialMatchesProfile(credential, profile)); + if (fallbackProfile) firstCompatibleProfileByCredential.set(credential.id, fallbackProfile.id); + } + const connectorRows: ConnectorTreeRow[] = scopedProfiles.map((profile) => ({ kind: "profile", id: `profile:${profile.id}`, profile })); + const credentialAttachedProfile = credentialAttachProfileId ? scopedProfiles.find((profile) => profile.id === credentialAttachProfileId) ?? null : null; + const credentialTestProfile = credentialDraft ? + credentialAttachedProfile ?? + (editingCredentialId ? scopedProfiles.find((profile) => profile.credential_profile_id === editingCredentialId) ?? scopedProfiles.find((profile) => credentialDraft.provider ? profile.provider === credentialDraft.provider : true) ?? null : null) : + null; + const profileMissingFields = profileDraftMissingFields(draft, editingProfileId); + const credentialMissingFields = credentialDraftMissingFields(credentialDraft, editingCredentialId, credentialAttachProfileId); + const profileMissingRequired = profileMissingFields.length > 0; + const credentialMissingRequired = credentialMissingFields.length > 0; + const profileSaveDisabled = !draft || saving || profileMissingRequired; + const credentialSaveDisabled = !credentialDraft || saving || credentialMissingRequired; + const profileSaveTooltip = requiredFieldsTooltip(profileMissingFields); + const credentialSaveTooltip = requiredFieldsTooltip(credentialMissingFields); + const canDiscoverProfileEndpoint = Boolean(draft && (draft.provider === "webdav" || draft.provider === "nextcloud")); + const credentialTestUnsupported = Boolean(credentialTestProfile && credentialTestProfile.provider !== "webdav" && credentialTestProfile.provider !== "nextcloud"); + const credentialTestDisabled = !credentialDraft || saving || testingCredentialLogin || !credentialTestProfile || credentialTestUnsupported; + const credentialTestTooltip = credentialTestHelpText(credentialTestProfile, credentialTestUnsupported); + const profileCredentialOptions = draft ? credentialOptionsForProfileDraft(scopedCredentials, draft) : []; + const profileCredentialSelectValue = draft ? credentialProfileIdForDraft(draft, scopedCredentials) : ""; + const connectorColumns: ConnectionTreeColumn[] = [ + { + id: "connection", + header: "i18n:govoplan-files.connection_credential.178babe0", + width: "minmax(260px, 1.5fr)", + render: (row) => row.kind === "profile" ? + : + + }, + { + id: "endpoint", + header: "i18n:govoplan-files.endpoint.92ec6350", + width: "minmax(190px, 1fr)", + render: (row) => row.kind === "profile" ? + {row.profile.endpoint_url || "i18n:govoplan-files.no_endpoint.d0ea68c4"} : + {row.credential.provider ? providerLabel(row.credential.provider) : "i18n:govoplan-files.any_provider.b3fc542f"} + }, + { + id: "policy", + header: "i18n:govoplan-files.policy.bb9cf141", + width: "minmax(160px, 0.8fr)", + render: (row) => {connectorPolicySummary(row)} + }, + { + id: "status", + header: "i18n:govoplan-files.status.bae7d5be", + width: "120px", + render: (row) => { + const enabled = row.kind === "profile" ? row.profile.enabled : row.credential.enabled; + return {enabled ? "i18n:govoplan-files.enabled.df174a3f" : "i18n:govoplan-files.disabled.f4f4473d"}; + } + }]; + + + function connectorChildren(row: ConnectorTreeRow): ConnectorTreeRow[] { + if (row.kind !== "profile") return []; + return scopedCredentials. + filter((credential) => firstCompatibleProfileByCredential.get(credential.id) === row.profile.id). + map((credential) => ({ kind: "credential" as const, id: `credential:${row.profile.id}:${credential.id}`, credential, profile: row.profile })); + } + + function renderConnectorActions(row: ConnectorTreeRow) { + if (row.kind === "credential") { + const readOnly = row.credential.source_kind !== "database"; + return ( +
+ + +
); + + } + const readOnly = row.profile.source_kind !== "database"; + return ( +
+ + + + +
); + + } + + return ( +
+ {error && {error}} + {message && !error && {message}} + + {!canWrite && + GLOBAL or TENANT > File Connectors" + }} /> + } + + {targetSelectionRequired && !hasSelectableTarget && + + } + + {targetSelectionRequired && + +
+ + + +
+ + } + + {scopeReady && + <> + + + +
: + null + }> + + + row.id} + getChildren={connectorChildren} + renderActions={renderConnectorActions} + emptyText="i18n:govoplan-files.no_file_connections_configured.3ef02049" /> + + + + void savePolicyDraft()} disabled={saving || loading}> +