Compare commits

..
25 Commits
Author SHA1 Message Date
zemion edee29af51 Release v0.1.16
Module Package Release / publish-packages (push) Successful in 12s
2026-08-05 19:52:09 +02:00
zemion 0293e49def Release v0.1.15
Module Package Release / publish-packages (push) Successful in 12s
2026-08-04 15:18:12 +02:00
zemion aaa364b8d4 Make package publication retries hash-safe 2026-08-04 14:32:19 +02:00
zemion 4bb17c0f4c Harden module package publication 2026-08-04 14:02:40 +02:00
zemion 92e649477f Pin S3 and SMB connector peers 2026-08-04 10:40:46 +02:00
zemion 0c68e904cf Add protected package release workflow 2026-08-04 04:14:04 +02:00
zemion 8ec31b16ee Add native permission-aware file search source 2026-08-04 03:03:26 +02:00
zemion 7e6be4b017 Add governed Files integrity operations UI 2026-08-04 01:04:39 +02:00
zemion 04882f1628 Fix PostgreSQL file visibility queries 2026-08-04 01:04:39 +02:00
zemion d8ae506ff8 Migrate Files surfaces to interface patterns 2026-08-03 09:35:52 +02:00
zemion 6baf2a421b Fence managed file object effects 2026-08-03 04:21:52 +02:00
zemion b993d8e31a Provide managed generated artifact storage 2026-08-02 12:38:15 +02:00
zemion 359c4e9570 feat: support optional encrypted file content 2026-08-02 03:40:54 +02:00
zemion 233ce40983 feat: declare governed external provider state 2026-08-01 17:48:32 +02:00
zemion f084a0f3a9 Add managed storage operational probes 2026-07-31 22:48:07 +02:00
zemion b752dea610 Add safe archive preview and extraction workflows 2026-07-31 02:48:56 +02:00
zemion 159a012833 feat: classify file administration surfaces 2026-07-30 17:42:06 +02:00
zemion 835eacfc5d feat: harden file sharing and integrity 2026-07-30 14:26:47 +02:00
zemion 85606d5580 Add server-backed file property filters 2026-07-30 05:22:09 +02:00
zemion 632cc6cf7d perf(files): batch tenant summaries 2026-07-30 01:15:53 +02:00
zemion 86a905a3a7 refactor(api): split files workflow routers 2026-07-29 20:11:17 +02:00
zemion 5b868272b9 security: harden connector policy and add dashboard widget 2026-07-29 14:16:29 +02:00
zemion 93eea839c8 Declare file workspace View surfaces 2026-07-28 21:04:55 +02:00
zemion b667e7ff0e Integrate file connectors with credential envelopes 2026-07-28 19:33:01 +02:00
zemion 0fd8f6e02a docs: update GovOPlaN repository links 2026-07-27 15:46:51 +02:00
92 changed files with 15861 additions and 4211 deletions
+270
View File
@@ -0,0 +1,270 @@
name: Module Package Release
on:
push:
tags:
- "v*"
workflow_dispatch:
inputs:
release_tag:
description: Existing protected version tag to publish
required: true
type: string
jobs:
publish-packages:
runs-on: ubuntu-latest
env:
GITEA_REPOSITORY: ${{ gitea.repository }}
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
with:
fetch-depth: 0
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065
with:
python-version: "3.12"
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020
with:
node-version: "22"
- name: Select and validate protected release tag
shell: bash
env:
REQUESTED_TAG: ${{ inputs.release_tag }}
TRIGGER_TAG: ${{ gitea.ref_name }}
run: |
set -euo pipefail
tag="${REQUESTED_TAG:-$TRIGGER_TAG}"
case "$tag" in
v[0-9]*.[0-9]*.[0-9]*) ;;
*) echo "Release tag must start with a SemVer-shaped vX.Y.Z value" >&2; exit 1 ;;
esac
git fetch --force origin "refs/tags/$tag:refs/tags/$tag" refs/heads/main:refs/remotes/origin/main
tag_commit="$(git rev-list -n 1 "$tag")"
git merge-base --is-ancestor "$tag_commit" refs/remotes/origin/main || {
echo "Release tag is not contained in main" >&2
exit 1
}
git checkout --detach "$tag"
printf 'RELEASE_TAG=%s\n' "$tag" >> "$GITEA_ENV"
printf 'SOURCE_DATE_EPOCH=%s\n' "$(git show -s --format=%ct HEAD)" >> "$GITEA_ENV"
- name: Validate package versions
run: |
python - <<'PY'
import json
from pathlib import Path
import os
import re
import tomllib
tag = os.environ["RELEASE_TAG"]
expected = tag.removeprefix("v")
project = tomllib.loads(Path("pyproject.toml").read_text(encoding="utf-8"))["project"]
if project.get("version") != expected:
raise SystemExit(f"pyproject version {project.get('version')!r} does not match {tag}")
if re.fullmatch(r"govoplan-[a-z0-9-]+", str(project.get("name", ""))) is None:
raise SystemExit("Python distribution name must use the govoplan-* namespace")
webui = Path("webui/package.json")
if webui.is_file():
package = json.loads(webui.read_text(encoding="utf-8"))
if package.get("version") != expected:
raise SystemExit(f"WebUI version {package.get('version')!r} does not match {tag}")
if re.fullmatch(r"@govoplan/[a-z0-9-]+-webui", str(package.get("name", ""))) is None:
raise SystemExit("WebUI package name must use the @govoplan/*-webui namespace")
release = Path("webui/package.release.json")
if release.is_file():
release_package = json.loads(release.read_text(encoding="utf-8"))
if (
release_package.get("name") != package.get("name")
or release_package.get("version") != expected
):
raise SystemExit("WebUI release package identity does not match package.json and the release tag")
PY
- name: Build immutable package artifacts
shell: bash
run: |
set -euo pipefail
python -m pip install --disable-pip-version-check build==1.5.0 twine==7.0.0
rm -rf dist .package-webui
python -m build --wheel --outdir dist
python -m twine check dist/*.whl
if [[ -f webui/package.json ]]; then
mkdir .package-webui
cp -a webui/. .package-webui/
rm -rf .package-webui/node_modules .package-webui/dist
if [[ -f .package-webui/package.release.json ]]; then
cp .package-webui/package.release.json .package-webui/package.json
fi
node <<'NODE'
const fs = require("node:fs");
const path = ".package-webui/package.json";
const packageJson = JSON.parse(fs.readFileSync(path, "utf8"));
const groups = ["dependencies", "optionalDependencies", "peerDependencies"];
for (const group of groups) {
for (const [name, specifier] of Object.entries(packageJson[group] || {})) {
if (!name.startsWith("@govoplan/")) continue;
if (typeof specifier !== "string") {
throw new Error(`${group}.${name} must use a string version`);
}
const packageSlug = name.slice("@govoplan/".length);
if (!packageSlug.endsWith("-webui")) {
throw new Error(`${group}.${name} is outside the WebUI package namespace`);
}
const repository = `govoplan-${packageSlug.slice(0, -"-webui".length)}`;
const escapedRepository = repository.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
const gitTag = specifier.match(
new RegExp(
`^git\\+(?:ssh://git@|https://)git\\.add-ideas\\.de/(?:GovOPlaN|add-ideas)/${escapedRepository}\\.git#v([0-9]+\\.[0-9]+\\.[0-9]+)$`,
),
);
if (gitTag) {
packageJson[group][name] = gitTag[1];
continue;
}
if (specifier.startsWith("file:") || specifier.startsWith("git+")) {
throw new Error(
`${group}.${name} must resolve to an exact registry version for publication`,
);
}
}
}
delete packageJson.private;
fs.writeFileSync(path, `${JSON.stringify(packageJson, null, 2)}\n`);
NODE
npm pkg delete private --prefix .package-webui
(cd .package-webui && npm pack --ignore-scripts --pack-destination ../dist)
fi
python - <<'PY'
import hashlib
import json
from pathlib import Path
import os
import subprocess
artifacts = []
for path in sorted(Path("dist").iterdir()):
if path.suffix not in {".whl", ".tgz"}:
continue
digest = hashlib.sha256(path.read_bytes()).hexdigest()
artifacts.append({"filename": path.name, "sha256": digest, "size": path.stat().st_size})
payload = {
"schema_version": "1",
"repository": os.environ["GITEA_REPOSITORY"],
"tag": os.environ["RELEASE_TAG"],
"commit": subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip(),
"artifacts": artifacts,
}
Path("dist/package-artifacts.json").write_text(
json.dumps(payload, indent=2, sort_keys=True) + "\n",
encoding="utf-8",
)
PY
- name: Retain package hash evidence
uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32
with:
name: module-packages-${{ gitea.ref_name }}
path: dist/package-artifacts.json
- name: Check immutable registry state
shell: bash
env:
PACKAGE_TOKEN: ${{ secrets.GOVOPLAN_PACKAGE_TOKEN }}
run: |
set -euo pipefail
test -n "$PACKAGE_TOKEN"
python - <<'PY'
import hashlib
import json
import os
from pathlib import Path
import tomllib
from urllib.error import HTTPError
from urllib.parse import quote
from urllib.request import Request, urlopen
api_root = "https://git.add-ideas.de/api/v1/packages/GovOPlaN"
token = os.environ["PACKAGE_TOKEN"]
def should_publish(kind, name, version, path):
package_url = "/".join(
(api_root, kind, quote(name, safe=""), quote(version, safe=""), "files")
)
request = Request(
package_url,
headers={"Accept": "application/json", "Authorization": f"token {token}"},
)
try:
with urlopen(request, timeout=30) as response:
files = json.load(response)
except HTTPError as exc:
if exc.code == 404:
print(f"{kind} package {name}=={version} is not published yet")
return True
raise
if not isinstance(files, list) or len(files) != 1:
raise SystemExit(
f"immutable {kind} package {name}=={version} has an unexpected file set"
)
expected_sha256 = hashlib.sha256(path.read_bytes()).hexdigest()
if files[0].get("sha256") != expected_sha256:
raise SystemExit(
f"immutable {kind} package {name}=={version} already exists with a different SHA-256"
)
print(f"verified existing {kind} package {name}=={version} ({expected_sha256})")
return False
project = tomllib.loads(Path("pyproject.toml").read_text(encoding="utf-8"))["project"]
wheels = tuple(Path("dist").glob("*.whl"))
if len(wheels) != 1:
raise SystemExit("release build must contain exactly one wheel")
publish_pypi = should_publish(
"pypi", str(project["name"]), str(project["version"]), wheels[0]
)
tarballs = tuple(Path("dist").glob("*.tgz"))
if len(tarballs) > 1:
raise SystemExit("release build must contain at most one npm package")
publish_npm = False
if tarballs:
webui = json.loads(
Path(".package-webui/package.json").read_text(encoding="utf-8")
)
publish_npm = should_publish(
"npm", str(webui["name"]), str(webui["version"]), tarballs[0]
)
with Path(os.environ["GITEA_ENV"]).open("a", encoding="utf-8") as env_file:
env_file.write(f"PUBLISH_PYPI={int(publish_pypi)}\n")
env_file.write(f"PUBLISH_NPM={int(publish_npm)}\n")
PY
- name: Publish wheel and WebUI package
shell: bash
env:
PACKAGE_USERNAME: ${{ secrets.GOVOPLAN_PACKAGE_USERNAME }}
PACKAGE_TOKEN: ${{ secrets.GOVOPLAN_PACKAGE_TOKEN }}
run: |
set -euo pipefail
test -n "$PACKAGE_USERNAME"
test -n "$PACKAGE_TOKEN"
if [[ "$PUBLISH_PYPI" == 1 ]]; then
TWINE_USERNAME="$PACKAGE_USERNAME" TWINE_PASSWORD="$PACKAGE_TOKEN" \
python -m twine upload --non-interactive \
--repository-url https://git.add-ideas.de/api/packages/GovOPlaN/pypi \
dist/*.whl
else
echo "Exact wheel is already present; skipping immutable retry."
fi
shopt -s nullglob
webui_packages=(dist/*.tgz)
if (( ${#webui_packages[@]} )) && [[ "$PUBLISH_NPM" == 1 ]]; then
npmrc="$(mktemp)"
trap 'rm -f "$npmrc"' EXIT
chmod 600 "$npmrc"
printf '%s\n' \
'@govoplan:registry=https://git.add-ideas.de/api/packages/GovOPlaN/npm/' \
"//git.add-ideas.de/api/packages/GovOPlaN/npm/:_authToken=$PACKAGE_TOKEN" \
> "$npmrc"
NPM_CONFIG_USERCONFIG="$npmrc" npm publish "./${webui_packages[0]}" \
--ignore-scripts --access public \
--registry https://git.add-ideas.de/api/packages/GovOPlaN/npm/
elif (( ${#webui_packages[@]} )); then
echo "Exact WebUI package is already present; skipping immutable retry."
fi
+6
View File
@@ -1,5 +1,11 @@
# GovOPlaN Files Codex Guide
## Documentation Contract
- Treat documentation as part of every behavior change. Update this module's manifest-driven `DocumentationTopic` contributions for affected user and administrator behavior.
- Keep feature content here; `govoplan-docs` projects it without importing Files internals.
- Maintain a static user/admin baseline and run `/mnt/DATA/git/govoplan/tools/checks/check-manifest-shapes.py` after behavior or manifest changes.
## Scope
This repository owns the `files` module: managed file storage APIs, file metadata, shares, uploads/downloads, folder and pattern helpers, backend module manifest, and `@govoplan/files-webui`.
+53 -15
View File
@@ -105,7 +105,8 @@ run profile policy with the `browse` operation, and support Seafile,
WebDAV/Nextcloud, and SMB when the optional `smb` extra is installed. Seafile
profiles browse libraries and directories via Seafile's read-only API; profiles
can still opt into WebDAV browsing by setting `metadata.webdav_endpoint_url` or
`metadata.browse_protocol` to `webdav`.
`metadata.browse_protocol` to `webdav`. S3-compatible profiles support bucket
and prefix browsing when the optional `s3` extra is installed.
`POST /api/v1/files/connectors/profiles/{profile_id}/import` imports a Seafile
or WebDAV/Nextcloud/SMB file into managed storage through the same governance,
conflict handling, source provenance, and connector audit path as direct
@@ -113,17 +114,30 @@ uploads. The Seafile provider uses account-token auth and the native file
download-link API; Nextcloud and generic WebDAV profiles use authenticated `GET`
requests against the configured WebDAV endpoint. SMB profiles use
`smb://server[:port]/share[/path]` endpoints and deployment-owned or encrypted
stored credentials through `smbprotocol`. Because that SDK cannot accept a
preconnected socket and may follow DFS referrals to additional hosts, live SMB
access fails closed in both public-only and private-network deployments. It will
remain disabled until every initial connection and referral target can be
policy-validated and pinned.
stored credentials through `smbprotocol`. Files installs a pinned transport for
every initial session, reconnect, alias, and DFS referral target. The deployment
private-network policy is re-evaluated immediately before each socket opens.
The S3 browse/import implementation and S3 managed-storage backend currently
fail closed before creating a `boto3` client. Botocore does not yet use the
GovOPlaN pinned HTTP transport and may manage redirects itself, so neither an
explicit endpoint nor SDK endpoint discovery is allowed until both peer pinning
and redirect revalidation are enforced.
S3 connector browse/import binds botocore HTTP and HTTPS pools to the same pinned
socket policy. Retries, redirects, endpoint discovery, bucket aliases, and new
connections are therefore revalidated while TLS keeps the configured hostname
for SNI and certificate checks. Connector clients do not use outbound proxies or
ambient AWS credential discovery; configure credentials on the governed profile,
or explicitly use an anonymous profile for public objects. An incompatible SDK
upgrade fails closed before a usable client/session is returned.
Durable platform storage has a separate deployment boundary: installer-owned
Garage is accepted only at its exact generated endpoint, while an
operator-selected external backend requires a clean HTTPS origin and
`FILE_STORAGE_S3_ENDPOINT_TRUSTED=true`. That flag is deployment configuration,
cannot be supplied through a Files connector profile, and does not replace
operator responsibility for DNS, certificates, egress, bucket policy,
versioning, and recovery.
The actual local/S3 backend implementation is owned by Core so Files, Campaign,
and workers resolve the same object namespace. Files owns file metadata and key
layout. Node-local storage is supported only for `local` or one-host
`host-shared` profiles; multi-host `shared` deployments require S3.
Destructive Files-module retirement applies the same credential lifecycle before
dropping tables. Every remaining Files-owned encrypted connector secret is
@@ -139,17 +153,41 @@ Connector and collaboration ownership boundaries are documented in
`docs/CONNECTOR_BOUNDARY.md` and `docs/DOCUMENT_COLLABORATION_BOUNDARY.md`.
The role-adaptive user, administration, integration, and operator guide is the
[Files handbook](docs/FILES_HANDBOOK.md).
The Files route, connector surfaces, consequence classes, and pattern-language
verification are recorded in
[Files interface pattern migration](docs/INTERFACE_PATTERN_MIGRATION.md).
ZIP uploads are processed without buffering the whole archive in memory. The API
spools incoming ZIP request bodies to a bounded temporary file, then extracts
members with per-file and total extracted-size limits before storing managed
files.
Archive imports use a two-phase preview and confirmation flow for ZIP, TAR,
TAR.GZ, TAR.BZ2, and TAR.XZ. Requests are spooled to bounded temporary files;
the server validates paths, entry count, expanded size, and expansion ratio,
then returns a 30-minute tenant/user-bound preview token. Confirmation reuploads
the original archive and stores only the selected members. Password-protected
ZIP passwords remain request-only and are never included in the preview token.
Managed blob writes and applied orphan cleanup use Core's durable recovery
ledger. Intent, request digests, recovery mode, and a distributed lease are
committed before physical storage effects; the Files session commit verifies
database and streamed object evidence, while rollback compensates only a newly
reserved unreferenced key. New object keys are opaque and do not retain the
uploaded filename. Uncertain or mismatched effects remain visible through Ops.
Operators with `files:file:admin` can run bounded, resumable integrity scans in
Administration. Scan batches and finding actions carry monotonic revisions;
stale resume, recheck, or cleanup requests fail before touching object storage.
Orphan cleanup always requires a dry-run preview followed by separate
confirmation and records recovery-ledger evidence.
Bulk rename and transfer APIs are owner-scoped: callers must provide the active
user or group file space with `owner_type` and `owner_id`. The storage layer
keeps a named legacy file-only helper for historical callers that lack owner
context, and regression tests cover its write-access checks.
Optional producer modules can persist generated output through the provider-
neutral `files.artifact_store` capability. Files remains responsible for upload
authorization, ownership, path normalization, versioning, and blob storage;
producers receive stable file/version references without importing Files
internals. See [Generated Artifact Store](docs/GENERATED_ARTIFACT_STORE.md).
## Release packaging
The repository root includes a `package.json` for git-based WebUI installs. It exports the package `@govoplan/files-webui` from `webui/src` so release builds can depend on tagged git refs instead of local `file:` paths.
+9 -5
View File
@@ -19,6 +19,7 @@ normal product workflows and can share the same governance model:
- Nextcloud through WebDAV
- generic WebDAV
- SMB through the optional `smb` extra
- S3-compatible stores through the optional `s3` extra
These providers are surfaced through connector descriptors at
`GET /api/v1/files/connectors/providers`. Provider descriptors declare whether
@@ -60,11 +61,14 @@ Every provider must:
- use a transport that pins every connection to a policy-validated DNS/IP answer
and revalidates redirects; SDK transports without that guarantee fail closed
Live SMB access is disabled in all modes until `smbprotocol` initial connections
and DFS referral targets can be pinned and policy-validated. An explicit IP is
not sufficient because the server may still issue a referral to another peer.
Live S3 access is likewise disabled until `boto3`/botocore can be bound to the
pinned transport, including SDK-managed redirects and endpoint discovery.
SMB initial connections, reconnects, aliases, and DFS referral targets are
created through a Files-owned pinned `smbprotocol` transport. S3 HTTP/HTTPS pools
use the equivalent botocore adapter for every connection selected by retries,
redirects, endpoint discovery, and virtual-host addressing. Both adapters apply
the deployment-wide private-network policy at socket creation. S3 keeps the
configured hostname for TLS SNI and certificate verification, but does not use
outbound proxies or ambient AWS credential discovery. An incompatible optional
SDK release fails closed before a usable session or client is returned.
## Non-Goals For Files
+212 -69
View File
@@ -24,17 +24,27 @@ a document collaboration engine, or a records-management system.
| Verify a release or scenario | [Acceptance scenarios](#acceptance-scenarios) |
| Check whether an idea exists today | [Implemented and planned boundary](#implemented-and-planned-boundary) |
The Files-owned interface archetypes, consequence classes, disabled-state
wording, and verification evidence are recorded in
[Files Interface Pattern Migration](INTERFACE_PATTERN_MIGRATION.md).
## The service contract
A managed file is a tenant-scoped logical asset with exactly one user or group
owner, a normalized path, and a current version. The current version points to a
blob record containing the storage location, SHA-256 checksum, byte size, and
content type. Current service paths append a version when connector sync finds
content type. A protected blob also records its Encryption envelope, protection
discriminator, stored-ciphertext checksum, and stored-ciphertext size. Current
service paths append a version when connector sync finds
changed content; they do not mutate the previous version record.
Content with the same tenant, SHA-256 checksum, and size can reuse one blob.
This is storage deduplication and integrity evidence, not proof of authorship or
source authenticity.
Unprotected content with the same tenant, plaintext SHA-256 checksum, size, and
protection discriminator can reuse one blob. Protected content is deduplicated
only inside the same vault/profile discriminator; ciphertext is never silently
reused across protection boundaries. Plaintext checksums remain semantic
version evidence, while download and integrity scans verify stored ciphertext
before asking Encryption to open it. Neither digest proves authorship or source
authenticity.
The main domain objects are:
@@ -42,7 +52,7 @@ The main domain objects are:
| --- | --- | --- |
| File asset | The user-facing file identity, owner, logical path, description, metadata, and current version | Created, organized, shared, and soft-deleted |
| File version | A numbered snapshot of one asset and its blob | Appended by connector sync when bytes change; retained |
| File blob | Stored bytes plus checksum, size, backend, key, reference count, and optional retention timestamp | Reused within a tenant; no automated garbage collection |
| File blob | Stored plaintext semantics plus stored-byte integrity, backend key, optional Encryption envelope, reference count, and retention timestamp | Reused only within a tenant and matching protection boundary; no automated garbage collection |
| Folder | An explicit logical path in a user or group space | Created, moved/renamed through organize operations, and soft-deleted |
| Share | A grant from an asset to a user, group, tenant, or campaign with `read`, `write`, or `manage` permission | Created or updated; no public revocation endpoint yet |
| Connector profile | A governed external endpoint, scope, optional credential link, policy, and descriptive capabilities | Created, updated, disabled, or credential-scrubbed on deletion |
@@ -94,21 +104,29 @@ An ordinary upload does not append a new version to an existing asset. The
`overwrite` strategy retires the old asset at that path. Version-preserving
updates currently belong to connector sync.
### Upload and unpack ZIP files
### Preview and unpack archives
The UI can unpack a ZIP upload. The request body is spooled to a bounded
temporary file rather than buffered wholly in memory. The defaults are:
The UI previews ZIP, TAR, TAR.GZ, TAR.BZ2, and TAR.XZ before writing managed
files. Users may select individual files or complete folders. The request is
spooled to a bounded temporary file rather than buffered wholly in memory. The
defaults are:
- 250 MiB compressed request and total extracted data;
- 250 MiB compressed request;
- 50 MiB per extracted member;
- 1,000 non-directory members;
- encrypted archives are rejected;
- member paths are normalized and `..` traversal is rejected;
- the actual bytes read are counted, not only the ZIP header declarations.
- 2 GiB total expanded data;
- 10,000 declared entries;
- 100:1 maximum expansion ratio;
- a 30-minute preview token bound to the tenant, user, archive digest, and
destination;
- password-protected ZIP support with request-only password handling;
- traversal, duplicate paths, links, devices, and other special entries are
rejected;
- actual bytes read are counted, not only archive header declarations.
The byte limits use `FILE_UPLOAD_ZIP_MAX_BYTES` and
`FILE_UPLOAD_MAX_BYTES`. The 1,000-member limit is currently fixed in the
extraction service.
The browser retains the selected archive and password until confirmation.
Confirmation reuploads the archive, verifies the token and digest, repeats all
safety checks, and commits only the selected files. No preview archive or
password is retained server-side.
### Organize files and folders
@@ -171,6 +189,14 @@ inside the chosen owner space:
Browse, import, and sync never write, rename, or delete the remote source.
Connector administration separates endpoint profiles, reusable credentials,
and inherited policy. Ordinary setup uses typed fields and provider discovery;
provider metadata JSON is available only under advanced compatibility options.
Read-only deployment entries explain where they must be changed, and disabled
actions identify the missing permission, target, input, or running operation.
The contextual help icon opens the configured Help Center topic when Docs is
enabled and the hosted GovOPlaN documentation otherwise.
## Process perspective
### Managed ingestion
@@ -187,9 +213,10 @@ The managed-file flow is:
8. Connector-originated operations also emit their connector audit evidence.
Blob storage is not part of the database transaction. An object may therefore
be left without committed metadata after a process or database failure. No
automatic orphan reconciliation exists yet; operators must account for this in
integrity checks and retention plans.
be left without committed metadata after a process or database failure. The
operator integrity API scans database blobs and the tenant storage prefix in
bounded, resumable phases. It reports orphan objects before any cleanup and
never deletes them as part of a scan.
### Governed connector import
@@ -378,15 +405,15 @@ in a development/test runtime.
| Seafile | Read-only native API browse/download-link import and manual sync implemented using the pinned HTTP transport; WebDAV opt-in supported |
| Nextcloud | Read-only WebDAV browse/import/manual sync implemented using the pinned HTTP transport |
| Generic WebDAV | Read-only browse/import/manual sync implemented using the pinned HTTP transport |
| SMB | Browse/import code and descriptor implemented, but every live connection fails closed until initial connections and DFS referrals can be policy-validated and pinned |
| S3 connector | Browse/import code and descriptor implemented, but every live SDK connection fails closed until botocore connections, redirects, and endpoint discovery can be validated and pinned |
| SMB | Read-only browse/import/manual sync implemented through a pinned smbprotocol transport for initial peers, reconnects, aliases, and DFS referral targets |
| S3 connector | Read-only bucket/prefix browse, import, and manual sync implemented through pinned botocore pools covering retries, redirects, endpoint discovery, and provider aliases |
| SharePoint and OneDrive | Provider keys/descriptors reserved; live Microsoft Graph browse/import is planned |
| NFS and local connector | Described as optional future providers; the local managed-storage backend is a different feature |
Provider descriptors are available from
`GET /api/v1/files/connectors/providers`. Use their `implemented`, `installed`,
and support fields for display, but expect a fail-closed transport error where
the table above says live access is disabled.
and support fields for display. An incompatible optional SDK release fails closed
before it can return a usable client or session.
## Operator runbook
@@ -398,8 +425,15 @@ the table above says live access is disabled.
| `FILE_STORAGE_LOCAL_ROOT` | `runtime/files` | Primary local write/read root |
| `FILE_STORAGE_LOCAL_FALLBACK_ROOTS` | empty | Comma-separated older read-only roots checked after the primary root |
| `FILE_STORAGE_S3_ENDPOINT_URL` and related `FILE_STORAGE_S3_*` values | deployment-specific | S3-compatible endpoint, region, credentials, and bucket |
| `FILE_UPLOAD_MAX_BYTES` | 50 MiB | Direct-upload and ZIP-member maximum |
| `FILE_UPLOAD_ZIP_MAX_BYTES` | 250 MiB | ZIP request and extracted-total maximum |
| `FILE_STORAGE_S3_DEPLOYMENT_MANAGED` | `false` | Installer-only trust marker for the exact `http://garage:3900` service; never use it for another endpoint |
| `FILE_STORAGE_S3_ENDPOINT_TRUSTED` | `false` | Deployment-owner acknowledgement for one clean HTTPS external S3 origin; never expose this through connector configuration |
| `GOVOPLAN_STATE_PROFILE` | `local` | Selects `local`, one-host `host-shared`, or multi-host `shared` state validation |
| `FILE_UPLOAD_MAX_BYTES` | 50 MiB | Direct-upload and extracted archive-member maximum |
| `FILE_UPLOAD_ZIP_MAX_BYTES` | 250 MiB | Compressed archive request maximum (legacy name retained for compatibility) |
| `FILE_ARCHIVE_MAX_ENTRIES` | 10,000 | Maximum declared archive entries |
| `FILE_ARCHIVE_MAX_EXPANDED_BYTES` | 2 GiB | Maximum expanded archive bytes |
| `FILE_ARCHIVE_MAX_EXPANSION_RATIO` | 100 | Maximum expanded-to-compressed ratio |
| `FILE_ARCHIVE_PREVIEW_TTL_SECONDS` | 1,800 | Lifetime of the sealed archive preview token |
| `MASTER_KEY_B64` | development fallback only | Encrypts database-managed connector secrets |
The local backend is the operational baseline. It resolves every storage key
@@ -407,16 +441,19 @@ under the configured root and rejects escape attempts. Fallback roots support a
controlled storage-root migration: new writes go to the primary root while
reads can still find older objects.
The S3 managed-storage adapter currently fails closed before creating a boto3
client because the SDK cannot yet guarantee connection-time DNS/IP pinning and
redirect revalidation. Do not select `FILE_STORAGE_BACKEND=s3` for a live Files
deployment until that boundary is implemented and the status above changes.
The supported installer may provision a deployment-owned Garage service at the
exact `http://garage:3900` endpoint and set
`FILE_STORAGE_S3_DEPLOYMENT_MANAGED=true`. An operator-selected external S3
backend instead requires a clean HTTPS origin and
`FILE_STORAGE_S3_ENDPOINT_TRUSTED=true`. Both are deployment authority, not a
general connector or private-network bypass. Core owns the shared backend
implementation; Files owns metadata and the Files key namespace.
Multiple API replicas require the same durable blob namespace. Separate local
container filesystems will produce incomplete reads. Until a pinned shared
object-storage transport is available, use one durable shared mount or constrain
Files traffic to a deployment topology that preserves one consistent local
root.
container filesystems will produce incomplete reads. Use `host-shared` with one
durable shared mount only for same-host replicas. Independent hosts require the
`shared` profile with external S3, PostgreSQL, Redis, a stable installation id,
and one immutable module composition.
### Connector egress
@@ -441,15 +478,29 @@ The built-in HTTP transport:
- bounds structured responses to 16 MiB and file transfers to 512 MiB by
default.
The S3 SDK adapter applies the same socket rule to every botocore pool selected
for a retry, redirect, discovered endpoint, or virtual-host bucket alias. The
original authority remains in the request and TLS SNI/certificate check. S3
connector clients use no outbound proxy and never discover ambient AWS
credentials: configure both access and secret keys on the governed profile, or
use an anonymous profile for a public source.
The SMB adapter owns a separate connection cache and replaces smbprotocol's TCP
factory process-wide with the stricter pinned socket. Initial peers, reconnects,
server aliases, domain-controller connections, and DFS referral targets therefore
pass the same policy at connection time. Signing is required by default; enable
SMB encryption on the profile where the server supports it.
Override the connector response limits with
`GOVOPLAN_CONNECTOR_MAX_STRUCTURED_RESPONSE_BYTES` and
`GOVOPLAN_CONNECTOR_MAX_FILE_TRANSFER_BYTES`. The smaller applicable limit wins
when an import is also subject to `FILE_UPLOAD_MAX_BYTES`.
Never work around a pinning failure by adding a raw IP, disabling TLS, or
enabling private networks. SMB may redirect through DFS, and S3 SDKs may perform
their own redirects or endpoint discovery; both remain disabled even for an IP
literal until every connection peer can be governed.
Never work around a connector pinning failure by adding a raw IP, disabling TLS,
or enabling private networks. A failure means the peer policy rejected an actual
connection destination or the installed SDK no longer exposes the verified
transport seam. The separately configured platform S3 backend is trusted only by
the deployment owner and is not selectable by a user or connector profile.
### Backup and restore
@@ -459,13 +510,87 @@ include:
- Files database rows, including asset/version/blob relationships, shares,
connector settings, and campaign attachment-use evidence;
- every object below `FILE_STORAGE_LOCAL_ROOT` and any still-used fallback root;
or the complete S3 bucket/prefix and version/lifecycle evidence for an S3
backend;
- the exact `MASTER_KEY_B64` needed to decrypt retained connector credentials;
- deployment-owned connector profile files, referenced CA bundles, and secret
environment configuration where those definitions are in use.
There is no Files backup/restore API and no automated blob integrity or orphan
reconciliation job. Use a write quiesce or coordinated snapshots so database
references and objects represent the same recovery point.
There is no Files backup/restore API. Use a write quiesce or coordinated
snapshots so database references and objects represent the same recovery point.
The integrity API verifies a restored set, but it does not replace a coordinated
backup.
Operators normally use **Administration > File integrity**. The equivalent API
creates a scan with `POST /api/v1/files/integrity/scans`, then calls
`POST /api/v1/files/integrity/scans/{scan_id}/run` with the scan's current
`expected_revision` until it reports `completed`. Each call advances at most
the persisted batch size, so a stopped operator or worker can resume from the
committed blob/object cursors. Concurrent or stale actions receive `409` before
the storage backend is invoked; reload the scan and inspect the newer state.
Findings distinguish:
- `missing`: metadata references an absent object;
- `size_mismatch` or `checksum_mismatch`: bytes do not match immutable blob
metadata and the blob is quarantined;
- `orphan_object`: an object exists in the tenant Files prefix without a
corresponding blob row.
Missing or corrupt blobs fail closed for ordinary downloads and Campaign
attachment materialization. After restoring the expected bytes, use the finding
`recheck` action with its current `expected_revision`. Orphan cleanup starts
with a dry-run preview and requires separate destructive confirmation. The
confirmation reuses the finding revision from that preview, rechecks that no
database reference exists, remains scoped to the scanned tenant prefix, and is
idempotent. Both applied and dry-run actions emit audit evidence. A shared
reference blocks deletion. Files currently has no legal-hold or hard-purge
model, so retention-controlled objects must not be treated as cleanup
candidates until those controls are implemented.
### Recovery ledger for object effects
Every managed blob creation or integrity repair starts a Core recovery
operation in an independent committed transaction before Files protects or
writes bytes. The operation records tenant/blob identifiers, an opaque object
locator or locator digest, semantic SHA-256/size evidence, the recovery mode,
and a distributed lease fence. It never records file contents, ZIP passwords,
connector credentials, or a newly uploaded filename. New object keys are opaque;
legacy filename-bearing keys remain readable but repair operations record only
their digest and recover through the blob ID.
The Files business transaction then creates or updates the blob, version, and
asset rows. Its actual SQLAlchemy commit or rollback settles every pending
operation:
- commit reloads the blob through an independent session and streams the object
to verify its stored-byte SHA-256 and size before recording success;
- rollback deletes only a newly reserved object after independently proving
that no `FileBlob` references it, then records verified compensation;
- a repaired existing object is forward-completed only when its identity,
envelope, semantic evidence, and stored bytes all match;
- missing or mismatched bytes quarantine a committed blob and leave the
operation `recovery_required`; an unavailable probe remains
`outcome_unknown` rather than becoming an ordinary upload failure.
Applied orphan cleanup has its own forward-recovery operation. The database
reference check and tenant-prefix check happen before deletion; object absence
and the durable finding state are verified afterward. If the caller transaction
rolls back after deletion, Files may forward-complete only that existing
finding after rechecking that the key is still unreferenced.
Archive preview and confirmation use bounded process-local temporary staging.
Staging is not authoritative and is removed on every handled exit; extracted
members enter the same per-blob recovery boundary as direct uploads. A hard
process loss may leave a temporary OS file for normal host temporary-file
cleanup, but cannot make that staging path a managed Files object.
Use the Ops recovery-operation view to inspect `files` operations. Do not retry
a busy or unresolved blob blindly: first verify the FileBlob row, object hash,
integrity state, and any Encryption envelope named by the blob. Hard purge,
legal hold, and two-way remote connector mutation are not implemented yet, so
they cannot claim recovery-ledger adoption; their owning work remains tracked
separately.
After restore:
@@ -493,9 +618,10 @@ Legacy external secret references are detached and identified as non-owned;
Files never calls a provider delete operation for them.
The retirement executor drops database tables but does not delete corresponding
objects from the configured blob backend. Operators must include those orphaned
objects in the approved retention/destruction plan. Validate the installer
snapshot and independent blob backup before retirement.
objects from the configured blob backend. Operators must include those objects
in the approved retention/destruction plan, report them with an integrity scan,
and explicitly approve cleanup. Validate the installer snapshot and independent
blob backup before retirement.
### Operational signals
@@ -506,7 +632,7 @@ Use these symptoms as routing hints:
| `Stored object does not exist` | Database/blob restore mismatch, wrong root, or missing shared storage |
| `Stored secret cannot be decrypted` | Wrong or rotated `MASTER_KEY_B64` |
| Private/non-public endpoint blocked | Deployment-wide egress policy is public-only or DNS returned a forbidden answer |
| SDK cannot pin redirects/referrals | Expected fail-closed S3/SMB boundary, not a transient connector outage |
| SDK peer-pinning seam is unavailable | Optional S3/SMB SDK is incompatible; keep access fail-closed and validate the supported dependency range before upgrade |
| Connector response exceeds limit | Remote payload exceeds connector or upload limit |
| Profile is not visible | Scope, disabled state, campaign access, or policy mismatch |
| Group removal is vetoed | The group still owns a file/folder/connector space or is a share target |
@@ -535,7 +661,7 @@ All routes below are under `/api/v1/files`.
| Area | Routes |
| --- | --- |
| Spaces and content | `GET /spaces`, `GET /`, `GET /folders`, `GET /delta` |
| Upload and folders | `POST /upload`, `POST /upload-zip`, `POST /folders`, `POST /folders/delete` |
| Upload and folders | `POST /upload`, `POST /upload-zip` (compatibility), `POST /archive-preview`, `POST /archive-confirm`, `POST /folders`, `POST /folders/delete` |
| File access | `GET /{file_id}`, `GET /{file_id}/download`, `DELETE /{file_id}`, `POST /bulk-delete` |
| Organization | `POST /bulk-rename`, `POST /transfer`, `POST /archive.zip`, `POST /resolve-patterns` |
| Sharing | `POST /{file_id}/shares`, `POST /bulk-shares` |
@@ -580,21 +706,29 @@ Files baseline indiscriminately.
- Owner and share checks protect individual resources after operation-scope
checks.
- Logical paths reject traversal and storage keys cannot escape the local root.
- Upload, ZIP extraction, connector response, and S3 stream code use bounded
reads; encrypted ZIP archives are rejected.
- Upload, archive extraction, connector response, and S3 stream code use bounded
reads. Archive previews are sealed and short-lived; ZIP passwords are
request-only.
- Connector HTTP sockets use connection-time DNS/IP validation and pinning,
redirects are refused, and unsafe SDK transports fail before client creation.
- Database-managed connector passwords/tokens are encrypted; responses redact
secrets and deployment references.
- API metadata is recursively checked for secret-like values.
- Downloads use sanitized attachment filenames.
- SHA-256 and byte size are recorded for every blob/version.
- Plaintext semantic and stored-byte SHA-256/size evidence are recorded for
every protected blob; they are identical for unprotected blobs.
- Upload and archive-confirm APIs can select an Encryption vault. Protected
writes and reads fail closed if the optional Encryption capability is absent.
- Managed-object writes and applied orphan cleanup start lease-fenced Core
recovery operations before their physical effects; terminal success and
compensation require independent database and object checks.
The module does **not** currently provide malware scanning, content disarm and
reconstruction, a file-type allowlist, per-user quota, at-rest encryption for
local blob bytes, or a dedicated preview sandbox. Deployments that require
these controls must supply them outside Files until explicit module contracts
exist.
reconstruction, a file-type allowlist, per-user quota, automatic encryption
policy assignment, client E2EE, or a dedicated preview sandbox. The optional
server-envelope profile protects selected managed blob bytes at rest but remains
server-decryptable. Deployments that require the other controls must supply
them outside Files until explicit module contracts exist.
### Provenance
@@ -657,8 +791,9 @@ delete against an already scrubbed tombstone does not recreate secret evidence.
File versions and blobs are effectively retained indefinitely today. Although a
blob has `ref_count` and `retained_until` fields, no complete retention-policy,
legal-hold, hard-purge, or garbage-collection service enforces them. There is
also no supported user restore endpoint for soft-deleted assets/folders and no
automated orphan-object reconciliation.
also no supported user restore endpoint for soft-deleted assets/folders.
Integrity reconciliation is operator-triggered and is not a retention or
automatic garbage-collection policy.
Do not promise erasure, timed retention, legal hold, or self-service recovery
from the current soft-delete behavior. Those require an explicit, auditable
@@ -675,12 +810,14 @@ her personal or Finance space, then the asset has the selected owner and tenant.
Given Bob is outside Finance and has no share or Files admin permission, the
same asset ID must not make the asset readable to Bob.
### Safe ZIP import
### Safe archive import
Given a ZIP member contains `../../secret.txt`, is encrypted, exceeds 50 MiB,
or pushes actual extracted bytes above 250 MiB, unpacking must fail without a
committed managed asset. A valid archive must preserve normalized relative paths
below the chosen logical folder.
Given an archive member contains `../../secret.txt`, is a link or special
filesystem object, exceeds 50 MiB, pushes actual expanded bytes above 2 GiB, or
exceeds the 100:1 expansion ratio, preview or confirmation must fail without a
committed managed asset. A password-protected ZIP requires the correct
request-only password. A valid confirmation must match its unexpired preview
token and preserve normalized relative paths below the chosen logical folder.
### Explicit conflicts
@@ -699,9 +836,11 @@ bypass the same rule.
### Pinned connector transport
Given public-only mode and DNS returns any private address, the connection must
be rejected before a socket opens. Given private mode, the HTTP connection must
still use the validated address and reject redirects. SMB and S3 must fail
before their SDK clients connect until all SDK-managed peers can be pinned.
be rejected before a socket opens. Given private mode, every HTTP, S3, and SMB
connection must still use an address from the answer validated for that exact
attempt. Botocore retries, redirects, endpoint discovery, and aliases, plus SMB
reconnects and DFS referrals, must pass through the pinned factories. A changed
or unsupported SDK seam must fail before a usable client/session is returned.
### Imported evidence and sync
@@ -735,14 +874,14 @@ returning different content or credentials.
| Area | Implemented now | Planned or explicitly outside the current boundary |
| --- | --- | --- |
| Managed storage | Local durable-root backend, fallback read roots, tenant blob deduplication, checksums | Operational S3 after pinned SDK transport ([#34](https://git.add-ideas.de/add-ideas/govoplan-files/issues/34)); automated integrity/orphan reconciliation ([#36](https://git.add-ideas.de/add-ideas/govoplan-files/issues/36)) |
| Upload | Bounded direct upload, drag-and-drop UI, ZIP spool/extract limits, explicit conflicts | Malware scanning, quotas, type policy, resumable/chunked upload |
| Managed storage | Core local/S3 backend, exact managed-Garage or explicitly trusted HTTPS external S3, state-profile validation, fallback local read roots, tenant blob deduplication, checksums, bounded resumable integrity scans, quarantine, dry-run-first orphan cleanup, and Core-ledger verification/forward recovery | Scheduled scan execution and deployment-specific S3 HA/backup automation |
| Upload | Bounded direct upload, drag-and-drop UI, archive preview/selective extraction, password-protected ZIP support, explicit conflicts, opaque new object keys, and rollback compensation | Malware scanning, quotas, type policy, resumable/chunked upload |
| Organization | Folders, bulk rename preview/apply, move/copy, drag-and-drop, ZIP download, pattern resolution | General file-history UI and user-driven append-version/restore |
| Sharing | User/group/tenant/campaign grant or permission update through API; campaign linkage display | Share revocation/expiry and complete general share-management UI ([#37](https://git.add-ideas.de/add-ideas/govoplan-files/issues/37)) |
| Deletion/retention | Soft-delete assets/folders/spaces; immediate audited connector-secret scrubbing | File restore API, hard purge, retention policy, legal hold, and blob GC ([#38](https://git.add-ideas.de/add-ideas/govoplan-files/issues/38)) |
| Sharing | User/group/tenant/campaign grants, expiry, idempotent revocation, searchable share-management UI, and campaign linkage display | Richer policy-driven share lifecycles |
| Deletion/retention | Soft-delete assets/folders/spaces; immediate audited connector-secret scrubbing | File restore API, hard purge, retention policy, legal hold, and blob GC ([#38](https://git.add-ideas.de/GovOPlaN/govoplan-files/issues/38)) |
| Connector governance | Scoped profiles/credentials/policies, effective source explanation, separate credentials, linked user/group spaces | Provider-owned external secret lifecycle; API `secret_ref` remains rejected |
| HTTP connectors | Pinned, bounded, no-redirect Seafile and WebDAV/Nextcloud browse/import/manual sync | Background/folder sync, remote mutation, long-running transfer workers |
| SMB and S3 connectors | Provider descriptors and browse/import logic | Live access only after SDK connections plus DFS referrals/redirects are validated and pinned ([SMB #35](https://git.add-ideas.de/add-ideas/govoplan-files/issues/35), [S3 #34](https://git.add-ideas.de/add-ideas/govoplan-files/issues/34)) |
| SMB and S3 connectors | Provider descriptors, browse/import/manual sync, pinned SDK transports, and redirect/retry/referral transport-contract tests | Live topology smoke evidence, provider-specific OAuth, remote writes, and background indexing remain separate deployment or connector-module concerns |
| Other providers | Reserved SharePoint/OneDrive keys and NFS/local descriptors | Graph/OAuth/provider paging, NFS deployment integration, DMS connectors |
| Connector spaces | User/group link, browse, manual selected-file sync, edit/disable/delete | Background sync, remote writes/deletes, full conflict-reporting jobs |
| Profile capabilities | Stored and displayed | Enforce capability flags as an independent operation gate |
@@ -764,11 +903,15 @@ Before releasing Files:
5. Exercise an allowed and denied owner/share path.
6. Exercise upload, ZIP bounds, conflict handling, download, and soft deletion.
7. Exercise connector policy explanation and one pinned HTTP provider where
configured; verify S3/SMB still fail closed.
configured; verify the deployment-managed or trusted external S3 backend if
selected, and verify one configured S3 and SMB connector while recording the
actual target topology and private-network policy.
8. Verify credential deletion scrubs dependents and produces audit evidence.
9. Verify a campaign attachment snapshot still identifies its exact version and
checksum after the current file changes.
10. Update the implemented/planned table whenever a boundary changes.
10. Exercise a committed upload, a rolled-back upload, object tamper detection,
and applied orphan cleanup; inspect their `files` operations in Ops.
11. Update the implemented/planned table whenever a boundary changes.
## Related documents
+23
View File
@@ -0,0 +1,23 @@
# Generated Artifact Store
Files implements Core's optional `files.artifact_store` capability for modules
that generate deterministic output without owning file storage.
The producer supplies bytes, filename, content type, destination folder,
optional idempotency key, and bounded non-secret provenance. Files applies the
actor's `files:file:upload` permission, tenant/user ownership, path rules,
versioning, configured blob backend, and conflict behavior. An idempotency key
is represented as source provenance so an unchanged retry does not create an
unrelated file version.
The shared Files session owns finalization. Before a new managed object is
written, Files commits a lease-fenced Core recovery operation containing only
identifiers and digests. The caller's eventual session commit independently
verifies both `FileBlob` metadata and stored bytes; rollback compensates only an
unreferenced key. Producers must therefore complete the supplied transaction
normally and must not bypass or replace Files session lifecycle handling.
The response contains only file/version identifiers, display path, media type,
size, digest, and storage provenance. Producers must not put credentials,
tokens, or rendered plaintext into metadata. Storing an artifact proves Files
accepted it; it does not prove printing, mailing, or any other external effect.
+55
View File
@@ -0,0 +1,55 @@
# Files Interface Pattern Migration
This inventory records the Files-owned part of the GovOPlaN interface pattern
language. Core owns the shell and shared components; Files owns the composition
and consequences described here.
## Surface inventory
| Surface | Primary task | Archetype | Consequence | Pattern evidence |
| --- | --- | --- | --- | --- |
| `/files` space and folder panes | Browse managed and connected content without losing location | Directory/explorer | Low for navigation; medium for exposing filenames and provenance | Full-height two-pane workspace, bounded panes, stable selection and contextual Help Center link |
| `/files` toolbar and property filters | Find and act on the current selection | Explorer actions and local filtering | Medium for upload, move, copy, share and synchronization; high for delete | Actions remain beside the affected list, disabled controls explain permission/state/selection blockers, destructive work uses `ConfirmDialog` |
| Upload/archive, transfer, rename and connector-import dialogs | Supply and review one bounded change | Adaptive create/edit or guided import | Medium to high because files, paths and external bytes change | Shared `Dialog`, `FileDropZone`, validation, conflict review, unsaved inputs and explicit confirmation |
| File share dialog | Inspect and change access | Review/decision | High because another actor gains access | Shared dialog, access explanation, stable row actions and destructive confirmation |
| System/tenant/group/user connector surfaces | Compare connections, credentials and effective policy | Administration/configuration | High because endpoints, secrets and inherited policy control external access | Shared `ConnectionTree`, adaptive forms, `ActionBlockerHint`, policy provenance and contextual admin help |
| Connection and credential editors | Create or edit one governed endpoint or secret | Adaptive create/edit | High because a saved change may enable remote access | Relevant fields only, typed credential controls, discovery/test, advanced compatibility section, unsaved-change guard and disabled-save reasons |
| Connector policy card | Narrow inherited connector access | Effective-policy editor | High because deny/allow changes affect lower scopes | Typed reference selectors, deny precedence warning, effective source evidence and permission blocker |
| `files.widget.spaces` | See available managed/connected spaces and open Files | Dashboard widget | Low; names and provider state may still be sensitive | Shared loading/alert/status components, bounded item count, permission-filtered contribution |
| Files chooser capability used by another module | Select a managed snapshot without importing Files internals | Directory chooser | Medium because the exact selected version becomes another module's input | Shared dialog/confirmation, capability boundary and exact file/version evidence |
## State and consequence contract
- Loading, errors, success, empty results, access explanations and confirmation
use Core components. Files does not reproduce the application shell.
- A connector that comes from deployment settings remains visible but read-only;
its action explains that bootstrap configuration and a restart are required.
- Missing permission, target, selection, endpoint, or compatible provider is an
explained disabled state. It is not represented only by color or absence.
- Provider metadata JSON is an expert compatibility escape hatch inside the
collapsed shared advanced-options component. Ordinary connector setup uses
typed provider, endpoint, credential, capability and policy controls.
- Endpoint discovery and credential tests are explicit and report their result;
they do not save the draft. Save remains the only committing action.
- Connector/profile disable and managed-file delete remain confirmed actions and
state their immediate effect. Soft deletion must not be described as purge.
- External connector data, paths and credential references are rendered only in
already-authorized administration or explorer contexts. Secret values are
never returned for rendering.
## Accessibility and responsive evidence
Shared `Dialog` owns focus entry, Escape handling and focus return. Form and
toolbar DOM order is the keyboard order; disabled-action tooltips are themselves
focusable and expose the reason. The connector form uses semantic sections and
labels, status is textual as well as colored, and result alerts are announced by
the shared alert component. The explorer collapses to one column below 1050 px;
connector forms and action rows collapse below 760 px while preserving source
order. Long provider choices scroll inside the segmented control rather than
expanding the page.
The focused structural test guards these contracts, optional-module boundaries,
confirmation, contextual help, advanced-only JSON and responsive rules. Core's
TypeScript build, structural localization audit, module-permutation suite and
full-product bundle check provide the integration gates.
+8 -8
View File
@@ -1,6 +1,6 @@
{
"name": "@govoplan/files-webui",
"version": "0.1.9",
"version": "0.1.16",
"private": true,
"type": "module",
"main": "webui/src/index.ts",
@@ -19,14 +19,14 @@
"LICENSE"
],
"peerDependencies": {
"@govoplan/core-webui": "^0.1.9",
"lucide-react": "^1.23.0",
"react": "^19.0.0",
"react-dom": "^19.0.0",
"react-router-dom": "^7.1.1",
"@vitejs/plugin-react": "^4.3.4",
"@vitejs/plugin-react": "^5.2.0",
"vite": "^7.3.6",
"typescript": "^5.7.2",
"vite": "^6.0.6"
"react": ">=19.2.7 <20",
"react-dom": ">=19.2.7 <20",
"react-router": ">=8.3.0 <9",
"lucide-react": "^1.23.0",
"@govoplan/core-webui": "^0.1.16"
},
"peerDependenciesMeta": {
"@govoplan/core-webui": {
+3 -2
View File
@@ -4,15 +4,16 @@ build-backend = "setuptools.build_meta"
[project]
name = "govoplan-files"
version = "0.1.9"
version = "0.1.16"
description = "GovOPlaN files module with backend and WebUI integration."
readme = "README.md"
requires-python = ">=3.12"
license = { file = "LICENSE" }
authors = [{ name = "GovOPlaN" }]
dependencies = [
"govoplan-core>=0.1.9",
"govoplan-core>=0.1.16",
"defusedxml>=0.7,<1",
"pyzipper>=0.3.6,<1",
"python-multipart>=0.0.31,<1",
]
+89 -2
View File
@@ -8,6 +8,11 @@ from sqlalchemy import or_
from govoplan_core.core.access import AccessDecisionProvenance, PrincipalRef
from govoplan_core.core.files import FileAccessProvider
from govoplan_core.core.files import (
ManagedArtifactRef,
ManagedArtifactStore,
ManagedArtifactWriteRequest,
)
from govoplan_core.core.modules import ModuleContext
from govoplan_core.security.module_permissions import scopes_grant_compatible
from govoplan_files.backend.db.models import FileAsset, FileFolder, FileShare
@@ -20,8 +25,13 @@ from govoplan_files.backend.storage.campaign_attachments import (
share_assets_with_campaign,
)
from govoplan_files.backend.storage.campaign_usage import record_campaign_attachment_uses_for_jobs
from govoplan_files.backend.storage.files import current_version_and_blob
from govoplan_files.backend.storage.files import (
create_file_asset,
current_version_and_blob,
sync_file_asset_from_source,
)
from govoplan_files.backend.storage.paths import normalize_folder
from govoplan_files.backend.storage.share_state import effective_file_share_clause
VIRTUAL_FOLDER_RESOURCE_PREFIX = "virtual-folder:v1"
@@ -59,6 +69,83 @@ def campaign_capability(context: ModuleContext) -> FilesCampaignCapability:
return FilesCampaignCapability()
class FilesArtifactStore(ManagedArtifactStore):
def store_artifact(
self,
session: object,
principal: object,
*,
request: ManagedArtifactWriteRequest,
) -> ManagedArtifactRef:
if not hasattr(session, "query") or not hasattr(session, "flush"):
raise TypeError("Files artifact storage requires a SQLAlchemy session.")
if not hasattr(principal, "has") or not principal.has("files:file:upload"):
raise PermissionError("Managed artifact storage requires files:file:upload.")
user = getattr(principal, "user", None)
user_id = str(getattr(user, "id", "") or "")
tenant_id = str(getattr(principal, "tenant_id", "") or "")
if not user_id or not tenant_id:
raise PermissionError("Managed artifact storage requires a tenant user principal.")
metadata = dict(request.metadata)
if request.idempotency_key:
metadata["source_provenance"] = {
"source_type": "generated_artifact",
"connector_id": "files.artifact_store",
"provider": str(metadata.get("producer_module") or "platform"),
"external_id": request.idempotency_key,
"revision": str(metadata.get("output_sha256") or "") or None,
}
stored, _action, _previous_version_id = sync_file_asset_from_source(
session,
tenant_id=tenant_id,
owner_type="user",
owner_id=user_id,
user_id=user_id,
filename=request.filename,
data=request.payload,
metadata=metadata,
folder=request.folder,
content_type=request.content_type,
conflict_strategy="rename",
is_admin=principal.has("files:file:admin"),
)
else:
stored = create_file_asset(
session,
tenant_id=tenant_id,
owner_type="user",
owner_id=user_id,
user_id=user_id,
filename=request.filename,
data=request.payload,
folder=request.folder,
content_type=request.content_type,
description=request.description,
metadata=metadata,
conflict_strategy="rename",
is_admin=principal.has("files:file:admin"),
)
return ManagedArtifactRef(
file_asset_id=stored.asset.id,
file_version_id=stored.version.id,
filename=stored.version.filename_at_upload,
display_path=stored.asset.display_path,
content_type=stored.version.content_type or request.content_type,
size_bytes=stored.version.size_bytes,
sha256=stored.version.checksum_sha256,
provenance={
"module": "files",
"owner_type": stored.asset.owner_type,
"managed": True,
},
)
def artifact_store_capability(context: ModuleContext) -> FilesArtifactStore:
configure_runtime(registry=context.registry, settings=context.settings)
return FilesArtifactStore()
class FilesAccessService(FileAccessProvider):
def explain_resource_provenance(
self,
@@ -96,7 +183,7 @@ class FilesAccessService(FileAccessProvider):
.filter(
FileShare.tenant_id == asset.tenant_id,
FileShare.file_asset_id == asset.id,
FileShare.revoked_at.is_(None),
effective_file_share_clause(),
FileShare.permission.in_(sorted(permission_values)),
or_(
(FileShare.target_type == "user") & (FileShare.target_id == principal.membership_id),
+11 -1
View File
@@ -117,7 +117,15 @@ def _record_share_visibility_change(session: OrmSession, share: FileShare) -> No
return
if not state.pending and not has_attr_changes(
state,
("file_asset_id", "target_type", "target_id", "permission", "revoked_at"),
(
"file_asset_id",
"target_type",
"target_id",
"permission",
"expires_at",
"revoked_at",
"revoked_by_user_id",
),
):
return
_ensure_id(share)
@@ -136,7 +144,9 @@ def _record_share_visibility_change(session: OrmSession, share: FileShare) -> No
"share_target_type": share.target_type,
"share_target_id": share.target_id,
"share_permission": share.permission,
"share_expires_at": _isoformat(share.expires_at),
"share_revoked_at": _isoformat(share.revoked_at),
"share_revoked_by_user_id": share.revoked_by_user_id,
},
)
+66 -1
View File
@@ -16,7 +16,15 @@ def new_uuid() -> str:
class FileBlob(Base, TimestampMixin):
__tablename__ = "file_blobs"
__table_args__ = (UniqueConstraint("tenant_id", "checksum_sha256", "size_bytes", name="uq_file_blobs_tenant_checksum_size"),)
__table_args__ = (
UniqueConstraint(
"tenant_id",
"checksum_sha256",
"size_bytes",
"protection_discriminator",
name="uq_file_blobs_tenant_checksum_size_protection",
),
)
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
@@ -25,9 +33,64 @@ class FileBlob(Base, TimestampMixin):
storage_key: Mapped[str] = mapped_column(String(1000), nullable=False)
checksum_sha256: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
size_bytes: Mapped[int] = mapped_column(Integer, nullable=False)
protection_discriminator: Mapped[str] = mapped_column(String(320), default="plaintext", nullable=False, index=True)
encryption_envelope_id: Mapped[str | None] = mapped_column(String(255), nullable=True, index=True)
storage_checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
storage_size_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True)
content_type: Mapped[str | None] = mapped_column(String(255))
ref_count: Mapped[int] = mapped_column(Integer, default=1, nullable=False)
retained_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
integrity_status: Mapped[str] = mapped_column(String(30), default="unchecked", nullable=False, index=True)
integrity_checked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
integrity_failure: Mapped[str | None] = mapped_column(String(100), nullable=True)
quarantined_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
class FileIntegrityScan(Base, TimestampMixin):
__tablename__ = "file_integrity_scans"
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
storage_backend: Mapped[str] = mapped_column(String(50), nullable=False)
storage_prefix: Mapped[str] = mapped_column(String(1000), nullable=False)
status: Mapped[str] = mapped_column(String(30), default="pending", nullable=False, index=True)
revision: Mapped[int] = mapped_column(Integer, default=1, nullable=False)
phase: Mapped[str] = mapped_column(String(30), default="blobs", nullable=False)
verify_checksums: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False)
batch_size: Mapped[int] = mapped_column(Integer, default=100, nullable=False)
blob_cursor: Mapped[str | None] = mapped_column(String(36), nullable=True)
object_cursor: Mapped[str | None] = mapped_column(String(1000), nullable=True)
scanned_blob_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
verified_blob_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
quarantined_blob_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
scanned_object_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
orphan_object_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
completed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
last_error: Mapped[str | None] = mapped_column(String(255), nullable=True)
class FileIntegrityFinding(Base, TimestampMixin):
__tablename__ = "file_integrity_findings"
__table_args__ = (
Index("ix_file_integrity_findings_scan_state", "scan_id", "state"),
)
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
scan_id: Mapped[str] = mapped_column(ForeignKey("file_integrity_scans.id", ondelete="CASCADE"), nullable=False, index=True)
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
kind: Mapped[str] = mapped_column(String(40), nullable=False, index=True)
state: Mapped[str] = mapped_column(String(30), default="open", nullable=False, index=True)
revision: Mapped[int] = mapped_column(Integer, default=1, nullable=False)
blob_id: Mapped[str | None] = mapped_column(ForeignKey("file_blobs.id", ondelete="SET NULL"), nullable=True, index=True)
storage_key: Mapped[str] = mapped_column(String(1000), nullable=False)
expected_size_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True)
observed_size_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True)
expected_checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
observed_checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
resolved_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
resolved_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
class FileFolder(Base, TimestampMixin):
@@ -105,7 +168,9 @@ class FileShare(Base, TimestampMixin):
target_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
permission: Mapped[str] = mapped_column(String(20), default="read", nullable=False)
created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
revoked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
revoked_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
class FileConnectorProfile(Base, TimestampMixin):
+78 -22
View File
@@ -12,7 +12,7 @@ from govoplan_core.core.modules import (
DocumentationLink,
DocumentationTopic,
)
from govoplan_files.backend.storage.archives import ZIP_UPLOAD_MAX_FILES
from govoplan_files.backend.storage.archives import ARCHIVE_UPLOAD_MAX_ENTRIES
from govoplan_files.backend.storage.access import user_group_ids
from govoplan_files.backend.storage.connector_visibility import (
connector_profile_usable_for_import,
@@ -22,6 +22,9 @@ from govoplan_files.backend.storage.connector_visibility import (
_DEFAULT_UPLOAD_MAX_BYTES = 50 * 1024 * 1024
_DEFAULT_ZIP_MAX_BYTES = 250 * 1024 * 1024
_DEFAULT_ARCHIVE_MAX_EXPANDED_BYTES = 2 * 1024 * 1024 * 1024
_DEFAULT_ARCHIVE_MAX_EXPANSION_RATIO = 100
_DEFAULT_ARCHIVE_PREVIEW_TTL_SECONDS = 30 * 60
_FILES_READ_SCOPE = "files:file:read"
_FILES_UPLOAD_SCOPE = "files:file:upload"
@@ -42,11 +45,47 @@ def documentation_topics(
"file_upload_zip_max_bytes",
default=_DEFAULT_ZIP_MAX_BYTES,
)
archive_expanded_limit = _configured_positive_int(
context.settings,
"file_archive_max_expanded_bytes",
default=_DEFAULT_ARCHIVE_MAX_EXPANDED_BYTES,
)
archive_entry_limit = _configured_positive_int(
context.settings,
"file_archive_max_entries",
default=ARCHIVE_UPLOAD_MAX_ENTRIES,
)
archive_ratio_limit = _configured_positive_int(
context.settings,
"file_archive_max_expansion_ratio",
default=_DEFAULT_ARCHIVE_MAX_EXPANSION_RATIO,
)
archive_preview_ttl = _configured_positive_int(
context.settings,
"file_archive_preview_ttl_seconds",
default=_DEFAULT_ARCHIVE_PREVIEW_TTL_SECONDS,
)
topics: list[DocumentationTopic] = []
if upload_limit is not None:
topics.append(_upload_topic(upload_limit))
if upload_limit is not None and zip_limit is not None:
topics.append(_zip_topic(upload_limit, zip_limit))
if (
upload_limit is not None
and zip_limit is not None
and archive_expanded_limit is not None
and archive_entry_limit is not None
and archive_ratio_limit is not None
and archive_preview_ttl is not None
):
topics.append(
_archive_topic(
upload_limit,
zip_limit,
archive_expanded_limit,
archive_entry_limit,
archive_ratio_limit,
archive_preview_ttl,
)
)
topics.append(_connector_import_topic(context))
return tuple(topics)
@@ -118,19 +157,29 @@ def _upload_topic(max_bytes: int) -> DocumentationTopic:
)
def _zip_topic(max_file_bytes: int, max_zip_bytes: int) -> DocumentationTopic:
def _archive_topic(
max_file_bytes: int,
max_archive_request_bytes: int,
max_expanded_bytes: int,
max_entries: int,
max_expansion_ratio: int,
preview_ttl_seconds: int,
) -> DocumentationTopic:
member_limit = _format_byte_limit(max_file_bytes)
archive_limit = _format_byte_limit(max_zip_bytes)
request_limit = _format_byte_limit(max_archive_request_bytes)
expanded_limit = _format_byte_limit(max_expanded_bytes)
preview_minutes = max(1, preview_ttl_seconds // 60)
return DocumentationTopic(
id="files.workflow.upload-and-unpack-zip",
title="Upload and unpack a ZIP archive",
title="Preview and unpack an archive",
summary=(
f"Safely unpack up to {ZIP_UPLOAD_MAX_FILES:,} files from a ZIP whose request and extracted total are each limited to {archive_limit}."
f"Review and selectively unpack up to {max_entries:,} entries from ZIP or TAR archives before any managed file is created."
),
body=(
f"The current deployment limits the ZIP request and actual extracted total to {archive_limit}, each member to {member_limit}, "
f"and the archive to {ZIP_UPLOAD_MAX_FILES:,} non-directory members. Encrypted archives and unsafe member paths are rejected. "
"Actual extracted bytes are counted instead of trusting ZIP headers."
f"The deployment accepts ZIP, TAR, TAR.GZ, TAR.BZ2, and TAR.XZ requests up to {request_limit}, limits actual expanded data to {expanded_limit}, "
f"each member to {member_limit}, the archive to {max_entries:,} entries, and expansion to {max_expansion_ratio}:1. "
f"The server-issued preview expires after {preview_minutes} minutes. Password-protected ZIP archives are supported; passwords remain request-only. "
"Unsafe paths and special filesystem entries are rejected. Actual extracted bytes are counted instead of trusting archive headers."
),
layer="configured",
documentation_types=("user",),
@@ -161,35 +210,42 @@ def _zip_topic(max_file_bytes: int, max_zip_bytes: int) -> DocumentationTopic:
"help_contexts": ["files.list"],
"prerequisites": [
"You may view and upload managed files.",
"The archive is not encrypted and fits the current configured limits.",
"The archive fits the configured request, expansion, size, and entry limits.",
"The destination personal or group space grants this account write access.",
],
"steps": [
"Open Files and choose the managed destination space and folder.",
"Enable Unpack ZIP uploads, then choose or drag the ZIP archive.",
"Enable Preview and unpack archive, then choose or drag one supported archive.",
"Review the discovered entries, supply a ZIP password when required, and select the files or folders to import.",
"Resolve every destination conflict explicitly.",
"Wait for extraction and finalization to finish before leaving the page.",
"Confirm the selection, then wait for extraction and finalization to finish before leaving the page.",
],
"outcome": "Accepted archive members are stored as separate governed managed assets below the selected folder.",
"verification": "Confirm the expected member paths and inspect representative file sizes, checksums, and versions.",
"constraints": [
{
"id": "zip-request-and-total",
"label": "Maximum ZIP request and extracted total",
"description": f"Both the compressed request and the actual extracted total are limited to {archive_limit}.",
"values": [archive_limit],
"id": "archive-request-and-total",
"label": "Maximum archive request and expanded total",
"description": f"The compressed request is limited to {request_limit}; actual expanded data is limited to {expanded_limit}.",
"values": [request_limit, expanded_limit],
},
{
"id": "zip-member-size",
"id": "archive-member-size",
"label": "Maximum extracted member size",
"description": f"Each extracted file is limited to {member_limit}.",
"values": [member_limit],
},
{
"id": "zip-member-count",
"label": "Maximum file count",
"description": f"A ZIP may contain at most {ZIP_UPLOAD_MAX_FILES:,} non-directory members.",
"values": [f"{ZIP_UPLOAD_MAX_FILES:,} files"],
"id": "archive-entry-count",
"label": "Maximum entry count",
"description": f"An archive may contain at most {max_entries:,} declared entries.",
"values": [f"{max_entries:,} entries"],
},
{
"id": "archive-expansion-ratio",
"label": "Maximum expansion ratio",
"description": f"Declared and actual output may not exceed {max_expansion_ratio} times the compressed request size.",
"values": [f"{max_expansion_ratio}:1"],
},
],
"related_topic_ids": [
+327 -28
View File
@@ -6,13 +6,18 @@ from pathlib import Path
from sqlalchemy import inspect
from govoplan_core.core.access import CAPABILITY_AUTH_PERMISSION_EVALUATOR, CAPABILITY_AUTH_PRINCIPAL_RESOLVER
from govoplan_core.core.files import CAPABILITY_FILES_ACCESS
from govoplan_core.core.encryption import CAPABILITY_ENCRYPTION_CONTENT_CIPHER
from govoplan_core.core.files import (
CAPABILITY_FILES_ACCESS,
CAPABILITY_FILES_ARTIFACT_STORE,
)
from govoplan_core.core.module_guards import drop_table_retirement_provider, persistent_table_uninstall_guard
from govoplan_core.core.modules import (
DocumentationCondition,
DocumentationLink,
DocumentationTopic,
FrontendModule,
FrontendRoute,
MigrationSpec,
ModuleContext,
ModuleInterfaceProvider,
@@ -22,16 +27,33 @@ from govoplan_core.core.modules import (
PermissionDefinition,
RoleTemplate,
)
from govoplan_core.core.operations import OperationalCheckProviderRegistration
from govoplan_core.core.provider_governance import (
ExternalProviderDeclaration,
ExternalProviderStateProviderRegistration,
ProviderBehaviorDeclaration,
ProviderObjectDeclaration,
declared_module_architecture,
)
from govoplan_core.core.search import SearchSourceProviderRegistration
from govoplan_core.core.views import ViewSurface
from govoplan_core.db.base import Base
from govoplan_files.backend.change_tracking import register_files_change_tracking
from govoplan_files.backend.db import models as file_models # noqa: F401 - populate Files ORM metadata
from govoplan_files.backend.documentation import documentation_topics
from govoplan_files.backend.provider_state import (
REMOTE_STORAGE_PROVIDER_ID,
remote_storage_provider_states,
)
from govoplan_files.backend.search_source import create_files_search_source
register_files_change_tracking()
_files_table_retirement_provider = drop_table_retirement_provider(
file_models.FileBlob,
file_models.FileIntegrityScan,
file_models.FileIntegrityFinding,
file_models.FileFolder,
file_models.FileAsset,
file_models.FileVersion,
@@ -98,7 +120,7 @@ PERMISSIONS = (
_permission("files:file:download", "Download files", "Download managed files and generated archives."),
_permission("files:file:upload", "Upload files", "Upload new managed file versions."),
_permission("files:file:organize", "Organize files", "Create folders, rename, move or copy managed files."),
_permission("files:file:share", "Share files", "Grant or update managed file shares; revocation is not available yet."),
_permission("files:file:share", "Share files", "List, grant, update, expire, and revoke managed file shares."),
_permission("files:file:delete", "Delete files", "Delete or hide managed files and folders where policy allows it."),
_permission("files:file:admin", "Administer file spaces", "Administer all file spaces in the tenant."),
)
@@ -138,6 +160,43 @@ def _tenant_summary(session, tenant_id: str) -> dict[str, int]:
}
def _tenant_summary_batch(session, tenant_ids) -> dict[str, dict[str, int]]:
from sqlalchemy import func
from govoplan_files.backend.db.models import FileAsset, FileConnectorCredential, FileConnectorPolicy, FileConnectorProfile, FileConnectorSpace
ids = tuple(dict.fromkeys(str(tenant_id) for tenant_id in tenant_ids if tenant_id))
if not ids:
return {}
counts: dict[str, dict[str, int]] = {
tenant_id: {
"files": 0,
"connector_credentials": 0,
"connector_policies": 0,
"connector_profiles": 0,
"connector_spaces": 0,
}
for tenant_id in ids
}
models = (
("files", FileAsset),
("connector_credentials", FileConnectorCredential),
("connector_policies", FileConnectorPolicy),
("connector_profiles", FileConnectorProfile),
("connector_spaces", FileConnectorSpace),
)
for count_key, model in models:
rows = (
session.query(model.tenant_id, func.count(model.id))
.filter(model.tenant_id.in_(ids))
.group_by(model.tenant_id)
.all()
)
for tenant_id, count in rows:
counts[tenant_id][count_key] = int(count)
return counts
def _veto_group_delete(session, tenant_id: str, group_id: str) -> None:
from govoplan_files.backend.db.models import FileAsset, FileConnectorSpace, FileFolder, FileShare
@@ -165,15 +224,68 @@ def _files_router(context: ModuleContext):
return router
REMOTE_STORAGE_PROVIDER = ExternalProviderDeclaration(
id=REMOTE_STORAGE_PROVIDER_ID,
module_id="files",
label="Remote file storage mirror",
maturity="synchronize",
operations=("discover", "search", "read", "synchronize", "preview"),
objects=(
ProviderObjectDeclaration(
object_type="remote_folder",
field_groups=("identity", "hierarchy", "display", "source_metadata"),
authority_modes=("external_authoritative", "external_mirror"),
default_authority_mode="external_mirror",
),
ProviderObjectDeclaration(
object_type="remote_file",
field_groups=("identity", "content", "version", "source_metadata"),
authority_modes=("external_authoritative", "external_mirror"),
default_authority_mode="external_mirror",
),
),
behavior=ProviderBehaviorDeclaration(
revision_tokens="Remote path, provider revision, size, and content digest are retained on managed imports.",
concurrency="A sync compares the frozen source reference and revision before creating a new managed version.",
freshness="Provider state distinguishes software availability from an unobserved live remote binding.",
health="Unsupported providers or missing optional transports fail closed; live health remains unknown without a probe.",
max_read_items=5000,
idempotency="Source profile, remote object reference, and revision/digest suppress duplicate managed versions.",
retry="Operators repeat bounded browse/import after a classified transport failure; effects are not blindly retried.",
timeout_seconds=30,
conflicts="Managed-file conflict policy is explicit; the current provider never mutates the remote source.",
outcome_unknown="An interrupted download is discarded unless its complete digest and managed version commit are confirmed.",
outcome_unknown_supported=True,
evidence="Managed versions retain connector profile, remote object identity, source revision, digest, and acquisition time.",
audit_event_types=(
"files.connector.accessed",
"files.connector.imported",
"files.connector.synced",
),
correction="A later acquisition creates a new managed version and preserves prior provenance.",
rollback="Remote reads require no remote rollback; incomplete local objects are reconciled as orphans.",
compensation="A wrongly imported managed version can be retired under Files policy without deleting the source.",
reconciliation="Re-read source metadata and digest, then compare the committed managed version and object-store inventory.",
outage="Previously imported managed versions remain available while remote spaces report unknown or stale state.",
classifications=("internal", "confidential", "personal"),
purposes=("governed file acquisition", "managed evidence snapshot"),
retention="Files retention applies to managed versions; external retention remains provider-owned.",
secret_handling="Credentials remain encrypted or deployment-owned and never appear in provider state or provenance.",
),
documentation_topic_ids=("files.governed-connectors-and-provenance",),
)
manifest = ModuleManifest(
id="files",
name="Files",
version="0.1.9",
version="0.1.16",
required_capabilities=(CAPABILITY_AUTH_PRINCIPAL_RESOLVER, CAPABILITY_AUTH_PERMISSION_EVALUATOR),
optional_dependencies=("campaigns",),
optional_dependencies=("campaigns", "encryption", "search"),
provides_interfaces=(
ModuleInterfaceProvider(name="files.access", version="0.1.6"),
ModuleInterfaceProvider(name="files.campaign_attachments", version="0.1.6"),
ModuleInterfaceProvider(name=CAPABILITY_FILES_ARTIFACT_STORE, version="0.1.14"),
),
requires_interfaces=(
ModuleInterfaceRequirement(
@@ -182,19 +294,89 @@ manifest = ModuleManifest(
version_max_exclusive="0.2.0",
optional=True,
),
ModuleInterfaceRequirement(
name=CAPABILITY_ENCRYPTION_CONTENT_CIPHER,
version_min="1.0.0",
version_max_exclusive="2.0.0",
optional=True,
),
ModuleInterfaceRequirement(
name="search.source",
version_min="1.0.0",
version_max_exclusive="2.0.0",
optional=True,
),
),
permissions=PERMISSIONS,
route_factory=_files_router,
role_templates=ROLE_TEMPLATES,
tenant_summary_providers=(_tenant_summary,),
tenant_summary_batch_providers=(_tenant_summary_batch,),
search_sources=(
SearchSourceProviderRegistration(
id="files.objects",
factory=create_files_search_source,
),
),
delete_veto_providers={"group": (_veto_group_delete,)},
nav_items=(NavItem(path="/files", label="Files", icon="folder", required_any=("files:file:read",), order=40),),
frontend=FrontendModule(
module_id="files",
package_name="@govoplan/files-webui",
routes=(
FrontendRoute(
path="/files",
component="FilesPage",
required_any=("files:file:read",),
order=40,
),
),
nav_items=(NavItem(path="/files", label="Files", icon="folder", required_any=("files:file:read",), order=40),),
view_surfaces=(
ViewSurface(id="files.admin.system-connectors", module_id="files", kind="section", label="System file connections", order=75),
ViewSurface(id="files.admin.tenant-connectors", module_id="files", kind="section", label="Tenant file connections", order=65),
ViewSurface(id="files.admin.tenant-integrity", module_id="files", kind="section", label="File integrity", order=66),
ViewSurface(id="files.admin.group-connectors", module_id="files", kind="section", label="Group file connections", order=65),
ViewSurface(id="files.admin.user-connectors", module_id="files", kind="section", label="User file connections", order=65),
ViewSurface(id="files.settings.connectors", module_id="files", kind="section", label="Personal file connections", order=20),
ViewSurface(id="files.widget.spaces", module_id="files", kind="section", label="File spaces widget", order=35),
),
),
documentation=(
DocumentationTopic(
id="files.search.managed-content",
title="Search managed files and folders",
summary="Expose file names, logical paths, and descriptions to permission-aware platform Search.",
body=(
"When Search is installed, Files contributes managed files and folders to its derived index. "
"Every result is tenant-bounded and rechecks current ownership, group membership, direct shares, "
"expiry, revocation, deletion, and Files permissions before it is returned. Committed file and "
"share changes are delivered through the platform event outbox; an administrator can rebuild the "
"derived index without changing authoritative Files data."
),
layer="configured",
documentation_types=("admin", "user"),
audience=("file_user", "file_manager", "administrator"),
related_modules=("search",),
order=41,
conditions=(
DocumentationCondition(
required_modules=("files",),
any_scopes=("files:file:read", "files:file:admin"),
),
),
links=(
DocumentationLink(label="Files", href="/files", kind="runtime"),
DocumentationLink(label="Search", href="/search", kind="runtime"),
DocumentationLink(label="Files handbook", href="govoplan-files/docs/FILES_HANDBOOK.md", kind="repository"),
),
metadata={
"kind": "reference",
"route": "/files",
"screen": "Files search contribution",
"help_contexts": ["files.list"],
},
),
DocumentationTopic(
id="files.workflow.organize-managed-files",
title="Organize managed files and folders",
@@ -288,11 +470,11 @@ manifest = ModuleManifest(
),
DocumentationTopic(
id="files.workflow.share-managed-files",
title="Grant or update access to managed files",
summary="Grant or update read, write, or manage access through a supporting workflow or API client without changing ownership.",
title="Manage access to managed files",
summary="List, grant, update, expire, or revoke direct read, write, and manage access without changing ownership.",
body=(
"The Files service can grant or update a share for a user, group, tenant, or campaign. The Files page does not yet provide a general share editor, and the API has no share-revocation route. "
"Do not use this task when access must later be revoked until that explicit capability is implemented."
"File owners and file administrators can manage direct shares for users, groups, the tenant, and Campaign. Expired and revoked grants stop authorizing access immediately while independent active grants remain effective. "
"The Files share dialog lists active and historical grants, and revocation is idempotent."
),
layer="available",
documentation_types=("user",),
@@ -306,7 +488,7 @@ manifest = ModuleManifest(
),
links=(
DocumentationLink(label="Files", href="/files", kind="runtime"),
DocumentationLink(label="Grant or update a share", href="/api/v1/files/{file_id}/shares", kind="api"),
DocumentationLink(label="List and manage shares", href="/api/v1/files/{file_id}/shares", kind="api"),
DocumentationLink(label="Files handbook", href="govoplan-files/docs/FILES_HANDBOOK.md", kind="repository"),
),
related_modules=("campaigns",),
@@ -317,22 +499,21 @@ manifest = ModuleManifest(
"screen": "Files",
"help_contexts": ["files.list"],
"prerequisites": [
"You may view and share the managed file and have write/manage access to that specific asset; the global share permission alone does not grant resource access.",
"A supporting workflow or API client is available; the Files page has no general share editor yet.",
"The process does not require share revocation, because no revocation route exists yet.",
"You have the Files share permission and own the file, or administer file spaces for the active tenant.",
"The intended user, group, tenant, or Campaign target exists and is active.",
],
"steps": [
"Open Files and verify the file owner, logical path, current version, and checksum.",
"Through the supporting workflow, identify the intended user, group, tenant, or campaign and choose read, write, or manage access.",
"Select one managed file, choose Manage shares, and review the effective and historical direct grants.",
"Select the intended user, group, or tenant and choose read, write, or manage access plus an optional expiry.",
"Grant or update the share without changing file ownership.",
"Have the intended recipient verify access and an unrelated account verify denial.",
"Revoke a grant when it is no longer needed; repeated revocation is a no-op.",
],
"limitations": [
"The Files page has no general share editor.",
"Shares can be granted or updated, but not revoked through the current API.",
"Campaign-target grants are normally created by the Campaign integration rather than selected manually in the Files dialog.",
"A user may retain access through another active direct grant or ownership path after one share is revoked.",
],
"outcome": "The requested access is granted or updated while the managed asset keeps its owner.",
"verification": "Test one intended and one denied access path. Do not claim that the share can later be revoked.",
"outcome": "Direct access has the requested permission and lifetime while the managed asset keeps its owner.",
"verification": "Test one intended and one denied path, then expire or revoke the grant and verify that only independent access paths remain.",
"related_topic_ids": [
"files.workflow.find-and-download-files",
"files.assurance.process-and-release-readiness",
@@ -434,6 +615,11 @@ manifest = ModuleManifest(
"route": "/admin?section=tenant-file-connectors",
"screen": "File connections",
"section": "Profiles, credentials, and effective connector policy",
"help_contexts": [
"files.connectors",
"files.connector.credentials",
"files.connector.policy",
],
"security_invariants": [
"New API-managed external secret references fail closed until Files can prove ownership and provider-side deletion.",
"Deletion and destructive retirement scrub Files-owned encrypted connector material before completion and emit non-secret audit evidence.",
@@ -449,10 +635,10 @@ manifest = ModuleManifest(
DocumentationTopic(
id="files.reference.integrity-recovery-and-fail-closed-transports",
title="Operate Files integrity, recovery, and connector transport safety",
summary="Back up database evidence, blob bytes, and the encryption key as one recovery unit, and keep unsupported SDK transports fail-closed.",
summary="Back up database evidence, blob ciphertext, and Encryption custody as one recovery unit, and pin every SDK-managed connector peer.",
body=(
"Local durable storage is the operational baseline. Recover Files from a coordinated database/blob snapshot with the original master key, then verify representative checksums and access paths. "
"Live S3 managed storage/connectors and SMB connectors fail closed until botocore redirects/endpoint discovery and SMB initial connections/DFS referrals support connection-time DNS/IP pinning. Destructive module retirement drops database tables but does not remove backend blob objects."
"Local durable storage is the operational baseline. Recover Files from a coordinated database/blob snapshot with the matching Encryption tables and original deployment master key, then run the bounded resumable integrity scan from Administration and verify representative protected and unprotected access paths. Each scan batch and finding action requires the revision shown to the operator, so a stale screen cannot recheck or delete after concurrent reconciliation. Protected scans verify stored ciphertext before decryption and then verify plaintext semantic evidence. Managed blob creation/repair and applied orphan cleanup commit lease-fenced Core recovery intent before object effects; success, compensation, and forward completion require independent database and object checks, while mismatch is quarantined and unresolved work remains visible in Ops. Missing or mismatched blobs are quarantined; orphan objects are reported before dry-run-first, explicitly authorized cleanup. "
"S3 connector pools pin every retry, redirect, discovered endpoint, and provider alias while retaining the configured TLS authority; outbound proxies and ambient credential discovery are disabled. SMB initial connections, reconnects, aliases, and DFS referrals use a Files-owned pinned transport and cache. Both apply the deployment private-network policy immediately before each socket opens and fail closed if an SDK no longer exposes the verified transport seam. Installer-owned Garage storage is supported only at the exact deployment service endpoint with its explicit trust marker. Destructive module retirement drops database tables but does not remove backend blob objects."
),
layer="configured",
documentation_types=("admin",),
@@ -471,17 +657,24 @@ manifest = ModuleManifest(
),
links=(
DocumentationLink(label="System file connections", href="/admin?section=system-file-connectors", kind="runtime"),
DocumentationLink(label="File integrity operations", href="/admin?section=tenant-file-integrity", kind="runtime"),
DocumentationLink(label="Connector provider status", href="/api/v1/files/connectors/providers", kind="api"),
DocumentationLink(label="Create an integrity scan", href="/api/v1/files/integrity/scans", kind="api"),
DocumentationLink(label="Files handbook", href="govoplan-files/docs/FILES_HANDBOOK.md", kind="repository"),
),
related_modules=("audit", "ops"),
related_modules=("audit", "encryption", "ops"),
unlocks=("Recoverable managed-file evidence without weakening outbound peer validation.",),
configuration_keys=(
"FILE_STORAGE_BACKEND",
"FILE_STORAGE_LOCAL_ROOT",
"FILE_STORAGE_LOCAL_FALLBACK_ROOTS",
"FILE_STORAGE_S3_DEPLOYMENT_MANAGED",
"FILE_UPLOAD_MAX_BYTES",
"FILE_UPLOAD_ZIP_MAX_BYTES",
"FILE_ARCHIVE_MAX_ENTRIES",
"FILE_ARCHIVE_MAX_EXPANDED_BYTES",
"FILE_ARCHIVE_MAX_EXPANSION_RATIO",
"FILE_ARCHIVE_PREVIEW_TTL_SECONDS",
"MASTER_KEY_B64",
"GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS",
"GOVOPLAN_CONNECTOR_MAX_STRUCTURED_RESPONSE_BYTES",
@@ -492,8 +685,8 @@ manifest = ModuleManifest(
"route": "/admin?section=system-file-connectors",
"screen": "System file connections and deployment operations",
"section": "Storage integrity, backup/recovery, and fail-closed transports",
"recovery_unit": ["Files database rows", "managed blob namespace", "MASTER_KEY_B64", "deployment-owned connector configuration"],
"verification": "After restore, download representative files, compare their bytes with recorded SHA-256 values, verify authorized and denied access, and test one permitted pinned HTTP connector.",
"recovery_unit": ["Files database rows", "Encryption envelope and wrapped-key rows", "managed blob namespace", "MASTER_KEY_B64", "deployment-owned connector configuration"],
"verification": "After restore, complete a checksum-enabled integrity scan, resolve every missing/corrupt finding, approve or retain every reported orphan, inspect Files recovery operations in Ops, verify authorized and denied access, and test configured pinned HTTP, S3, and SMB connectors against their recorded target topology.",
"related_topic_ids": [
"files.governed-connectors-and-provenance",
"files.reference.snapshot-provenance-and-capabilities",
@@ -545,7 +738,7 @@ manifest = ModuleManifest(
summary="Check a process against the implemented Files boundary, exercise permitted and denied paths, and retain evidence before approving a release or operational use.",
body=(
"A process owner must distinguish implemented controls from planned capabilities before relying on Files. A release is not ready until all package and manifest versions align and representative authorization, upload limits, conflict handling, download, deletion, connector, and recovery paths have been exercised. "
"Current limitations include no general share-management UI or share revocation, no self-service restore or hard purge, no enforced retention or legal hold, and no dedicated canonical audit event for every ordinary Files mutation. Record these limitations in the process assessment instead of treating soft deletion or change-sequence entries as stronger evidence."
"Current limitations include no self-service restore or hard purge, no enforced retention or legal hold, and no dedicated canonical audit event for every ordinary Files mutation. Share grant, change, expiry, and revocation operations do emit dedicated audit records. Record remaining limitations in the process assessment instead of treating soft deletion or change-sequence entries as stronger evidence."
),
layer="configured",
documentation_types=("admin", "user"),
@@ -576,6 +769,10 @@ manifest = ModuleManifest(
"FILE_STORAGE_LOCAL_ROOT",
"FILE_UPLOAD_MAX_BYTES",
"FILE_UPLOAD_ZIP_MAX_BYTES",
"FILE_ARCHIVE_MAX_ENTRIES",
"FILE_ARCHIVE_MAX_EXPANDED_BYTES",
"FILE_ARCHIVE_MAX_EXPANSION_RATIO",
"FILE_ARCHIVE_PREVIEW_TTL_SECONDS",
"MASTER_KEY_B64",
"GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS",
),
@@ -593,8 +790,8 @@ manifest = ModuleManifest(
"Compare the process requirements with the handbook's implemented/planned boundary and record every unsupported requirement or compensating control.",
"Confirm version alignment, apply migrations on clean and upgrade databases, and run the repository plus meta-repository security and static-analysis gates.",
"Exercise allowed and denied personal, group, and share access with representative accounts.",
"Exercise bounded upload and ZIP handling, every required conflict strategy, organization, download, and soft deletion.",
"Where connectors are configured, verify policy explanation and one pinned HTTP provider; verify live S3 and SMB access still fails closed.",
"Exercise bounded archive preview and confirmation, every required conflict strategy, organization, download, and soft deletion.",
"Where connectors are configured, verify policy explanation and one pinned HTTP provider; verify installer-owned Garage when selected, then exercise configured S3 retries/aliases and SMB reconnect/referral targets under the deployment private-network policy, confirming an incompatible SDK transport seam fails closed.",
"Restore a coordinated database/blob/key backup and compare representative downloaded bytes with their recorded SHA-256 checksums.",
"Record the tested versions, results, known limitations, evidence locations, residual risks, owner, and approval decision.",
],
@@ -614,6 +811,69 @@ manifest = ModuleManifest(
],
},
),
DocumentationTopic(
id="files.reference.shared-storage-profile",
title="Operate Files with shared object storage",
summary="Choose local, host-shared, or S3-backed storage consistently with the runtime topology.",
body="Core supplies the common local/S3 object-storage backend while Files owns file metadata and object-key semantics. Local storage is valid for one runtime process; a shared host volume supports same-host replicas; independent hosts require an explicitly trusted HTTPS S3-compatible endpoint. Restore PostgreSQL, objects, and the master key to one coordinated recovery point.",
layer="configured",
documentation_types=("admin",),
audience=("file_admin", "operator", "system_admin"),
order=54,
conditions=(
DocumentationCondition(
required_modules=("files",),
any_scopes=("files:file:admin", "system:settings:read"),
),
),
links=(
DocumentationLink(label="Files", href="/files", kind="runtime"),
DocumentationLink(label="Files handbook", href="govoplan-files/docs/FILES_HANDBOOK.md", kind="repository"),
),
related_modules=("ops", "campaigns"),
configuration_keys=(
"GOVOPLAN_STATE_PROFILE",
"GOVOPLAN_INSTALLATION_ID",
"FILE_STORAGE_BACKEND",
"FILE_STORAGE_LOCAL_ROOT",
"FILE_STORAGE_S3_ENDPOINT_URL",
"FILE_STORAGE_S3_ENDPOINT_TRUSTED",
"FILE_STORAGE_S3_DEPLOYMENT_MANAGED",
),
metadata={
"kind": "reference",
"route": "/ops",
"screen": "Storage and runtime posture",
"limitations": [
"Installer-managed Garage is single-node unless an external multi-node cluster is operated separately.",
"The application does not create or verify production PostgreSQL/object/key backups.",
],
"verification": "Run the Files storage round-trip check and a coordinated restore drill against the exact deployment topology.",
},
),
DocumentationTopic(
id="files.reference.generated-artifact-store",
title="Store generated module artifacts",
summary="Let optional producer modules persist generated output through the Files authority boundary.",
body="The files.artifact_store capability accepts generated bytes plus bounded non-secret provenance, applies Files upload authorization, ownership, path, version, and blob-storage rules, and returns provider-neutral file/version references. Idempotency uses source provenance. Artifact acceptance does not prove printing, mailing, or another external effect.",
layer="available",
documentation_types=("admin", "user"),
audience=("file_admin", "operator", "module_admin", "integrator"),
order=55,
conditions=(
DocumentationCondition(
required_modules=("files",),
any_scopes=("files:file:upload", "files:file:admin"),
),
),
links=(
DocumentationLink(label="Files", href="/files", kind="runtime"),
DocumentationLink(label="Files handbook", href="govoplan-files/docs/FILES_HANDBOOK.md", kind="repository"),
DocumentationLink(label="Generated artifact contract", href="govoplan-files/docs/GENERATED_ARTIFACT_STORE.md", kind="repository"),
),
related_modules=("templates", "campaigns", "reporting"),
metadata={"kind": "reference", "route": "/files"},
),
),
documentation_providers=(documentation_topics,),
migration_spec=MigrationSpec(
@@ -641,8 +901,47 @@ manifest = ModuleManifest(
),
capability_factories={
CAPABILITY_FILES_ACCESS: lambda context: __import__("govoplan_files.backend.capabilities", fromlist=["access_capability"]).access_capability(context),
CAPABILITY_FILES_ARTIFACT_STORE: lambda context: __import__("govoplan_files.backend.capabilities", fromlist=["artifact_store_capability"]).artifact_store_capability(context),
"files.campaign_attachments": lambda context: __import__("govoplan_files.backend.capabilities", fromlist=["campaign_capability"]).campaign_capability(context),
},
operational_check_providers=(
OperationalCheckProviderRegistration(
module_id="files",
check_id="files.managed_storage_roundtrip",
provider=lambda: __import__(
"govoplan_files.backend.operational_checks",
fromlist=["managed_storage_roundtrip_check"],
).managed_storage_roundtrip_check(),
cache_seconds=60,
),
),
external_providers=(REMOTE_STORAGE_PROVIDER,),
external_provider_state_providers=(
ExternalProviderStateProviderRegistration(
module_id="files",
provider_id=REMOTE_STORAGE_PROVIDER_ID,
provider=remote_storage_provider_states,
),
),
architecture=declared_module_architecture(
layer="content_records_evidence",
kind="domain",
maturity="vertical_slice",
documentation_ref="docs/FILES_HANDBOOK.md",
test_ref="tests/test_storage_backends.py",
known_limits=("Target-environment multi-node recovery drills and writable remote connector effects are not reference-ready; hard purge and legal hold remain unimplemented.",),
supported_authority_modes=(
"native_authoritative",
"external_authoritative",
"external_mirror",
),
owned_concepts=("file asset", "file version", "folder", "share", "connector space"),
non_owned_concepts=("record disposition", "campaign attachment rule", "external storage object"),
target_tested_providers=(REMOTE_STORAGE_PROVIDER_ID,),
recovery_docs=("docs/FILES_HANDBOOK.md",),
security_docs=("docs/CONNECTOR_BOUNDARY.md",),
operations_docs=("docs/FILES_HANDBOOK.md",),
),
)
@@ -0,0 +1,53 @@
"""file share lifecycle
Revision ID: b8c9d0e1f2a4
Revises: a7b8c9d0e1f3
Create Date: 2026-07-30 00:00:00.000000
"""
from __future__ import annotations
from alembic import op
import sqlalchemy as sa
revision = "b8c9d0e1f2a4"
down_revision = "a7b8c9d0e1f3"
branch_labels = None
depends_on = None
def upgrade() -> None:
with op.batch_alter_table("file_shares") as batch_op:
batch_op.add_column(
sa.Column("expires_at", sa.DateTime(timezone=True), nullable=True)
)
batch_op.add_column(
sa.Column("revoked_by_user_id", sa.String(length=36), nullable=True)
)
batch_op.create_foreign_key(
op.f("fk_file_shares_revoked_by_user_id_access_users"),
"access_users",
["revoked_by_user_id"],
["id"],
ondelete="SET NULL",
)
batch_op.create_index(
op.f("ix_file_shares_expires_at"), ["expires_at"], unique=False
)
batch_op.create_index(
op.f("ix_file_shares_revoked_by_user_id"),
["revoked_by_user_id"],
unique=False,
)
def downgrade() -> None:
with op.batch_alter_table("file_shares") as batch_op:
batch_op.drop_index(op.f("ix_file_shares_revoked_by_user_id"))
batch_op.drop_index(op.f("ix_file_shares_expires_at"))
batch_op.drop_constraint(
op.f("fk_file_shares_revoked_by_user_id_access_users"),
type_="foreignkey",
)
batch_op.drop_column("revoked_by_user_id")
batch_op.drop_column("expires_at")
@@ -0,0 +1,168 @@
"""file integrity reconciliation
Revision ID: c9d0e1f2a3b5
Revises: b8c9d0e1f2a4
Create Date: 2026-07-30 00:00:00.000000
"""
from __future__ import annotations
from alembic import op
import sqlalchemy as sa
revision = "c9d0e1f2a3b5"
down_revision = "b8c9d0e1f2a4"
branch_labels = None
depends_on = None
def upgrade() -> None:
with op.batch_alter_table("file_blobs") as batch_op:
batch_op.add_column(
sa.Column(
"integrity_status",
sa.String(length=30),
nullable=False,
server_default="unchecked",
)
)
batch_op.add_column(
sa.Column("integrity_checked_at", sa.DateTime(timezone=True), nullable=True)
)
batch_op.add_column(
sa.Column("integrity_failure", sa.String(length=100), nullable=True)
)
batch_op.add_column(
sa.Column("quarantined_at", sa.DateTime(timezone=True), nullable=True)
)
batch_op.create_index(
op.f("ix_file_blobs_integrity_status"),
["integrity_status"],
unique=False,
)
batch_op.create_index(
op.f("ix_file_blobs_integrity_checked_at"),
["integrity_checked_at"],
unique=False,
)
batch_op.create_index(
op.f("ix_file_blobs_quarantined_at"),
["quarantined_at"],
unique=False,
)
op.create_table(
"file_integrity_scans",
sa.Column("id", sa.String(length=36), nullable=False),
sa.Column("tenant_id", sa.String(length=36), nullable=False),
sa.Column("storage_backend", sa.String(length=50), nullable=False),
sa.Column("storage_prefix", sa.String(length=1000), nullable=False),
sa.Column("status", sa.String(length=30), nullable=False),
sa.Column("phase", sa.String(length=30), nullable=False),
sa.Column("verify_checksums", sa.Boolean(), nullable=False),
sa.Column("batch_size", sa.Integer(), nullable=False),
sa.Column("blob_cursor", sa.String(length=36), nullable=True),
sa.Column("object_cursor", sa.String(length=1000), nullable=True),
sa.Column("scanned_blob_count", sa.Integer(), nullable=False),
sa.Column("verified_blob_count", sa.Integer(), nullable=False),
sa.Column("quarantined_blob_count", sa.Integer(), nullable=False),
sa.Column("scanned_object_count", sa.Integer(), nullable=False),
sa.Column("orphan_object_count", sa.Integer(), nullable=False),
sa.Column("created_by_user_id", sa.String(length=36), nullable=True),
sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("last_error", sa.String(length=255), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.ForeignKeyConstraint(
["created_by_user_id"],
["access_users.id"],
name=op.f(
"fk_file_integrity_scans_created_by_user_id_access_users"
),
ondelete="SET NULL",
),
sa.PrimaryKeyConstraint("id", name=op.f("pk_file_integrity_scans")),
)
for column in ("tenant_id", "status", "created_by_user_id"):
op.create_index(
op.f(f"ix_file_integrity_scans_{column}"),
"file_integrity_scans",
[column],
unique=False,
)
op.create_table(
"file_integrity_findings",
sa.Column("id", sa.String(length=36), nullable=False),
sa.Column("scan_id", sa.String(length=36), nullable=False),
sa.Column("tenant_id", sa.String(length=36), nullable=False),
sa.Column("kind", sa.String(length=40), nullable=False),
sa.Column("state", sa.String(length=30), nullable=False),
sa.Column("blob_id", sa.String(length=36), nullable=True),
sa.Column("storage_key", sa.String(length=1000), nullable=False),
sa.Column("expected_size_bytes", sa.Integer(), nullable=True),
sa.Column("observed_size_bytes", sa.Integer(), nullable=True),
sa.Column("expected_checksum_sha256", sa.String(length=64), nullable=True),
sa.Column("observed_checksum_sha256", sa.String(length=64), nullable=True),
sa.Column("resolved_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("resolved_by_user_id", sa.String(length=36), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.ForeignKeyConstraint(
["blob_id"],
["file_blobs.id"],
name=op.f("fk_file_integrity_findings_blob_id_file_blobs"),
ondelete="SET NULL",
),
sa.ForeignKeyConstraint(
["resolved_by_user_id"],
["access_users.id"],
name=op.f(
"fk_file_integrity_findings_resolved_by_user_id_access_users"
),
ondelete="SET NULL",
),
sa.ForeignKeyConstraint(
["scan_id"],
["file_integrity_scans.id"],
name=op.f(
"fk_file_integrity_findings_scan_id_file_integrity_scans"
),
ondelete="CASCADE",
),
sa.PrimaryKeyConstraint("id", name=op.f("pk_file_integrity_findings")),
)
for column in (
"scan_id",
"tenant_id",
"kind",
"state",
"blob_id",
"resolved_by_user_id",
):
op.create_index(
op.f(f"ix_file_integrity_findings_{column}"),
"file_integrity_findings",
[column],
unique=False,
)
op.create_index(
"ix_file_integrity_findings_scan_state",
"file_integrity_findings",
["scan_id", "state"],
unique=False,
)
def downgrade() -> None:
op.drop_table("file_integrity_findings")
op.drop_table("file_integrity_scans")
with op.batch_alter_table("file_blobs") as batch_op:
batch_op.drop_index(op.f("ix_file_blobs_quarantined_at"))
batch_op.drop_index(op.f("ix_file_blobs_integrity_checked_at"))
batch_op.drop_index(op.f("ix_file_blobs_integrity_status"))
batch_op.drop_column("quarantined_at")
batch_op.drop_column("integrity_failure")
batch_op.drop_column("integrity_checked_at")
batch_op.drop_column("integrity_status")
@@ -0,0 +1,79 @@
"""file content-protection metadata
Revision ID: d0e1f2a3b4c6
Revises: c9d0e1f2a3b5
Create Date: 2026-08-02 00:00:00.000000
"""
from __future__ import annotations
from alembic import op
import sqlalchemy as sa
revision = "d0e1f2a3b4c6"
down_revision = "c9d0e1f2a3b5"
branch_labels = None
depends_on = None
def upgrade() -> None:
with op.batch_alter_table("file_blobs") as batch_op:
batch_op.add_column(
sa.Column(
"protection_discriminator",
sa.String(length=320),
nullable=False,
server_default="plaintext",
)
)
batch_op.add_column(
sa.Column("encryption_envelope_id", sa.String(length=255), nullable=True)
)
batch_op.add_column(
sa.Column("storage_checksum_sha256", sa.String(length=64), nullable=True)
)
batch_op.add_column(
sa.Column("storage_size_bytes", sa.Integer(), nullable=True)
)
batch_op.drop_constraint(
"uq_file_blobs_tenant_checksum_size",
type_="unique",
)
batch_op.create_unique_constraint(
"uq_file_blobs_tenant_checksum_size_protection",
[
"tenant_id",
"checksum_sha256",
"size_bytes",
"protection_discriminator",
],
)
batch_op.create_index(
op.f("ix_file_blobs_protection_discriminator"),
["protection_discriminator"],
unique=False,
)
batch_op.create_index(
op.f("ix_file_blobs_encryption_envelope_id"),
["encryption_envelope_id"],
unique=False,
)
def downgrade() -> None:
with op.batch_alter_table("file_blobs") as batch_op:
batch_op.drop_index(op.f("ix_file_blobs_encryption_envelope_id"))
batch_op.drop_index(op.f("ix_file_blobs_protection_discriminator"))
batch_op.drop_constraint(
"uq_file_blobs_tenant_checksum_size_protection",
type_="unique",
)
batch_op.create_unique_constraint(
"uq_file_blobs_tenant_checksum_size",
["tenant_id", "checksum_sha256", "size_bytes"],
)
batch_op.drop_column("storage_size_bytes")
batch_op.drop_column("storage_checksum_sha256")
batch_op.drop_column("encryption_envelope_id")
batch_op.drop_column("protection_discriminator")
@@ -0,0 +1,34 @@
"""add stale-action revisions to Files integrity operations
Revision ID: f1a2b3c4d5e7
Revises: d0e1f2a3b4c6
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
revision = "f1a2b3c4d5e7"
down_revision = "d0e1f2a3b4c6"
branch_labels = None
depends_on = None
def upgrade() -> None:
with op.batch_alter_table("file_integrity_scans") as batch_op:
batch_op.add_column(
sa.Column("revision", sa.Integer(), nullable=False, server_default="1")
)
with op.batch_alter_table("file_integrity_findings") as batch_op:
batch_op.add_column(
sa.Column("revision", sa.Integer(), nullable=False, server_default="1")
)
def downgrade() -> None:
with op.batch_alter_table("file_integrity_findings") as batch_op:
batch_op.drop_column("revision")
with op.batch_alter_table("file_integrity_scans") as batch_op:
batch_op.drop_column("revision")
@@ -0,0 +1,67 @@
from __future__ import annotations
import hashlib
import secrets
from uuid import uuid4
from govoplan_core.core.operations import OperationalCheck
from govoplan_files.backend.storage.backends import (
StorageBackendError,
get_storage_backend,
)
def managed_storage_roundtrip_check() -> OperationalCheck:
"""Exercise the configured managed store without retaining probe data."""
backend = get_storage_backend()
key = f".govoplan-health/probes/{uuid4().hex}.bin"
payload = secrets.token_bytes(64)
expected_digest = hashlib.sha256(payload).hexdigest()
delete_error: Exception | None = None
try:
backend.put_bytes(key, payload, content_type="application/octet-stream")
stored = backend.get_bytes(key)
info = backend.stat(key)
if info.size_bytes != len(payload):
raise StorageBackendError("Managed storage returned an unexpected object size")
if hashlib.sha256(stored).hexdigest() != expected_digest:
raise StorageBackendError("Managed storage returned different bytes than were written")
except Exception as exc: # noqa: BLE001 - operational boundary reports provider failures.
return OperationalCheck(
id="files.managed_storage_roundtrip",
label="Managed file storage",
state="error",
detail=(
"The configured managed file store failed a bounded write/read/stat/delete "
f"probe ({type(exc).__name__})."
),
readiness_critical=True,
metrics={"backend": backend.name, "probe_bytes": len(payload)},
)
finally:
try:
backend.delete(key)
except Exception as exc: # noqa: BLE001 - reported below when the data probe passed.
delete_error = exc
if delete_error is not None:
return OperationalCheck(
id="files.managed_storage_roundtrip",
label="Managed file storage",
state="error",
detail=(
"Managed file bytes round-tripped, but probe cleanup failed "
f"({type(delete_error).__name__})."
),
readiness_critical=True,
metrics={"backend": backend.name, "probe_bytes": len(payload)},
)
return OperationalCheck(
id="files.managed_storage_roundtrip",
label="Managed file storage",
state="ok",
detail="The configured managed file store passed write, read, stat, integrity, and delete checks.",
metrics={"backend": backend.name, "probe_bytes": len(payload)},
)
@@ -0,0 +1,133 @@
from __future__ import annotations
from collections import defaultdict
from datetime import UTC, datetime
from sqlalchemy import or_, select
from sqlalchemy.orm import Session
from govoplan_core.core.provider_governance import (
ExternalProviderRuntimeState,
ExternalProviderStateContext,
)
from govoplan_files.backend.db.models import FileConnectorProfile, FileConnectorSpace
from govoplan_files.backend.storage.connector_providers import (
ConnectorProviderDescriptor,
connector_provider_descriptors,
)
REMOTE_STORAGE_PROVIDER_ID = "files.remote_storage"
def remote_storage_provider_states(
context: ExternalProviderStateContext,
) -> tuple[ExternalProviderRuntimeState, ...]:
if not isinstance(context.session, Session):
raise RuntimeError("Files provider state requires a database session.")
statement = select(FileConnectorProfile)
if context.tenant_id is not None:
statement = statement.where(
or_(
FileConnectorProfile.tenant_id.is_(None),
FileConnectorProfile.tenant_id == context.tenant_id,
)
)
profiles = tuple(
context.session.scalars(
statement.order_by(
FileConnectorProfile.tenant_id,
FileConnectorProfile.id,
).limit(context.max_items + 1)
)
)
if not profiles:
return ()
profile_ids = tuple(item.id for item in profiles)
space_statement = select(FileConnectorSpace).where(
FileConnectorSpace.connector_profile_id.in_(profile_ids),
FileConnectorSpace.deleted_at.is_(None),
)
if context.tenant_id is not None:
space_statement = space_statement.where(
FileConnectorSpace.tenant_id == context.tenant_id
)
spaces_by_profile: dict[str, list[FileConnectorSpace]] = defaultdict(list)
for space in context.session.scalars(space_statement):
spaces_by_profile[space.connector_profile_id].append(space)
descriptors = {
item.provider: item for item in connector_provider_descriptors()
}
observed_at = datetime.now(UTC)
return tuple(
_profile_state(
profile,
spaces=spaces_by_profile.get(profile.id, []),
descriptor=descriptors.get(profile.provider),
observed_at=observed_at,
)
for profile in profiles
)
def _profile_state(
profile: FileConnectorProfile,
*,
spaces: list[FileConnectorSpace],
descriptor: ConnectorProviderDescriptor | None,
observed_at: datetime,
) -> ExternalProviderRuntimeState:
active_spaces = tuple(item for item in spaces if item.is_active)
active = bool(profile.enabled)
implementation_ready = bool(
descriptor is not None and descriptor.implemented and descriptor.installed
)
health = (
"inactive"
if not active
else "error"
if not implementation_ready
else "unknown"
)
recovery = (
"not_applicable"
if not active
else "unsupported"
if not implementation_ready
else "attention"
)
detail = (
"Remote-storage connector profile is disabled."
if not active
else "The configured provider is not available in this runtime."
if not implementation_ready
else "Software support is available; no live remote health observation is retained."
)
return ExternalProviderRuntimeState(
provider_id=REMOTE_STORAGE_PROVIDER_ID,
binding_ref=f"files:connector-profile:{profile.id}",
authority_mode="external_mirror",
observed_at=observed_at,
configured=True,
active=active,
health=health,
freshness="unknown" if active else "not_applicable",
conflict="not_applicable",
recovery=recovery,
detail=detail,
metrics={
"provider": profile.provider,
"configured_spaces": len(spaces),
"active_spaces": len(active_spaces),
"write_requested_spaces": sum(
1 for item in active_spaces if not item.read_only
),
"software_implemented": bool(descriptor and descriptor.implemented),
"optional_dependency_available": bool(descriptor and descriptor.installed),
},
)
__all__ = ["REMOTE_STORAGE_PROVIDER_ID", "remote_storage_provider_states"]
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -0,0 +1 @@
"""Focused HTTP route modules for the Files API."""
+134
View File
@@ -0,0 +1,134 @@
from __future__ import annotations
from io import BytesIO
from fastapi import APIRouter, Depends
from fastapi.responses import StreamingResponse
from sqlalchemy.orm import Session
from govoplan_core.auth import ApiPrincipal, require_scope
from govoplan_files.backend.schemas import (
BulkDeleteRequest,
BulkDeleteResponse,
FileAssetResponse,
)
from govoplan_core.db.session import get_session
from govoplan_files.backend.storage.common import FileStorageError
from govoplan_files.backend.storage.files import (
get_asset_for_user,
read_asset_bytes,
soft_delete_assets,
)
from govoplan_files.backend.route_support import (
_asset_response,
_attachment_disposition,
_audit_connector_event,
_http_error,
_is_admin,
)
router = APIRouter(prefix="/files", tags=["files"])
@router.get("/{file_id}", response_model=FileAssetResponse)
def get_file(
file_id: str,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(require_scope("files:file:read")),
):
try:
asset = get_asset_for_user(
session,
tenant_id=principal.tenant_id,
user_id=principal.user.id,
asset_id=file_id,
is_admin=_is_admin(principal),
)
return _asset_response(session, asset, include_shares=True)
except FileStorageError as exc:
raise _http_error(exc, not_found=True) from exc
@router.get("/{file_id}/download")
def download_file(
file_id: str,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(require_scope("files:file:download")),
):
try:
asset = get_asset_for_user(
session,
tenant_id=principal.tenant_id,
user_id=principal.user.id,
asset_id=file_id,
is_admin=_is_admin(principal),
)
data, version, blob = read_asset_bytes(session, asset)
_audit_connector_event(
session,
principal,
action="files.connector.accessed",
asset=asset,
version=version,
blob=blob,
operation="download",
commit=True,
)
except FileStorageError as exc:
raise _http_error(exc, not_found=True) from exc
headers = {"Content-Disposition": _attachment_disposition(asset.filename)}
return StreamingResponse(
BytesIO(data),
media_type=blob.content_type or "application/octet-stream",
headers=headers,
)
@router.delete("/{file_id}", response_model=BulkDeleteResponse)
def delete_file(
file_id: str,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(require_scope("files:file:delete")),
):
try:
asset = get_asset_for_user(
session,
tenant_id=principal.tenant_id,
user_id=principal.user.id,
asset_id=file_id,
require_write=True,
is_admin=_is_admin(principal),
)
count = soft_delete_assets(session, [asset])
session.commit()
return BulkDeleteResponse(deleted_count=count)
except FileStorageError as exc:
session.rollback()
raise _http_error(exc, not_found=True) from exc
@router.post("/bulk-delete", response_model=BulkDeleteResponse)
def bulk_delete_files(
payload: BulkDeleteRequest,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(require_scope("files:file:delete")),
):
try:
assets = [
get_asset_for_user(
session,
tenant_id=principal.tenant_id,
user_id=principal.user.id,
asset_id=file_id,
require_write=True,
is_admin=_is_admin(principal),
)
for file_id in payload.file_ids
]
count = soft_delete_assets(session, assets)
session.commit()
return BulkDeleteResponse(deleted_count=count)
except FileStorageError as exc:
session.rollback()
raise _http_error(exc) from exc
@@ -0,0 +1,252 @@
from __future__ import annotations
import json
from fastapi import APIRouter, Depends, HTTPException, status
from sqlalchemy.orm import Session
from govoplan_core.auth import ApiPrincipal, require_any_scope, require_scope
from govoplan_files.backend.schemas import (
FileConnectorBrowseItem,
FileConnectorBrowseResponse,
FileConnectorImportRequest,
FileConnectorSyncResponse,
FileUploadResponse,
)
from govoplan_core.db.session import get_session
from govoplan_files.backend.storage.paths import UnsafeFilePathError
from govoplan_files.backend.storage.common import FileStorageError
from govoplan_files.backend.storage.connector_browse import (
ConnectorBrowseError,
ConnectorBrowseUnsupported,
browse_connector_profile,
normalize_connector_browse_path,
)
from govoplan_files.backend.storage.connector_imports import (
ConnectorImportError,
ConnectorImportUnsupported,
)
from govoplan_files.backend.storage.connector_deployment import (
connector_effective_endpoint_url,
)
from govoplan_files.backend.storage.connector_policy import (
ConnectorAccessRequest,
ConnectorPolicyDenied,
connector_policy_decision,
)
from govoplan_files.backend.storage.files import (
create_file_asset,
sync_file_asset_from_source,
)
from govoplan_files.backend.route_support import (
_asset_response,
_audit_connector_imports,
_audit_connector_sync,
_connector_browse_next_token,
_connector_policy_error,
_download_connector_payload,
_ensure_campaign_file_access,
_http_error,
_is_admin,
_visible_connector_profile,
)
router = APIRouter(prefix="/files", tags=["files"])
@router.post(
"/connectors/profiles/{profile_id}/import", response_model=FileUploadResponse
)
def import_connector_file(
profile_id: str,
payload: FileConnectorImportRequest,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(require_scope("files:file:upload")),
):
try:
if payload.campaign_id:
_ensure_campaign_file_access(session, principal, payload.campaign_id)
profile = _visible_connector_profile(
session, principal, profile_id, campaign_id=payload.campaign_id
)
_source_path, downloaded, metadata = _download_connector_payload(
profile, payload, operation="import"
)
target_owner = payload.owner_id or principal.user.id
stored = create_file_asset(
session,
tenant_id=principal.tenant_id,
owner_type=payload.owner_type,
owner_id=target_owner,
user_id=principal.user.id,
filename=downloaded.filename,
data=downloaded.data,
folder=payload.target_folder,
display_path=payload.target_path,
content_type=downloaded.content_type,
metadata=metadata,
campaign_id=payload.campaign_id,
conflict_strategy=payload.conflict_strategy,
is_admin=_is_admin(principal),
)
_audit_connector_imports(session, principal, [stored.asset])
session.commit()
except ConnectorPolicyDenied as exc:
session.rollback()
raise _connector_policy_error(exc) from exc
except ConnectorImportUnsupported as exc:
session.rollback()
raise HTTPException(
status_code=status.HTTP_501_NOT_IMPLEMENTED, detail=str(exc)
) from exc
except (
ConnectorImportError,
FileStorageError,
UnsafeFilePathError,
ValueError,
json.JSONDecodeError,
) as exc:
session.rollback()
raise _http_error(exc) from exc
return FileUploadResponse(
files=[_asset_response(session, stored.asset, include_shares=True)]
)
@router.post(
"/connectors/profiles/{profile_id}/sync", response_model=FileConnectorSyncResponse
)
def sync_connector_file(
profile_id: str,
payload: FileConnectorImportRequest,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(require_scope("files:file:upload")),
):
try:
if payload.campaign_id:
_ensure_campaign_file_access(session, principal, payload.campaign_id)
profile = _visible_connector_profile(
session, principal, profile_id, campaign_id=payload.campaign_id
)
_source_path, downloaded, metadata = _download_connector_payload(
profile, payload, operation="sync"
)
target_owner = payload.owner_id or principal.user.id
stored, sync_action, previous_version_id = sync_file_asset_from_source(
session,
tenant_id=principal.tenant_id,
owner_type=payload.owner_type,
owner_id=target_owner,
user_id=principal.user.id,
filename=downloaded.filename,
data=downloaded.data,
folder=payload.target_folder,
display_path=payload.target_path,
content_type=downloaded.content_type,
metadata=metadata,
campaign_id=payload.campaign_id,
conflict_strategy=payload.conflict_strategy,
is_admin=_is_admin(principal),
)
_audit_connector_sync(
session,
principal,
stored.asset,
sync_action=sync_action,
previous_version_id=previous_version_id,
)
session.commit()
except ConnectorPolicyDenied as exc:
session.rollback()
raise _connector_policy_error(exc) from exc
except ConnectorImportUnsupported as exc:
session.rollback()
raise HTTPException(
status_code=status.HTTP_501_NOT_IMPLEMENTED, detail=str(exc)
) from exc
except (
ConnectorImportError,
FileStorageError,
UnsafeFilePathError,
ValueError,
json.JSONDecodeError,
) as exc:
session.rollback()
raise _http_error(exc) from exc
return FileConnectorSyncResponse(
file=_asset_response(session, stored.asset, include_shares=True),
action=sync_action,
previous_version_id=previous_version_id,
current_version_id=stored.version.id,
)
@router.get(
"/connectors/profiles/{profile_id}/browse",
response_model=FileConnectorBrowseResponse,
)
def browse_connector_profile_items(
profile_id: str,
path: str | None = None,
library_id: str | None = None,
continuation_token: str | None = None,
campaign_id: str | None = None,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(
require_any_scope(
"files:file:read",
"files:file:upload",
"files:file:download",
"files:file:admin",
"system:settings:read",
"admin:settings:read",
)
),
):
try:
profile = _visible_connector_profile(
session, principal, profile_id, campaign_id=campaign_id
)
browse_path = normalize_connector_browse_path(path)
decision = connector_policy_decision(
ConnectorAccessRequest(
connector_id=profile.id,
credential_id=profile.credential_profile_id,
provider=profile.provider,
external_path=browse_path,
external_url=connector_effective_endpoint_url(
provider=profile.provider,
endpoint_url=profile.endpoint_url,
metadata=profile.metadata,
),
operation="browse",
),
profile.policy_sources,
)
if not decision.allowed:
raise ConnectorPolicyDenied(decision)
items = browse_connector_profile(
profile,
path=browse_path,
library_id=library_id,
continuation_token=continuation_token,
)
except ConnectorPolicyDenied as exc:
raise _connector_policy_error(exc) from exc
except ConnectorBrowseUnsupported as exc:
raise HTTPException(
status_code=status.HTTP_501_NOT_IMPLEMENTED, detail=str(exc)
) from exc
except (ConnectorBrowseError, OSError, ValueError, json.JSONDecodeError) as exc:
raise _http_error(exc) from exc
return FileConnectorBrowseResponse(
profile_id=profile.id,
provider=profile.provider,
path=browse_path,
library_id=library_id,
next_continuation_token=_connector_browse_next_token(items),
has_more=any(bool(item.metadata.get("listing_truncated")) for item in items),
decision=decision.to_dict(),
items=[FileConnectorBrowseItem(**item.to_response()) for item in items],
)
@@ -0,0 +1,261 @@
from __future__ import annotations
import json
from fastapi import APIRouter, Depends
from sqlalchemy.orm import Session
from govoplan_core.auth import ApiPrincipal, require_any_scope
from govoplan_files.backend.change_tracking import (
FILES_CONNECTOR_PROFILES_COLLECTION,
)
from govoplan_files.backend.schemas import (
FileConnectorProfileResponse,
FileConnectorProfileUpdateRequest,
)
from govoplan_core.db.session import get_session
from govoplan_files.backend.storage.common import FileStorageError
from govoplan_files.backend.storage.connector_credential_deletion import (
delete_connector_profile_row,
)
from govoplan_files.backend.storage.connector_deployment import (
connector_effective_endpoint_url,
)
from govoplan_files.backend.storage.connector_profile_store import (
connector_profile_from_row,
get_connector_profile_row,
update_connector_profile_row,
)
from govoplan_files.backend.storage.connector_policy import (
ConnectorPolicyDenied,
)
from govoplan_files.backend.route_support import (
FILES_CONNECTOR_PROFILE_RESOURCE,
_can_read_disabled_connector_profiles,
_connector_policy_error,
_credential_row_for_profile,
_ensure_connector_configuration_allowed,
_ensure_connector_local_policy_allowed,
_http_error,
_record_connector_settings_change,
_require_connector_profile_write,
_visible_connector_profile,
)
router = APIRouter(prefix="/files", tags=["files"])
@router.get(
"/connectors/profiles/{profile_id}", response_model=FileConnectorProfileResponse
)
def get_connector_profile(
profile_id: str,
campaign_id: str | None = None,
include_disabled: bool = False,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(
require_any_scope(
"files:file:read",
"files:file:upload",
"files:file:download",
"files:file:admin",
"system:settings:read",
"admin:settings:read",
)
),
):
try:
profile = _visible_connector_profile(
session,
principal,
profile_id,
campaign_id=campaign_id,
include_disabled=include_disabled
and _can_read_disabled_connector_profiles(principal),
include_effective_policy=False,
)
except (OSError, ValueError, json.JSONDecodeError) as exc:
raise _http_error(exc) from exc
return FileConnectorProfileResponse(**profile.to_response())
@router.patch(
"/connectors/profiles/{profile_id}", response_model=FileConnectorProfileResponse
)
def update_connector_profile(
profile_id: str,
payload: FileConnectorProfileUpdateRequest,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(
require_any_scope(
"files:file:admin", "system:settings:write", "admin:settings:write"
)
),
):
try:
row = get_connector_profile_row(
session,
tenant_id=principal.tenant_id,
profile_id=profile_id,
include_disabled=True,
)
except FileStorageError as exc:
raise _http_error(exc, not_found=True) from exc
_require_connector_profile_write(principal, row.scope_type)
credentials = payload.credentials
try:
credential_profile_id = (
payload.credential_profile_id
if payload.credential_profile_id is not None
else row.credential_profile_id
)
provider = payload.provider if payload.provider is not None else row.provider
credential_row = _credential_row_for_profile(
session,
principal,
credential_profile_id=credential_profile_id,
provider=provider,
profile_id=row.id,
scope_type=row.scope_type,
scope_id=row.scope_id,
include_disabled=True,
)
_ensure_connector_configuration_allowed(
session,
principal,
connector_id=row.id,
credential_id=credential_profile_id,
provider=provider,
endpoint_url=connector_effective_endpoint_url(
provider=provider,
endpoint_url=payload.endpoint_url
if payload.endpoint_url is not None
else row.endpoint_url,
metadata=payload.metadata
if payload.metadata is not None
else row.metadata_,
),
base_path=payload.base_path
if payload.base_path is not None
else row.base_path,
scope_type=row.scope_type,
scope_id=row.scope_id,
operation="configure",
)
if payload.policy is not None:
_ensure_connector_local_policy_allowed(
session,
principal,
scope_type=row.scope_type,
scope_id=row.scope_id,
policy=payload.policy,
)
update_connector_profile_row(
session,
row,
user_id=principal.user.id,
label=payload.label,
provider=payload.provider,
endpoint_url=payload.endpoint_url,
base_path=payload.base_path,
enabled=payload.enabled,
credential_profile_id=payload.credential_profile_id,
credential_mode=payload.credential_mode,
username=credentials.username if credentials else None,
password=credentials.password if credentials else None,
token=credentials.token if credentials else None,
password_env=credentials.password_env if credentials else None,
token_env=credentials.token_env if credentials else None,
secret_ref=credentials.secret_ref if credentials else None,
clear_password=payload.clear_password,
clear_token=payload.clear_token,
capabilities=payload.capabilities,
policy=payload.policy,
metadata=payload.metadata,
)
_record_connector_settings_change(
session,
collection=FILES_CONNECTOR_PROFILES_COLLECTION,
resource_type=FILES_CONNECTOR_PROFILE_RESOURCE,
resource_id=row.id,
operation="updated",
principal=principal,
tenant_id=row.tenant_id,
payload={
"scope_type": row.scope_type,
"scope_id": row.scope_id,
"provider": row.provider,
},
)
session.commit()
session.refresh(row)
return FileConnectorProfileResponse(
**connector_profile_from_row(
row, credential_row=credential_row
).to_response()
)
except ConnectorPolicyDenied as exc:
session.rollback()
raise _connector_policy_error(exc) from exc
except (FileStorageError, ValueError, json.JSONDecodeError) as exc:
session.rollback()
raise _http_error(exc) from exc
@router.delete(
"/connectors/profiles/{profile_id}", response_model=FileConnectorProfileResponse
)
def deactivate_connector_profile(
profile_id: str,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(
require_any_scope(
"files:file:admin", "system:settings:write", "admin:settings:write"
)
),
):
try:
row = get_connector_profile_row(
session,
tenant_id=principal.tenant_id,
profile_id=profile_id,
include_disabled=True,
)
except FileStorageError as exc:
raise _http_error(exc, not_found=True) from exc
_require_connector_profile_write(principal, row.scope_type)
try:
changed = delete_connector_profile_row(
session,
row,
deletion_reason="api_delete",
user_id=principal.user.id,
api_key_id=principal.api_key.id if principal.api_key else None,
)
if changed:
_record_connector_settings_change(
session,
collection=FILES_CONNECTOR_PROFILES_COLLECTION,
resource_type=FILES_CONNECTOR_PROFILE_RESOURCE,
resource_id=row.id,
operation="deleted",
principal=principal,
tenant_id=row.tenant_id,
payload={
"scope_type": row.scope_type,
"scope_id": row.scope_id,
"provider": row.provider,
},
)
session.commit()
session.refresh(row)
return FileConnectorProfileResponse(
**connector_profile_from_row(row).to_response()
)
except FileStorageError as exc:
session.rollback()
raise _http_error(exc) from exc
except Exception:
session.rollback()
raise
@@ -0,0 +1,852 @@
from __future__ import annotations
import json
from typing import Literal
from fastapi import APIRouter, Depends, Query, status
from sqlalchemy.orm import Session
from govoplan_core.auth import ApiPrincipal, require_any_scope
from govoplan_files.backend.change_tracking import (
FILES_CONNECTOR_CREDENTIALS_COLLECTION,
FILES_CONNECTOR_POLICIES_COLLECTION,
FILES_CONNECTOR_PROFILES_COLLECTION,
)
from govoplan_files.backend.schemas import (
FileConnectorCredentialCreateRequest,
FileConnectorCredentialResponse,
FileConnectorCredentialsResponse,
FileConnectorCredentialUpdateRequest,
FileConnectorDiscoveryRequest,
FileConnectorDiscoveryResponse,
FileConnectorSettingsDeltaResponse,
FileConnectorPolicyEvaluateRequest,
FileConnectorPolicyEvaluateResponse,
FileConnectorPolicyResponse,
FileConnectorPolicyUpdateRequest,
FileConnectorProfileCreateRequest,
FileConnectorProfileResponse,
FileConnectorProfilesResponse,
FileConnectorProviderResponse,
FileConnectorProvidersResponse,
)
from govoplan_core.db.session import get_session
from govoplan_files.backend.storage.common import FileStorageError
from govoplan_files.backend.storage.connector_credential_store import (
create_connector_credential_row,
get_connector_credential_row,
list_database_connector_credentials,
update_connector_credential_row,
)
from govoplan_files.backend.storage.connector_credential_deletion import (
delete_connector_credential_row,
)
from govoplan_files.backend.storage.connector_browse import (
ConnectorBrowseError,
ConnectorBrowseUnsupported,
browse_connector_profile,
)
from govoplan_files.backend.storage.connector_deployment import (
connector_effective_endpoint_url,
reject_api_controlled_deployment_references,
)
from govoplan_files.backend.storage.connector_profile_store import (
connector_profile_from_row,
create_connector_profile_row,
)
from govoplan_files.backend.storage.connector_providers import (
connector_provider_descriptors,
)
from govoplan_files.backend.storage.connector_policy import (
ConnectorAccessRequest,
ConnectorPolicyDenied,
connector_policy_decision,
connector_policy_sources_from_payload,
)
from govoplan_files.backend.storage.connector_policy_store import (
connector_policy_response,
set_connector_policy,
)
from govoplan_files.backend.route_support import (
FILES_CONNECTOR_CREDENTIAL_RESOURCE,
FILES_CONNECTOR_POLICY_RESOURCE,
FILES_CONNECTOR_PROFILE_RESOURCE,
_audit_connector_discovery_attempt,
_can_read_disabled_connector_profiles,
_connector_credential_response,
_connector_policy_error,
_credential_row_for_profile,
_discovery_profile_from_payload,
_ensure_campaign_file_access,
_ensure_connector_configuration_allowed,
_ensure_connector_credential_configuration_allowed,
_ensure_connector_local_policy_allowed,
_file_connector_policy_resource_id,
_file_connector_settings_entries,
_http_error,
_record_connector_settings_change,
_require_connector_credential_write,
_require_connector_policy_read,
_require_connector_profile_write,
_same_endpoint,
_visible_connector_profiles,
_webdav_discovery_candidates,
)
from govoplan_files.backend.services.connector_settings_delta import (
_full_file_connector_settings_delta_response,
_incremental_file_connector_settings_delta_response,
)
router = APIRouter(prefix="/files", tags=["files"])
@router.post(
"/connector-policy/evaluate", response_model=FileConnectorPolicyEvaluateResponse
)
def evaluate_connector_policy(
payload: FileConnectorPolicyEvaluateRequest,
principal: ApiPrincipal = Depends(
require_any_scope("files:file:read", "files:file:upload", "files:file:download")
),
):
del principal
sources = connector_policy_sources_from_payload(
[item.model_dump(mode="json") for item in payload.policy_sources]
)
request = ConnectorAccessRequest.from_provenance(
payload.source_provenance.model_dump(mode="json", exclude_none=True),
operation=payload.operation,
)
return FileConnectorPolicyEvaluateResponse(
decision=connector_policy_decision(request, sources).to_dict()
)
@router.get(
"/connectors/settings/delta", response_model=FileConnectorSettingsDeltaResponse
)
def connector_settings_delta(
scope_type: str = Query(default="tenant"),
scope_id: str | None = Query(default=None),
provider: str | None = None,
campaign_id: str | None = None,
include_disabled: bool = False,
include_inactive: bool = False,
owner_type: Literal["user", "group"] | None = None,
owner_id: str | None = None,
since: str | None = None,
limit: int = Query(default=100, ge=1, le=500),
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(
require_any_scope(
"files:file:admin", "system:settings:read", "admin:settings:read"
)
),
):
scope_type = scope_type.strip().casefold()
_require_connector_policy_read(principal, scope_type)
try:
if since is None:
return _full_file_connector_settings_delta_response(
session,
principal,
scope_type=scope_type,
scope_id=scope_id,
provider=provider,
campaign_id=campaign_id,
include_disabled=include_disabled,
include_inactive=include_inactive,
owner_type=owner_type,
owner_id=owner_id,
)
entries, has_more = _file_connector_settings_entries(
session, tenant_id=principal.tenant_id, since=since, limit=limit
)
if entries is None:
return _full_file_connector_settings_delta_response(
session,
principal,
scope_type=scope_type,
scope_id=scope_id,
provider=provider,
campaign_id=campaign_id,
include_disabled=include_disabled,
include_inactive=include_inactive,
owner_type=owner_type,
owner_id=owner_id,
)
return _incremental_file_connector_settings_delta_response(
session,
principal,
entries=entries,
has_more=has_more,
scope_type=scope_type,
scope_id=scope_id,
provider=provider,
campaign_id=campaign_id,
include_disabled=include_disabled,
include_inactive=include_inactive,
owner_type=owner_type,
owner_id=owner_id,
)
except (FileStorageError, ValueError, json.JSONDecodeError) as exc:
raise _http_error(exc) from exc
@router.get("/connectors/providers", response_model=FileConnectorProvidersResponse)
def list_connector_providers(
principal: ApiPrincipal = Depends(
require_any_scope("files:file:read", "files:file:upload", "files:file:admin")
),
):
del principal
return FileConnectorProvidersResponse(
providers=[
FileConnectorProviderResponse(**item.to_response())
for item in connector_provider_descriptors()
]
)
@router.post("/connectors/discover", response_model=FileConnectorDiscoveryResponse)
def discover_connector_endpoint(
payload: FileConnectorDiscoveryRequest,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(
require_any_scope(
"files:file:admin", "system:settings:write", "admin:settings:write"
)
),
):
try:
reject_api_controlled_deployment_references(
password_env=payload.credentials.password_env,
token_env=payload.credentials.token_env,
secret_ref=payload.credentials.secret_ref,
metadata=payload.metadata,
)
except ValueError as exc:
raise _http_error(exc) from exc
if payload.provider not in {"webdav", "nextcloud"}:
return FileConnectorDiscoveryResponse(
provider=payload.provider,
endpoint_url=None,
base_path=payload.base_path,
status="unsupported",
message=f"Discovery is not implemented for {payload.provider} connectors yet",
)
candidates: list[dict[str, str]] = []
for endpoint_url in _webdav_discovery_candidates(payload):
profile = _discovery_profile_from_payload(
payload, endpoint_url, principal=principal
)
try:
_ensure_connector_configuration_allowed(
session,
principal,
connector_id=None,
credential_id=None,
provider=profile.provider,
endpoint_url=endpoint_url,
base_path=profile.base_path,
scope_type="tenant",
scope_id=principal.tenant_id,
operation="discover",
)
_audit_connector_discovery_attempt(
session,
principal,
provider=profile.provider,
endpoint_url=endpoint_url,
base_path=profile.base_path,
)
browse_connector_profile(profile, path=payload.base_path or "")
except ConnectorPolicyDenied as exc:
raise _connector_policy_error(exc) from exc
except (
ConnectorBrowseError,
ConnectorBrowseUnsupported,
OSError,
ValueError,
json.JSONDecodeError,
) as exc:
message = str(exc)
if "credentials were rejected" in message.casefold():
if payload.require_valid_credentials:
candidates.append(
{
"endpoint_url": endpoint_url,
"status": "credentials_rejected",
"message": "The endpoint exists, but the credentials were rejected.",
}
)
return FileConnectorDiscoveryResponse(
provider=payload.provider,
endpoint_url=endpoint_url,
base_path=payload.base_path,
status="credentials_rejected",
message="The endpoint was found, but login failed with these credentials.",
candidates=candidates,
metadata={"discovered_by": "webdav-auth-challenge"},
)
candidates.append(
{
"endpoint_url": endpoint_url,
"status": "found",
"message": "The endpoint exists, but credentials are required or were rejected.",
}
)
return FileConnectorDiscoveryResponse(
provider=payload.provider,
endpoint_url=endpoint_url,
base_path=payload.base_path,
status="found",
message="The endpoint was found. Add working credentials before saving or testing the connection.",
candidates=candidates,
metadata={"discovered_by": "webdav-auth-challenge"},
)
candidates.append(
{"endpoint_url": endpoint_url, "status": "failed", "message": message}
)
continue
status_value = (
"usable" if _same_endpoint(endpoint_url, payload.endpoint_url) else "found"
)
message = (
"The supplied URL is directly usable."
if status_value == "usable"
else "A usable connector endpoint was discovered."
)
candidates.append(
{"endpoint_url": endpoint_url, "status": status_value, "message": message}
)
return FileConnectorDiscoveryResponse(
provider=payload.provider,
endpoint_url=endpoint_url,
base_path=payload.base_path,
status=status_value,
message=message,
candidates=candidates,
metadata={"discovered_by": "webdav-propfind"},
)
return FileConnectorDiscoveryResponse(
provider=payload.provider,
endpoint_url=None,
base_path=payload.base_path,
status="not_found",
message="No usable WebDAV endpoint was found for this server URL.",
candidates=candidates,
)
@router.get("/connectors/credentials", response_model=FileConnectorCredentialsResponse)
def list_connector_credentials(
provider: str | None = None,
include_disabled: bool = False,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(
require_any_scope(
"files:file:admin", "system:settings:read", "admin:settings:read"
)
),
):
provider_norm = provider.strip().casefold() if provider else None
credentials = list_database_connector_credentials(
session,
tenant_id=principal.tenant_id,
include_disabled=include_disabled
and _can_read_disabled_connector_profiles(principal),
)
return FileConnectorCredentialsResponse(
credentials=[
FileConnectorCredentialResponse(**credential.to_response())
for credential in credentials
if provider_norm is None or credential.provider in {None, provider_norm}
]
)
@router.get(
"/connectors/policies/{scope_type}", response_model=FileConnectorPolicyResponse
)
def read_connector_policy(
scope_type: str,
scope_id: str | None = Query(default=None),
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(
require_any_scope(
"files:file:admin", "system:settings:read", "admin:settings:read"
)
),
):
_require_connector_policy_read(principal, scope_type)
try:
return FileConnectorPolicyResponse(
**connector_policy_response(
session,
tenant_id=principal.tenant_id,
scope_type=scope_type,
scope_id=scope_id,
)
)
except (FileStorageError, ValueError, json.JSONDecodeError) as exc:
raise _http_error(exc, not_found=True) from exc
@router.put(
"/connectors/policies/{scope_type}", response_model=FileConnectorPolicyResponse
)
def write_connector_policy(
scope_type: str,
payload: FileConnectorPolicyUpdateRequest,
scope_id: str | None = Query(default=None),
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(
require_any_scope(
"files:file:admin", "system:settings:write", "admin:settings:write"
)
),
):
_require_connector_profile_write(principal, scope_type)
try:
set_connector_policy(
session,
tenant_id=principal.tenant_id,
user_id=principal.user.id,
scope_type=scope_type,
scope_id=scope_id,
policy=payload.policy,
)
_record_connector_settings_change(
session,
collection=FILES_CONNECTOR_POLICIES_COLLECTION,
resource_type=FILES_CONNECTOR_POLICY_RESOURCE,
resource_id=_file_connector_policy_resource_id(scope_type, scope_id),
operation="updated",
principal=principal,
tenant_id=None
if scope_type.strip().casefold() == "system"
else principal.tenant_id,
payload={"scope_type": scope_type.strip().casefold(), "scope_id": scope_id},
)
session.commit()
return FileConnectorPolicyResponse(
**connector_policy_response(
session,
tenant_id=principal.tenant_id,
scope_type=scope_type,
scope_id=scope_id,
)
)
except (FileStorageError, ValueError, json.JSONDecodeError) as exc:
session.rollback()
raise _http_error(exc) from exc
@router.post(
"/connectors/credentials",
response_model=FileConnectorCredentialResponse,
status_code=status.HTTP_201_CREATED,
)
def create_connector_credential(
payload: FileConnectorCredentialCreateRequest,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(
require_any_scope(
"files:file:admin", "system:settings:write", "admin:settings:write"
)
),
):
_require_connector_credential_write(principal, payload.scope_type)
credentials = payload.credentials
try:
_ensure_connector_credential_configuration_allowed(
session,
principal,
credential_id=payload.id,
provider=payload.provider,
scope_type=payload.scope_type,
scope_id=payload.scope_id,
operation="configure_credentials",
)
_ensure_connector_local_policy_allowed(
session,
principal,
scope_type=payload.scope_type,
scope_id=payload.scope_id,
policy=payload.policy,
)
row = create_connector_credential_row(
session,
tenant_id=principal.tenant_id,
user_id=principal.user.id,
credential_id=payload.id,
label=payload.label,
provider=payload.provider,
scope_type=payload.scope_type,
scope_id=payload.scope_id,
enabled=payload.enabled,
credential_mode=payload.credential_mode,
username=credentials.username,
password=credentials.password,
token=credentials.token,
password_env=credentials.password_env,
token_env=credentials.token_env,
secret_ref=credentials.secret_ref,
policy=payload.policy,
metadata=payload.metadata,
)
_record_connector_settings_change(
session,
collection=FILES_CONNECTOR_CREDENTIALS_COLLECTION,
resource_type=FILES_CONNECTOR_CREDENTIAL_RESOURCE,
resource_id=row.id,
operation="created",
principal=principal,
tenant_id=row.tenant_id,
payload={
"scope_type": row.scope_type,
"scope_id": row.scope_id,
"provider": row.provider,
},
)
session.commit()
session.refresh(row)
return _connector_credential_response(row)
except ConnectorPolicyDenied as exc:
session.rollback()
raise _connector_policy_error(exc) from exc
except (FileStorageError, ValueError, json.JSONDecodeError) as exc:
session.rollback()
raise _http_error(exc) from exc
@router.get(
"/connectors/credentials/{credential_id}",
response_model=FileConnectorCredentialResponse,
)
def get_connector_credential(
credential_id: str,
include_disabled: bool = False,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(
require_any_scope(
"files:file:admin", "system:settings:read", "admin:settings:read"
)
),
):
try:
row = get_connector_credential_row(
session,
tenant_id=principal.tenant_id,
credential_id=credential_id,
include_disabled=include_disabled
and _can_read_disabled_connector_profiles(principal),
)
return _connector_credential_response(row)
except FileStorageError as exc:
raise _http_error(exc, not_found=True) from exc
@router.patch(
"/connectors/credentials/{credential_id}",
response_model=FileConnectorCredentialResponse,
)
def update_connector_credential(
credential_id: str,
payload: FileConnectorCredentialUpdateRequest,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(
require_any_scope(
"files:file:admin", "system:settings:write", "admin:settings:write"
)
),
):
try:
row = get_connector_credential_row(
session,
tenant_id=principal.tenant_id,
credential_id=credential_id,
include_disabled=True,
)
except FileStorageError as exc:
raise _http_error(exc, not_found=True) from exc
_require_connector_credential_write(principal, row.scope_type)
credentials = payload.credentials
try:
provider = payload.provider if payload.provider is not None else row.provider
_ensure_connector_credential_configuration_allowed(
session,
principal,
credential_id=row.id,
provider=provider,
scope_type=row.scope_type,
scope_id=row.scope_id,
operation="configure_credentials",
)
if payload.policy is not None:
_ensure_connector_local_policy_allowed(
session,
principal,
scope_type=row.scope_type,
scope_id=row.scope_id,
policy=payload.policy,
)
update_connector_credential_row(
session,
row,
user_id=principal.user.id,
label=payload.label,
provider=payload.provider,
enabled=payload.enabled,
credential_mode=payload.credential_mode,
username=credentials.username if credentials else None,
password=credentials.password if credentials else None,
token=credentials.token if credentials else None,
password_env=credentials.password_env if credentials else None,
token_env=credentials.token_env if credentials else None,
secret_ref=credentials.secret_ref if credentials else None,
clear_password=payload.clear_password,
clear_token=payload.clear_token,
policy=payload.policy,
metadata=payload.metadata,
)
_record_connector_settings_change(
session,
collection=FILES_CONNECTOR_CREDENTIALS_COLLECTION,
resource_type=FILES_CONNECTOR_CREDENTIAL_RESOURCE,
resource_id=row.id,
operation="updated",
principal=principal,
tenant_id=row.tenant_id,
payload={
"scope_type": row.scope_type,
"scope_id": row.scope_id,
"provider": row.provider,
},
)
session.commit()
session.refresh(row)
return _connector_credential_response(row)
except ConnectorPolicyDenied as exc:
session.rollback()
raise _connector_policy_error(exc) from exc
except (FileStorageError, ValueError, json.JSONDecodeError) as exc:
session.rollback()
raise _http_error(exc) from exc
@router.delete(
"/connectors/credentials/{credential_id}",
response_model=FileConnectorCredentialResponse,
)
def deactivate_connector_credential(
credential_id: str,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(
require_any_scope(
"files:file:admin", "system:settings:write", "admin:settings:write"
)
),
):
try:
row = get_connector_credential_row(
session,
tenant_id=principal.tenant_id,
credential_id=credential_id,
include_disabled=True,
)
except FileStorageError as exc:
raise _http_error(exc, not_found=True) from exc
_require_connector_credential_write(principal, row.scope_type)
try:
deletion = delete_connector_credential_row(
session,
row,
deletion_reason="api_delete",
user_id=principal.user.id,
api_key_id=principal.api_key.id if principal.api_key else None,
)
if deletion.changed:
_record_connector_settings_change(
session,
collection=FILES_CONNECTOR_CREDENTIALS_COLLECTION,
resource_type=FILES_CONNECTOR_CREDENTIAL_RESOURCE,
resource_id=row.id,
operation="deleted",
principal=principal,
tenant_id=row.tenant_id,
payload={
"scope_type": row.scope_type,
"scope_id": row.scope_id,
"provider": row.provider,
},
)
for profile in deletion.affected_profiles:
_record_connector_settings_change(
session,
collection=FILES_CONNECTOR_PROFILES_COLLECTION,
resource_type=FILES_CONNECTOR_PROFILE_RESOURCE,
resource_id=profile.id,
operation="updated",
principal=principal,
tenant_id=profile.tenant_id,
payload={
"scope_type": profile.scope_type,
"scope_id": profile.scope_id,
"provider": profile.provider,
"reason": "credential_deleted",
},
)
session.commit()
session.refresh(row)
return _connector_credential_response(row)
except FileStorageError as exc:
session.rollback()
raise _http_error(exc) from exc
except Exception:
session.rollback()
raise
@router.get("/connectors/profiles", response_model=FileConnectorProfilesResponse)
def list_connector_profiles(
provider: str | None = None,
campaign_id: str | None = None,
include_disabled: bool = False,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(
require_any_scope(
"files:file:read",
"files:file:upload",
"files:file:download",
"files:file:admin",
"system:settings:read",
"admin:settings:read",
)
),
):
try:
if campaign_id:
_ensure_campaign_file_access(session, principal, campaign_id)
profiles = _visible_connector_profiles(
session,
principal,
provider=provider,
campaign_id=campaign_id,
include_disabled=include_disabled
and _can_read_disabled_connector_profiles(principal),
include_admin_scopes=_can_read_disabled_connector_profiles(principal),
include_effective_policy=False,
)
except (OSError, ValueError, json.JSONDecodeError) as exc:
raise _http_error(exc) from exc
return FileConnectorProfilesResponse(
profiles=[
FileConnectorProfileResponse(**profile.to_response())
for profile in profiles
]
)
@router.post(
"/connectors/profiles",
response_model=FileConnectorProfileResponse,
status_code=status.HTTP_201_CREATED,
)
def create_connector_profile(
payload: FileConnectorProfileCreateRequest,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(
require_any_scope(
"files:file:admin", "system:settings:write", "admin:settings:write"
)
),
):
_require_connector_profile_write(principal, payload.scope_type)
credentials = payload.credentials
try:
credential_row = _credential_row_for_profile(
session,
principal,
credential_profile_id=payload.credential_profile_id,
provider=payload.provider,
profile_id=payload.id,
scope_type=payload.scope_type,
scope_id=payload.scope_id,
)
_ensure_connector_configuration_allowed(
session,
principal,
connector_id=payload.id,
credential_id=payload.credential_profile_id,
provider=payload.provider,
endpoint_url=connector_effective_endpoint_url(
provider=payload.provider,
endpoint_url=payload.endpoint_url,
metadata=payload.metadata,
),
base_path=payload.base_path,
scope_type=payload.scope_type,
scope_id=payload.scope_id,
operation="configure",
)
_ensure_connector_local_policy_allowed(
session,
principal,
scope_type=payload.scope_type,
scope_id=payload.scope_id,
policy=payload.policy,
)
row = create_connector_profile_row(
session,
tenant_id=principal.tenant_id,
user_id=principal.user.id,
profile_id=payload.id,
label=payload.label,
provider=payload.provider,
scope_type=payload.scope_type,
scope_id=payload.scope_id,
endpoint_url=payload.endpoint_url,
base_path=payload.base_path,
enabled=payload.enabled,
credential_profile_id=payload.credential_profile_id,
credential_mode=payload.credential_mode,
username=credentials.username,
password=credentials.password,
token=credentials.token,
password_env=credentials.password_env,
token_env=credentials.token_env,
secret_ref=credentials.secret_ref,
capabilities=payload.capabilities,
policy=payload.policy,
metadata=payload.metadata,
)
_record_connector_settings_change(
session,
collection=FILES_CONNECTOR_PROFILES_COLLECTION,
resource_type=FILES_CONNECTOR_PROFILE_RESOURCE,
resource_id=row.id,
operation="created",
principal=principal,
tenant_id=row.tenant_id,
payload={
"scope_type": row.scope_type,
"scope_id": row.scope_id,
"provider": row.provider,
},
)
session.commit()
session.refresh(row)
return FileConnectorProfileResponse(
**connector_profile_from_row(
row, credential_row=credential_row
).to_response()
)
except ConnectorPolicyDenied as exc:
session.rollback()
raise _connector_policy_error(exc) from exc
except (FileStorageError, ValueError, json.JSONDecodeError) as exc:
session.rollback()
raise _http_error(exc) from exc
@@ -0,0 +1,151 @@
from __future__ import annotations
from typing import Literal
from fastapi import APIRouter, Depends, Query
from sqlalchemy.orm import Session
from govoplan_core.auth import ApiPrincipal, require_scope
from govoplan_files.backend.schemas import (
FileFolderCreateRequest,
FileFolderDeleteRequest,
FileFolderDeleteResponse,
FileFolderResponse,
FileFoldersResponse,
)
from govoplan_core.db.session import get_session
from govoplan_files.backend.storage.paths import UnsafeFilePathError
from govoplan_files.backend.storage.common import FileStorageError
from govoplan_files.backend.storage.folders import (
create_folder,
list_folders_for_user,
list_folders_for_user_window,
soft_delete_folder,
)
from govoplan_files.backend.route_support import (
_folder_response,
_http_error,
_is_admin,
)
from govoplan_files.backend.services.list_queries import (
FOLDERS_LIST_CURSOR_SCOPE,
_cursor_page_size,
_files_delta_watermark,
_folder_cursor_values,
_folders_list_fingerprint,
_next_folder_list_cursor,
)
router = APIRouter(prefix="/files", tags=["files"])
@router.get("/folders", response_model=FileFoldersResponse)
def list_file_folders(
owner_type: Literal["user", "group"],
owner_id: str,
page_size: int | None = Query(default=None, ge=1, le=1000),
cursor: str | None = None,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(require_scope("files:file:read")),
):
try:
watermark = _files_delta_watermark(session, principal.tenant_id)
effective_page_size = _cursor_page_size(
FOLDERS_LIST_CURSOR_SCOPE, cursor, page_size
)
if effective_page_size is None:
folders = list_folders_for_user(
session,
tenant_id=principal.tenant_id,
user_id=principal.user.id,
owner_type=owner_type,
owner_id=owner_id,
is_admin=_is_admin(principal),
)
return FileFoldersResponse(
folders=[_folder_response(folder) for folder in folders],
watermark=watermark,
)
fingerprint = _folders_list_fingerprint(
principal,
owner_type=owner_type,
owner_id=owner_id,
page_size=effective_page_size,
)
after_path, after_id = _folder_cursor_values(cursor, fingerprint=fingerprint)
folders, has_more = list_folders_for_user_window(
session,
tenant_id=principal.tenant_id,
user_id=principal.user.id,
owner_type=owner_type,
owner_id=owner_id,
is_admin=_is_admin(principal),
page_size=effective_page_size,
after_path=after_path,
after_id=after_id,
)
return FileFoldersResponse(
folders=[_folder_response(folder) for folder in folders],
cursor=cursor,
next_cursor=_next_folder_list_cursor(
principal,
folders,
owner_type=owner_type,
owner_id=owner_id,
page_size=effective_page_size,
has_more=has_more,
),
watermark=watermark,
)
except FileStorageError as exc:
raise _http_error(exc) from exc
@router.post("/folders", response_model=FileFolderResponse)
def create_file_folder(
payload: FileFolderCreateRequest,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(require_scope("files:file:organize")),
):
try:
folder = create_folder(
session,
tenant_id=principal.tenant_id,
owner_type=payload.owner_type,
owner_id=payload.owner_id,
user_id=principal.user.id,
path=payload.path,
is_admin=_is_admin(principal),
)
session.commit()
return _folder_response(folder)
except (FileStorageError, UnsafeFilePathError, ValueError) as exc:
session.rollback()
raise _http_error(exc) from exc
@router.post("/folders/delete", response_model=FileFolderDeleteResponse)
def delete_file_folder(
payload: FileFolderDeleteRequest,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(require_scope("files:file:delete")),
):
try:
deleted_folders, deleted_files = soft_delete_folder(
session,
tenant_id=principal.tenant_id,
owner_type=payload.owner_type,
owner_id=payload.owner_id,
user_id=principal.user.id,
path=payload.path,
recursive=payload.recursive,
is_admin=_is_admin(principal),
)
session.commit()
return FileFolderDeleteResponse(
deleted_folders=deleted_folders, deleted_files=deleted_files
)
except (FileStorageError, UnsafeFilePathError, ValueError) as exc:
session.rollback()
raise _http_error(exc) from exc
@@ -0,0 +1,387 @@
from __future__ import annotations
import hashlib
from fastapi import APIRouter, Depends, HTTPException, Query, status
from sqlalchemy.orm import Session
from govoplan_core.auth import ApiPrincipal, require_scope
from govoplan_core.audit.logging import audit_from_principal
from govoplan_core.db.session import get_session
from govoplan_files.backend.db.models import (
FileIntegrityFinding,
FileIntegrityScan,
)
from govoplan_files.backend.schemas import (
FileIntegrityActionRequest,
FileIntegrityActionResponse,
FileIntegrityFindingResponse,
FileIntegrityFindingsResponse,
FileIntegrityScanCreateRequest,
FileIntegrityScanRunRequest,
FileIntegrityScanResponse,
FileIntegrityScansResponse,
)
from govoplan_files.backend.storage.backends import StorageBackendError
from govoplan_files.backend.storage.common import FileStorageError
from govoplan_files.backend.storage.integrity import (
cleanup_orphan_finding,
create_integrity_scan,
mark_integrity_scan_failed,
recheck_integrity_finding,
run_integrity_scan_batch,
)
router = APIRouter(prefix="/files/integrity", tags=["files-integrity"])
@router.get("/scans", response_model=FileIntegrityScansResponse)
def list_integrity_scans(
limit: int = Query(default=50, ge=1, le=200),
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(require_scope("files:file:admin")),
) -> FileIntegrityScansResponse:
rows = (
session.query(FileIntegrityScan)
.filter(FileIntegrityScan.tenant_id == principal.tenant_id)
.order_by(FileIntegrityScan.created_at.desc(), FileIntegrityScan.id.desc())
.limit(limit)
.all()
)
return FileIntegrityScansResponse(
scans=[_scan_response(row) for row in rows]
)
@router.post(
"/scans",
response_model=FileIntegrityScanResponse,
status_code=status.HTTP_201_CREATED,
)
def create_scan(
payload: FileIntegrityScanCreateRequest,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(require_scope("files:file:admin")),
) -> FileIntegrityScanResponse:
try:
scan = create_integrity_scan(
session,
tenant_id=principal.tenant_id,
user_id=principal.user.id,
verify_checksums=payload.verify_checksums,
batch_size=payload.batch_size,
)
audit_from_principal(
session,
principal,
action="files.integrity.scan_created",
object_type="file_integrity_scan",
object_id=scan.id,
details={
"storage_backend": scan.storage_backend,
"verify_checksums": scan.verify_checksums,
"batch_size": scan.batch_size,
},
)
session.commit()
return _scan_response(scan)
except (FileStorageError, StorageBackendError) as exc:
session.rollback()
raise _integrity_http_error(exc) from exc
@router.post("/scans/{scan_id}/run", response_model=FileIntegrityScanResponse)
def run_scan_batch(
scan_id: str,
payload: FileIntegrityScanRunRequest,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(require_scope("files:file:admin")),
) -> FileIntegrityScanResponse:
scan = _scan_for_tenant(
session,
scan_id,
principal.tenant_id,
for_update=True,
)
_assert_expected_revision(scan.revision, payload.expected_revision)
previous_status = scan.status
try:
run_integrity_scan_batch(session, scan)
if scan.status == "completed" and previous_status != "completed":
audit_from_principal(
session,
principal,
action="files.integrity.scan_completed",
object_type="file_integrity_scan",
object_id=scan.id,
details={
"verified_blob_count": scan.verified_blob_count,
"quarantined_blob_count": scan.quarantined_blob_count,
"orphan_object_count": scan.orphan_object_count,
},
)
session.commit()
return _scan_response(scan)
except (FileStorageError, StorageBackendError) as exc:
session.rollback()
scan = _scan_for_tenant(
session,
scan_id,
principal.tenant_id,
for_update=True,
)
mark_integrity_scan_failed(scan, error=exc)
audit_from_principal(
session,
principal,
action="files.integrity.scan_failed",
object_type="file_integrity_scan",
object_id=scan.id,
details={"error_type": type(exc).__name__},
)
session.commit()
raise _integrity_http_error(exc) from exc
@router.get(
"/scans/{scan_id}/findings",
response_model=FileIntegrityFindingsResponse,
)
def list_integrity_findings(
scan_id: str,
state_filter: str | None = Query(default=None, alias="state"),
limit: int = Query(default=500, ge=1, le=1000),
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(require_scope("files:file:admin")),
) -> FileIntegrityFindingsResponse:
scan = _scan_for_tenant(session, scan_id, principal.tenant_id)
query = session.query(FileIntegrityFinding).filter(
FileIntegrityFinding.scan_id == scan.id,
FileIntegrityFinding.tenant_id == principal.tenant_id,
)
if state_filter:
query = query.filter(FileIntegrityFinding.state == state_filter)
rows = (
query.order_by(
FileIntegrityFinding.created_at.asc(),
FileIntegrityFinding.id.asc(),
)
.limit(limit)
.all()
)
return FileIntegrityFindingsResponse(
findings=[_finding_response(row) for row in rows]
)
@router.post(
"/findings/{finding_id}/recheck",
response_model=FileIntegrityActionResponse,
)
def recheck_finding(
finding_id: str,
payload: FileIntegrityActionRequest,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(require_scope("files:file:admin")),
) -> FileIntegrityActionResponse:
finding = _finding_for_tenant(
session,
finding_id,
principal.tenant_id,
for_update=True,
)
_assert_expected_revision(finding.revision, payload.expected_revision)
try:
result = recheck_integrity_finding(
session,
finding,
user_id=principal.user.id,
dry_run=payload.dry_run,
)
_audit_integrity_action(session, principal, result)
session.commit()
return _action_response(result)
except (FileStorageError, StorageBackendError) as exc:
session.rollback()
raise _integrity_http_error(exc) from exc
@router.post(
"/findings/{finding_id}/cleanup",
response_model=FileIntegrityActionResponse,
)
def cleanup_finding(
finding_id: str,
payload: FileIntegrityActionRequest,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(require_scope("files:file:admin")),
) -> FileIntegrityActionResponse:
finding = _finding_for_tenant(
session,
finding_id,
principal.tenant_id,
for_update=True,
)
_assert_expected_revision(finding.revision, payload.expected_revision)
try:
result = cleanup_orphan_finding(
session,
finding,
user_id=principal.user.id,
dry_run=payload.dry_run,
)
_audit_integrity_action(session, principal, result)
session.commit()
return _action_response(result)
except (FileStorageError, StorageBackendError) as exc:
session.rollback()
raise _integrity_http_error(exc) from exc
def _scan_for_tenant(
session: Session,
scan_id: str,
tenant_id: str,
*,
for_update: bool = False,
) -> FileIntegrityScan:
query = session.query(FileIntegrityScan).filter(FileIntegrityScan.id == scan_id)
if for_update:
query = query.populate_existing().with_for_update()
scan = query.one_or_none()
if scan is None or scan.tenant_id != tenant_id:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="Integrity scan not found",
)
return scan
def _finding_for_tenant(
session: Session,
finding_id: str,
tenant_id: str,
*,
for_update: bool = False,
) -> FileIntegrityFinding:
query = session.query(FileIntegrityFinding).filter(
FileIntegrityFinding.id == finding_id
)
if for_update:
query = query.populate_existing().with_for_update()
finding = query.one_or_none()
if finding is None or finding.tenant_id != tenant_id:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="Integrity finding not found",
)
return finding
def _assert_expected_revision(current: int, expected: int) -> None:
if current != expected:
raise HTTPException(
status_code=status.HTTP_409_CONFLICT,
detail=(
"The integrity record changed after it was loaded; reload before "
"performing this action."
),
)
def _audit_integrity_action(session, principal, result) -> None:
audit_from_principal(
session,
principal,
action=f"files.integrity.{result.action}",
object_type="file_integrity_finding",
object_id=result.finding.id,
details={
"scan_id": result.finding.scan_id,
"kind": result.finding.kind,
"dry_run": result.dry_run,
"changed": result.changed,
"storage_key_sha256": hashlib.sha256(
result.finding.storage_key.encode("utf-8")
).hexdigest(),
"inspection": result.inspection.kind
if result.inspection
else None,
},
)
def _scan_response(scan: FileIntegrityScan) -> FileIntegrityScanResponse:
return FileIntegrityScanResponse(
id=scan.id,
tenant_id=scan.tenant_id,
storage_backend=scan.storage_backend,
storage_prefix=scan.storage_prefix,
status=scan.status,
revision=scan.revision,
phase=scan.phase,
verify_checksums=scan.verify_checksums,
batch_size=scan.batch_size,
scanned_blob_count=scan.scanned_blob_count,
verified_blob_count=scan.verified_blob_count,
quarantined_blob_count=scan.quarantined_blob_count,
scanned_object_count=scan.scanned_object_count,
orphan_object_count=scan.orphan_object_count,
created_by_user_id=scan.created_by_user_id,
started_at=scan.started_at.isoformat() if scan.started_at else None,
completed_at=scan.completed_at.isoformat()
if scan.completed_at
else None,
last_error=scan.last_error,
created_at=scan.created_at.isoformat(),
updated_at=scan.updated_at.isoformat(),
)
def _finding_response(
finding: FileIntegrityFinding,
) -> FileIntegrityFindingResponse:
return FileIntegrityFindingResponse(
id=finding.id,
scan_id=finding.scan_id,
tenant_id=finding.tenant_id,
kind=finding.kind,
state=finding.state,
revision=finding.revision,
blob_id=finding.blob_id,
storage_key=finding.storage_key,
expected_size_bytes=finding.expected_size_bytes,
observed_size_bytes=finding.observed_size_bytes,
expected_checksum_sha256=finding.expected_checksum_sha256,
observed_checksum_sha256=finding.observed_checksum_sha256,
resolved_at=finding.resolved_at.isoformat()
if finding.resolved_at
else None,
resolved_by_user_id=finding.resolved_by_user_id,
created_at=finding.created_at.isoformat(),
updated_at=finding.updated_at.isoformat(),
)
def _action_response(result) -> FileIntegrityActionResponse:
return FileIntegrityActionResponse(
action=result.action,
changed=result.changed,
dry_run=result.dry_run,
finding=_finding_response(result.finding),
inspection_kind=result.inspection.kind if result.inspection else None,
inspection_valid=result.inspection.valid if result.inspection else None,
)
def _integrity_http_error(exc: Exception) -> HTTPException:
if isinstance(exc, FileStorageError):
return HTTPException(
status_code=status.HTTP_409_CONFLICT,
detail=str(exc),
)
return HTTPException(
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
detail="The configured file storage backend could not complete the integrity operation",
)
@@ -0,0 +1,167 @@
from __future__ import annotations
from typing import Literal
from fastapi import APIRouter, Depends, Query
from sqlalchemy.orm import Session
from govoplan_core.auth import ApiPrincipal, require_scope
from govoplan_files.backend.schemas import (
FileDeltaResponse,
FileListResponse,
)
from govoplan_core.db.session import get_session
from govoplan_files.backend.storage.files import (
count_assets_for_user,
list_assets_for_user,
list_assets_for_user_window,
)
from govoplan_files.backend.route_support import (
_asset_list_response,
_ensure_campaign_file_access,
_ensure_list_owner_access,
_is_admin,
)
from govoplan_files.backend.services.list_queries import (
FILES_LIST_CURSOR_SCOPE,
_cursor_page_size,
_file_cursor_values,
_files_delta_response,
_files_delta_watermark,
_files_list_fingerprint,
_full_file_delta_response,
_next_file_list_cursor,
)
router = APIRouter(prefix="/files", tags=["files"])
@router.get("/delta", response_model=FileDeltaResponse)
def files_delta(
owner_type: Literal["user", "group"] | None = None,
owner_id: str | None = None,
campaign_id: str | None = None,
path_prefix: str | None = None,
since: str | None = None,
limit: int = Query(default=500, ge=1, le=1000),
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(require_scope("files:file:read")),
):
_ensure_list_owner_access(session, principal, owner_type, owner_id)
_ensure_campaign_file_access(session, principal, campaign_id)
if since is None:
return _full_file_delta_response(
session,
principal=principal,
owner_type=owner_type,
owner_id=owner_id,
campaign_id=campaign_id,
path_prefix=path_prefix,
)
return _files_delta_response(
session,
principal=principal,
owner_type=owner_type,
owner_id=owner_id,
campaign_id=campaign_id,
path_prefix=path_prefix,
since=since,
limit=limit,
)
@router.get("", response_model=FileListResponse)
def list_files(
owner_type: Literal["user", "group"] | None = None,
owner_id: str | None = None,
campaign_id: str | None = None,
path_prefix: str | None = None,
campaign_usage: Literal["linked", "unlinked"] | None = None,
audit_relevant: bool | None = None,
page_size: int | None = Query(default=None, ge=1, le=1000),
cursor: str | None = None,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(require_scope("files:file:read")),
):
_ensure_list_owner_access(session, principal, owner_type, owner_id)
_ensure_campaign_file_access(session, principal, campaign_id)
watermark = _files_delta_watermark(session, principal.tenant_id)
total = count_assets_for_user(
session,
tenant_id=principal.tenant_id,
user_id=principal.user.id,
owner_type=owner_type,
owner_id=owner_id,
campaign_id=campaign_id,
path_prefix=path_prefix,
campaign_usage=campaign_usage,
audit_relevant=audit_relevant,
is_admin=_is_admin(principal),
)
effective_page_size = _cursor_page_size(FILES_LIST_CURSOR_SCOPE, cursor, page_size)
if effective_page_size is not None:
fingerprint = _files_list_fingerprint(
principal,
owner_type=owner_type,
owner_id=owner_id,
campaign_id=campaign_id,
path_prefix=path_prefix,
campaign_usage=campaign_usage,
audit_relevant=audit_relevant,
page_size=effective_page_size,
)
after_display_path, after_updated_at, after_id = _file_cursor_values(
cursor, fingerprint=fingerprint
)
assets, has_more = list_assets_for_user_window(
session,
tenant_id=principal.tenant_id,
user_id=principal.user.id,
owner_type=owner_type,
owner_id=owner_id,
campaign_id=campaign_id,
path_prefix=path_prefix,
campaign_usage=campaign_usage,
audit_relevant=audit_relevant,
is_admin=_is_admin(principal),
page_size=effective_page_size,
after_display_path=after_display_path,
after_updated_at=after_updated_at,
after_id=after_id,
)
return FileListResponse(
files=_asset_list_response(session, assets, include_shares=True),
total=total,
cursor=cursor,
next_cursor=_next_file_list_cursor(
principal,
assets,
owner_type=owner_type,
owner_id=owner_id,
campaign_id=campaign_id,
path_prefix=path_prefix,
campaign_usage=campaign_usage,
audit_relevant=audit_relevant,
page_size=effective_page_size,
has_more=has_more,
),
watermark=watermark,
)
assets = list_assets_for_user(
session,
tenant_id=principal.tenant_id,
user_id=principal.user.id,
owner_type=owner_type,
owner_id=owner_id,
campaign_id=campaign_id,
path_prefix=path_prefix,
campaign_usage=campaign_usage,
audit_relevant=audit_relevant,
is_admin=_is_admin(principal),
)
return FileListResponse(
files=_asset_list_response(session, assets, include_shares=True),
total=total,
watermark=watermark,
)
+330
View File
@@ -0,0 +1,330 @@
from __future__ import annotations
from datetime import datetime, timezone
from fastapi import APIRouter, Depends, HTTPException, Query, status
from sqlalchemy.orm import Session
from govoplan_core.api.v1.schemas import (
ReferenceOptionListResponse,
ReferenceOptionResponse,
)
from govoplan_core.auth import ApiPrincipal, require_scope
from govoplan_core.audit.logging import audit_from_principal
from govoplan_core.core.references import (
access_scope_reference_page,
access_scope_reference_provider_available,
)
from govoplan_core.db.session import get_session
from govoplan_files.backend.runtime import get_registry
from govoplan_files.backend.schemas import (
BulkFileShareRequest,
BulkFileShareResponse,
FileShareRequest,
FileShareResponse,
FileSharesResponse,
)
from govoplan_files.backend.storage.common import FileStorageError
from govoplan_files.backend.storage.files import (
current_file_share_for_target,
get_asset_for_share_management,
list_file_shares,
revoke_file_share,
share_file,
share_files,
)
from govoplan_files.backend.route_support import (
_file_share_response,
_http_error,
_is_admin,
)
router = APIRouter(prefix="/files", tags=["files"])
@router.get(
"/{file_id}/share-target-options",
response_model=ReferenceOptionListResponse,
)
def search_share_targets(
file_id: str,
target_type: str,
q: str = "",
selected: list[str] = Query(default=[]),
limit: int = Query(default=50, ge=1, le=200),
cursor: str | None = None,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(require_scope("files:file:share")),
) -> ReferenceOptionListResponse:
if target_type not in {"user", "group"}:
raise HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
detail="Share target type must be user or group",
)
try:
get_asset_for_share_management(
session,
tenant_id=principal.tenant_id,
user_id=principal.user.id,
asset_id=file_id,
is_admin=_is_admin(principal),
)
registry = get_registry()
page = access_scope_reference_page(
registry,
principal,
scope_type=target_type,
reference_kind="membership" if target_type == "user" else "group",
query=q,
selected_values=selected,
limit=limit,
cursor=cursor,
administrative=True,
session=session,
)
except ValueError as exc:
raise HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
detail=str(exc),
) from exc
except FileStorageError as exc:
raise _http_error(exc) from exc
return ReferenceOptionListResponse(
options=[
ReferenceOptionResponse(**option.to_dict()) for option in page.options
],
provider_available=access_scope_reference_provider_available(registry),
next_cursor=page.next_cursor,
has_more=page.has_more,
)
@router.get("/{file_id}/shares", response_model=FileSharesResponse)
def list_shares(
file_id: str,
include_inactive: bool = False,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(require_scope("files:file:share")),
):
try:
asset = get_asset_for_share_management(
session,
tenant_id=principal.tenant_id,
user_id=principal.user.id,
asset_id=file_id,
is_admin=_is_admin(principal),
)
return FileSharesResponse(
shares=[
_file_share_response(share)
for share in list_file_shares(
session,
tenant_id=principal.tenant_id,
asset_id=asset.id,
include_inactive=include_inactive,
)
]
)
except FileStorageError as exc:
raise _http_error(exc) from exc
@router.post("/{file_id}/shares", response_model=FileShareResponse)
def create_share(
file_id: str,
payload: FileShareRequest,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(require_scope("files:file:share")),
):
try:
asset = get_asset_for_share_management(
session,
tenant_id=principal.tenant_id,
user_id=principal.user.id,
asset_id=file_id,
is_admin=_is_admin(principal),
)
previous = current_file_share_for_target(
session,
tenant_id=principal.tenant_id,
asset_id=asset.id,
target_type=payload.target_type,
target_id=payload.target_id,
)
previous_permission = previous.permission if previous else None
previous_expiry = previous.expires_at if previous else None
share = share_file(
session,
tenant_id=principal.tenant_id,
asset=asset,
target_type=payload.target_type,
target_id=payload.target_id,
permission=payload.permission,
user_id=principal.user.id,
expires_at=payload.expires_at,
)
session.flush()
action = _share_audit_action(
existed=previous is not None,
previous_permission=previous_permission,
permission=share.permission,
previous_expiry=previous_expiry,
expiry=share.expires_at,
)
if action:
_audit_share_change(session, principal, share, action=action)
session.commit()
return _file_share_response(share)
except FileStorageError as exc:
session.rollback()
raise _http_error(exc) from exc
@router.post("/bulk-shares", response_model=BulkFileShareResponse)
def create_bulk_shares(
payload: BulkFileShareRequest,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(require_scope("files:file:share")),
):
try:
file_ids = list(dict.fromkeys(payload.file_ids))
assets = [
get_asset_for_share_management(
session,
tenant_id=principal.tenant_id,
user_id=principal.user.id,
asset_id=file_id,
is_admin=_is_admin(principal),
)
for file_id in file_ids
]
previous_by_asset = {
asset.id: current_file_share_for_target(
session,
tenant_id=principal.tenant_id,
asset_id=asset.id,
target_type=payload.target_type,
target_id=payload.target_id,
)
for asset in assets
}
previous_values = {
asset_id: (
share.permission if share else None,
share.expires_at if share else None,
)
for asset_id, share in previous_by_asset.items()
}
shares = share_files(
session,
tenant_id=principal.tenant_id,
assets=assets,
target_type=payload.target_type,
target_id=payload.target_id,
permission=payload.permission,
user_id=principal.user.id,
expires_at=payload.expires_at,
)
session.flush()
for share in shares:
previous_permission, previous_expiry = previous_values[share.file_asset_id]
action = _share_audit_action(
existed=previous_by_asset[share.file_asset_id] is not None,
previous_permission=previous_permission,
permission=share.permission,
previous_expiry=previous_expiry,
expiry=share.expires_at,
)
if action:
_audit_share_change(session, principal, share, action=action)
session.commit()
return BulkFileShareResponse(
shared_count=len(shares),
shares=[_file_share_response(share) for share in shares],
)
except FileStorageError as exc:
session.rollback()
raise _http_error(exc) from exc
@router.delete("/{file_id}/shares/{share_id}", response_model=FileShareResponse)
def revoke_share(
file_id: str,
share_id: str,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(require_scope("files:file:share")),
):
try:
asset = get_asset_for_share_management(
session,
tenant_id=principal.tenant_id,
user_id=principal.user.id,
asset_id=file_id,
is_admin=_is_admin(principal),
)
share, changed = revoke_file_share(
session,
tenant_id=principal.tenant_id,
asset_id=asset.id,
share_id=share_id,
user_id=principal.user.id,
)
if changed:
_audit_share_change(
session, principal, share, action="files.share.revoked"
)
session.commit()
return _file_share_response(share)
except FileStorageError as exc:
session.rollback()
raise _http_error(exc) from exc
def _share_audit_action(
*,
existed: bool,
previous_permission: str | None,
permission: str,
previous_expiry: datetime | None,
expiry: datetime | None,
) -> str | None:
if not existed:
return "files.share.granted"
permission_changed = previous_permission != permission
expiry_changed = _normalized_expiry(previous_expiry) != _normalized_expiry(expiry)
if permission_changed:
return "files.share.changed"
if expiry_changed:
return "files.share.expiry_changed"
return None
def _audit_share_change(
session: Session,
principal: ApiPrincipal,
share,
*,
action: str,
) -> None:
audit_from_principal(
session,
principal,
action=action,
object_type="file_share",
object_id=share.id,
details={
"file_asset_id": share.file_asset_id,
"target_type": share.target_type,
"target_id": share.target_id,
"permission": share.permission,
"expires_at": _normalized_expiry(share.expires_at),
},
)
def _normalized_expiry(value: datetime | None) -> str | None:
if value is None:
return None
if value.tzinfo is None:
value = value.replace(tzinfo=timezone.utc)
return value.astimezone(timezone.utc).isoformat()
+292
View File
@@ -0,0 +1,292 @@
from __future__ import annotations
import json
from typing import Literal
from fastapi import APIRouter, Depends, HTTPException, status
from sqlalchemy.orm import Session
from govoplan_core.auth import ApiPrincipal, require_scope
from govoplan_files.backend.change_tracking import (
FILES_CONNECTOR_SPACES_COLLECTION,
)
from govoplan_files.backend.schemas import (
FileConnectorSpaceCreateRequest,
FileConnectorSpaceResponse,
FileConnectorSpacesResponse,
FileConnectorSpaceUpdateRequest,
FileSpaceResponse,
FileSpacesResponse,
)
from govoplan_core.db.session import get_session
from govoplan_files.backend.storage.access import group_refs_for_ids, user_group_ids
from govoplan_files.backend.storage.common import FileStorageError
from govoplan_files.backend.storage.connector_spaces import (
connector_space_owner_id,
create_connector_space,
get_connector_space_for_user,
list_connector_spaces_for_user,
soft_delete_connector_space,
update_connector_space,
)
from govoplan_files.backend.storage.connector_policy import (
ConnectorPolicyDenied,
)
from govoplan_files.backend.route_support import (
FILES_CONNECTOR_SPACE_RESOURCE,
_connector_policy_error,
_connector_space_file_space_response,
_connector_space_policy_decision,
_connector_space_response,
_http_error,
_is_admin,
_record_connector_settings_change,
_visible_connector_profile,
)
router = APIRouter(prefix="/files", tags=["files"])
@router.get("/spaces", response_model=FileSpacesResponse)
def list_file_spaces(
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(require_scope("files:file:read")),
):
spaces = [
FileSpaceResponse(
id=f"user:{principal.user.id}",
label="My files",
owner_type="user",
owner_id=principal.user.id,
description="Files owned by your user account.",
)
]
group_ids = user_group_ids(
session,
tenant_id=principal.tenant_id,
user_id=principal.user.id,
include_admin_groups=_is_admin(principal),
)
if group_ids:
groups = group_refs_for_ids(tenant_id=principal.tenant_id, group_ids=group_ids)
spaces.extend(
FileSpaceResponse(
id=f"group:{group.id}",
label=f"{group.name} files",
owner_type="group",
owner_id=group.id,
description="Files owned by this group.",
)
for group in groups
)
connector_spaces = list_connector_spaces_for_user(
session,
tenant_id=principal.tenant_id,
user_id=principal.user.id,
is_admin=_is_admin(principal),
)
spaces.extend(
_connector_space_file_space_response(space) for space in connector_spaces
)
return FileSpacesResponse(spaces=spaces)
@router.get("/connector-spaces", response_model=FileConnectorSpacesResponse)
def list_file_connector_spaces(
owner_type: Literal["user", "group"] | None = None,
owner_id: str | None = None,
include_inactive: bool = False,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(require_scope("files:file:read")),
):
try:
spaces = list_connector_spaces_for_user(
session,
tenant_id=principal.tenant_id,
user_id=principal.user.id,
owner_type=owner_type,
owner_id=owner_id,
include_inactive=include_inactive and _is_admin(principal),
is_admin=_is_admin(principal),
)
return FileConnectorSpacesResponse(
spaces=[_connector_space_response(space) for space in spaces]
)
except FileStorageError as exc:
raise _http_error(exc) from exc
@router.post(
"/connector-spaces",
response_model=FileConnectorSpaceResponse,
status_code=status.HTTP_201_CREATED,
)
def create_file_connector_space(
payload: FileConnectorSpaceCreateRequest,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(require_scope("files:file:organize")),
):
if payload.owner_type == "group" and not payload.owner_id:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="owner_id is required for group connector spaces",
)
target_owner = payload.owner_id or principal.user.id
try:
profile = _visible_connector_profile(
session, principal, payload.connector_profile_id
)
decision = _connector_space_policy_decision(
profile,
library_id=payload.library_id,
remote_path=payload.remote_path,
operation="link",
)
if not decision.allowed:
raise ConnectorPolicyDenied(decision)
space = create_connector_space(
session,
tenant_id=principal.tenant_id,
owner_type=payload.owner_type,
owner_id=target_owner,
user_id=principal.user.id,
label=payload.label,
profile=profile,
library_id=payload.library_id,
remote_path=payload.remote_path,
sync_mode=payload.sync_mode,
metadata=payload.metadata,
is_admin=_is_admin(principal),
)
_record_connector_settings_change(
session,
collection=FILES_CONNECTOR_SPACES_COLLECTION,
resource_type=FILES_CONNECTOR_SPACE_RESOURCE,
resource_id=space.id,
operation="created",
principal=principal,
tenant_id=space.tenant_id,
payload={
"owner_type": space.owner_type,
"owner_id": connector_space_owner_id(space),
},
)
session.commit()
return _connector_space_response(space)
except ConnectorPolicyDenied as exc:
session.rollback()
raise _connector_policy_error(exc) from exc
except (FileStorageError, ValueError, json.JSONDecodeError) as exc:
session.rollback()
raise _http_error(exc) from exc
@router.patch("/connector-spaces/{space_id}", response_model=FileConnectorSpaceResponse)
def update_file_connector_space(
space_id: str,
payload: FileConnectorSpaceUpdateRequest,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(require_scope("files:file:organize")),
):
try:
space = get_connector_space_for_user(
session,
tenant_id=principal.tenant_id,
user_id=principal.user.id,
space_id=space_id,
include_inactive=_is_admin(principal),
is_admin=_is_admin(principal),
)
except FileStorageError as exc:
raise _http_error(exc, not_found=True) from exc
try:
if payload.library_id is not None or payload.remote_path is not None:
profile = _visible_connector_profile(
session, principal, space.connector_profile_id
)
decision = _connector_space_policy_decision(
profile,
library_id=payload.library_id
if payload.library_id is not None
else space.library_id,
remote_path=payload.remote_path
if payload.remote_path is not None
else space.remote_path,
operation="link",
)
if not decision.allowed:
raise ConnectorPolicyDenied(decision)
update_connector_space(
session,
space,
user_id=principal.user.id,
label=payload.label,
library_id=payload.library_id,
remote_path=payload.remote_path,
sync_mode=payload.sync_mode,
is_active=payload.is_active,
metadata=payload.metadata,
is_admin=_is_admin(principal),
)
_record_connector_settings_change(
session,
collection=FILES_CONNECTOR_SPACES_COLLECTION,
resource_type=FILES_CONNECTOR_SPACE_RESOURCE,
resource_id=space.id,
operation="updated",
principal=principal,
tenant_id=space.tenant_id,
payload={
"owner_type": space.owner_type,
"owner_id": connector_space_owner_id(space),
},
)
session.commit()
return _connector_space_response(space)
except ConnectorPolicyDenied as exc:
session.rollback()
raise _connector_policy_error(exc) from exc
except (FileStorageError, ValueError, json.JSONDecodeError) as exc:
session.rollback()
raise _http_error(exc) from exc
@router.delete(
"/connector-spaces/{space_id}", response_model=FileConnectorSpaceResponse
)
def delete_file_connector_space(
space_id: str,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(require_scope("files:file:organize")),
):
try:
space = get_connector_space_for_user(
session,
tenant_id=principal.tenant_id,
user_id=principal.user.id,
space_id=space_id,
include_inactive=True,
is_admin=_is_admin(principal),
)
soft_delete_connector_space(
session, space, user_id=principal.user.id, is_admin=_is_admin(principal)
)
_record_connector_settings_change(
session,
collection=FILES_CONNECTOR_SPACES_COLLECTION,
resource_type=FILES_CONNECTOR_SPACE_RESOURCE,
resource_id=space.id,
operation="deleted",
principal=principal,
tenant_id=space.tenant_id,
payload={
"owner_type": space.owner_type,
"owner_id": connector_space_owner_id(space),
},
)
session.commit()
return _connector_space_response(space)
except FileStorageError as exc:
session.rollback()
raise _http_error(exc, not_found=True) from exc
@@ -0,0 +1,213 @@
from __future__ import annotations
import tempfile
from fastapi import APIRouter, Depends
from fastapi.responses import FileResponse
from starlette.background import BackgroundTask
from sqlalchemy.orm import Session
from govoplan_core.auth import ApiPrincipal, require_scope
from govoplan_files.backend.schemas import (
ArchiveRequest,
PatternMatchResponse,
PatternResolveRequest,
PatternResolveResponse,
RenamePreviewItem,
RenameRequest,
RenameResponse,
TransferRequest,
TransferResponse,
_conflict_resolutions,
)
from govoplan_core.db.session import get_session
from govoplan_files.backend.storage.paths import (
UnsafeFilePathError,
filename_from_path,
normalize_logical_path,
)
from govoplan_files.backend.storage.archives import create_zip_file
from govoplan_files.backend.storage.common import FileStorageError
from govoplan_files.backend.storage.files import (
get_asset_for_user,
list_assets_for_user,
)
from govoplan_files.backend.storage.search import resolve_patterns
from govoplan_files.backend.storage.transfers import (
rename_selection,
transfer_selection,
)
from govoplan_files.backend.route_support import (
_asset_response,
_attachment_disposition,
_audit_connector_access,
_cleanup_temp_file,
_ensure_campaign_file_access,
_ensure_list_owner_access,
_http_error,
_is_admin,
)
router = APIRouter(prefix="/files", tags=["files"])
@router.post("/bulk-rename", response_model=RenameResponse)
def bulk_rename(
payload: RenameRequest,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(require_scope("files:file:organize")),
):
try:
plan = rename_selection(
session,
tenant_id=principal.tenant_id,
user_id=principal.user.id,
file_ids=payload.file_ids,
folder_paths=payload.folder_paths,
owner_type=payload.owner_type,
owner_id=payload.owner_id,
mode=payload.mode,
new_name=payload.new_name,
find=payload.find,
replacement=payload.replacement,
prefix=payload.prefix,
suffix=payload.suffix,
recursive=payload.recursive,
dry_run=payload.dry_run,
is_admin=_is_admin(principal),
)
if not payload.dry_run:
session.commit()
return RenameResponse(
dry_run=payload.dry_run,
items=[
RenamePreviewItem(
kind=item.kind,
id=item.id,
file_id=item.id if item.kind == "file" else None,
folder_path=item.old_path if item.kind == "folder" else None,
old_path=item.old_path,
new_path=item.new_path,
)
for item in plan
],
)
except (FileStorageError, UnsafeFilePathError, ValueError) as exc:
session.rollback()
raise _http_error(exc) from exc
@router.post("/transfer", response_model=TransferResponse)
def transfer_files(
payload: TransferRequest,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(require_scope("files:file:organize")),
):
try:
files, folders = transfer_selection(
session,
tenant_id=principal.tenant_id,
user_id=principal.user.id,
operation=payload.operation,
file_ids=payload.file_ids,
folder_paths=payload.folder_paths,
source_owner_type=payload.source_owner_type,
source_owner_id=payload.source_owner_id,
target_owner_type=payload.target_owner_type,
target_owner_id=payload.target_owner_id,
target_folder=payload.target_folder,
conflict_strategy=payload.conflict_strategy,
conflict_resolutions=_conflict_resolutions(payload.conflict_resolutions),
is_admin=_is_admin(principal),
)
session.commit()
return TransferResponse(
operation=payload.operation, files=files, folders=folders
)
except (FileStorageError, UnsafeFilePathError, ValueError) as exc:
session.rollback()
raise _http_error(exc) from exc
@router.post("/archive.zip")
def download_archive(
payload: ArchiveRequest,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(require_scope("files:file:download")),
):
try:
assets = [
get_asset_for_user(
session,
tenant_id=principal.tenant_id,
user_id=principal.user.id,
asset_id=file_id,
is_admin=_is_admin(principal),
)
for file_id in payload.file_ids
]
tmp = tempfile.NamedTemporaryFile(
prefix="govoplan-files-", suffix=".zip", delete=False
)
tmp_path = tmp.name
tmp.close()
try:
create_zip_file(session, assets, tmp_path)
except Exception:
_cleanup_temp_file(tmp_path)
raise
_audit_connector_access(session, principal, assets, operation="archive")
except FileStorageError as exc:
raise _http_error(exc) from exc
filename = filename_from_path(
normalize_logical_path(payload.filename, fallback_filename="files.zip")
)
headers = {"Content-Disposition": _attachment_disposition(filename)}
return FileResponse(
tmp_path,
media_type="application/zip",
headers=headers,
background=BackgroundTask(_cleanup_temp_file, tmp_path),
)
@router.post("/resolve-patterns", response_model=PatternResolveResponse)
def resolve_file_patterns(
payload: PatternResolveRequest,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(require_scope("files:file:read")),
):
_ensure_list_owner_access(session, principal, payload.owner_type, payload.owner_id)
_ensure_campaign_file_access(session, principal, payload.campaign_id)
try:
assets = list_assets_for_user(
session,
tenant_id=principal.tenant_id,
user_id=principal.user.id,
owner_type=payload.owner_type,
owner_id=payload.owner_id,
campaign_id=payload.campaign_id,
path_prefix=payload.path_prefix,
is_admin=_is_admin(principal),
)
resolved, unmatched = resolve_patterns(
assets,
payload.patterns,
base_path=payload.path_prefix,
case_sensitive=payload.case_sensitive,
)
return PatternResolveResponse(
patterns=[
PatternMatchResponse(
pattern=item.pattern,
matches=[_asset_response(session, asset) for asset in item.matches],
)
for item in resolved
],
unmatched=[_asset_response(session, asset) for asset in unmatched]
if payload.include_unmatched
else [],
)
except (FileStorageError, UnsafeFilePathError, ValueError) as exc:
raise _http_error(exc) from exc
@@ -0,0 +1,487 @@
from __future__ import annotations
import hashlib
import hmac
import json
from datetime import UTC, datetime, timedelta
from typing import Literal
from fastapi import APIRouter, Depends, File as FastAPIFile, Form, UploadFile
from sqlalchemy.orm import Session
from govoplan_core.auth import ApiPrincipal, require_scope
from govoplan_core.security.secrets import (
TransientPayloadError,
open_transient_payload,
seal_transient_payload,
)
from govoplan_files.backend.schemas import (
ArchiveEntryResponse,
ArchivePreviewResponse,
ConflictResolutionRequest,
FileUploadResponse,
_conflict_resolutions,
)
from govoplan_files.backend.db.models import FileAsset
from govoplan_core.db.session import get_session
from govoplan_files.backend.runtime import settings
from govoplan_files.backend.storage.paths import (
UnsafeFilePathError,
normalize_folder,
)
from govoplan_files.backend.storage.archives import (
archive_format_for_filename,
extract_archive_upload,
extract_zip_upload,
inspect_archive,
)
from govoplan_files.backend.storage.common import FileStorageError
from govoplan_files.backend.storage.connector_policy import (
ConnectorPolicyDenied,
)
from govoplan_files.backend.storage.files import (
create_file_asset,
)
from govoplan_files.backend.route_support import (
_asset_response,
_audit_connector_imports,
_cleanup_temp_file,
_connector_policy_error,
_enforce_connector_policy,
_http_error,
_is_admin,
_read_limited_upload,
_source_metadata_from_form,
_spool_limited_upload_to_temp,
)
router = APIRouter(prefix="/files", tags=["files"])
_ARCHIVE_PREVIEW_PURPOSE = "files.archive-preview.v1"
def _archive_suffix(filename: str) -> str:
lowered = filename.casefold()
for suffix in (".tar.bz2", ".tar.gz", ".tar.xz", ".tbz2", ".tgz", ".txz", ".tar", ".zip"):
if lowered.endswith(suffix):
return suffix
return ".archive"
def _archive_sha256(path: str) -> str:
digest = hashlib.sha256()
with open(path, "rb") as source:
while chunk := source.read(1024 * 1024):
digest.update(chunk)
return digest.hexdigest()
def _archive_selected_paths(value: str) -> list[str]:
parsed = json.loads(value)
if not isinstance(parsed, list) or not parsed:
raise FileStorageError("Select at least one archive file or folder")
if len(parsed) > settings.file_archive_max_entries:
raise FileStorageError(
"Archive selection exceeds the configured entry limit"
)
selected: list[str] = []
for item in parsed:
if not isinstance(item, str) or not item.strip():
raise FileStorageError("Archive selection contains an invalid path")
if len(item) > 4096:
raise FileStorageError("Archive selection path is too long")
selected.append(item)
return selected
def _validate_archive_preview_token(
token: str,
*,
tenant_id: str,
user_id: str,
owner_type: str,
owner_id: str,
path: str,
campaign_id: str | None,
archive_format: str,
archive_sha256: str,
) -> None:
payload = open_transient_payload(
token,
ttl_seconds=settings.file_archive_preview_ttl_seconds,
)
expected = {
"purpose": _ARCHIVE_PREVIEW_PURPOSE,
"tenant_id": tenant_id,
"user_id": user_id,
"owner_type": owner_type,
"owner_id": owner_id,
"path": path,
"campaign_id": campaign_id or "",
"archive_format": archive_format,
}
if any(payload.get(key) != value for key, value in expected.items()):
raise FileStorageError(
"Archive preview does not match this upload destination"
)
token_digest = payload.get("archive_sha256")
if not isinstance(token_digest, str) or not hmac.compare_digest(
token_digest, archive_sha256
):
raise FileStorageError(
"Archive contents changed after preview; preview it again"
)
@router.post("/archive-preview", response_model=ArchivePreviewResponse)
def preview_archive_upload(
file: UploadFile = FastAPIFile(...),
owner_type: Literal["user", "group"] = Form(default="user"),
owner_id: str | None = Form(default=None),
path: str = Form(default=""),
campaign_id: str | None = Form(default=None),
password: str | None = Form(default=None),
principal: ApiPrincipal = Depends(require_scope("files:file:upload")),
):
target_owner = owner_id or principal.user.id
archive_path: str | None = None
filename = file.filename or "archive"
try:
archive_format = archive_format_for_filename(filename)
archive_path = _spool_limited_upload_to_temp(
file,
max_bytes=settings.file_upload_zip_max_bytes,
suffix=_archive_suffix(filename),
)
inspection = inspect_archive(
archive_path,
filename=filename,
password=password,
max_entries=settings.file_archive_max_entries,
max_expanded_bytes=settings.file_archive_max_expanded_bytes,
max_expansion_ratio=settings.file_archive_max_expansion_ratio,
)
digest = _archive_sha256(archive_path)
normalized_path = normalize_folder(path)
preview_token = seal_transient_payload(
{
"purpose": _ARCHIVE_PREVIEW_PURPOSE,
"tenant_id": principal.tenant_id,
"user_id": principal.user.id,
"owner_type": owner_type,
"owner_id": target_owner,
"path": normalized_path,
"campaign_id": campaign_id or "",
"archive_format": archive_format,
"archive_sha256": digest,
}
)
expires_at = datetime.now(UTC) + timedelta(
seconds=settings.file_archive_preview_ttl_seconds
)
return ArchivePreviewResponse(
preview_token=preview_token,
archive_format=inspection.archive_format,
entries=[
ArchiveEntryResponse(
path=entry.path,
kind=entry.kind,
size_bytes=entry.size_bytes,
compressed_size_bytes=entry.compressed_size_bytes,
encrypted=entry.encrypted,
)
for entry in inspection.entries
],
file_count=inspection.file_count,
directory_count=inspection.directory_count,
expanded_size_bytes=inspection.expanded_size_bytes,
compressed_size_bytes=inspection.compressed_size_bytes,
requires_password=inspection.requires_password,
password_verified=inspection.password_verified,
expires_at=expires_at.isoformat(),
)
except (FileStorageError, UnsafeFilePathError, ValueError) as exc:
raise _http_error(exc) from exc
finally:
if archive_path:
_cleanup_temp_file(archive_path)
@router.post("/archive-confirm", response_model=FileUploadResponse)
def confirm_archive_upload(
file: UploadFile = FastAPIFile(...),
preview_token: str = Form(...),
selected_paths_json: str = Form(...),
owner_type: Literal["user", "group"] = Form(default="user"),
owner_id: str | None = Form(default=None),
path: str = Form(default=""),
campaign_id: str | None = Form(default=None),
password: str | None = Form(default=None),
conflict_strategy: Literal["reject", "overwrite", "rename"] = Form(
default="reject"
),
conflict_resolutions_json: str | None = Form(default=None),
source_provenance_json: str | None = Form(default=None),
source_revision: str | None = Form(default=None),
connector_policy_json: str | None = Form(default=None),
encryption_vault_id: str | None = Form(default=None),
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(require_scope("files:file:upload")),
):
target_owner = owner_id or principal.user.id
archive_path: str | None = None
filename = file.filename or "archive"
try:
raw_resolutions = (
json.loads(conflict_resolutions_json)
if conflict_resolutions_json
else []
)
upload_resolutions = _conflict_resolutions(
[ConflictResolutionRequest(**item) for item in raw_resolutions]
)
selected_paths = _archive_selected_paths(selected_paths_json)
_enforce_connector_policy(
source_provenance_json,
connector_policy_json,
operation="import",
)
metadata = _source_metadata_from_form(
source_provenance_json, source_revision
)
archive_format = archive_format_for_filename(filename)
normalized_path = normalize_folder(path)
archive_path = _spool_limited_upload_to_temp(
file,
max_bytes=settings.file_upload_zip_max_bytes,
suffix=_archive_suffix(filename),
)
digest = _archive_sha256(archive_path)
_validate_archive_preview_token(
preview_token,
tenant_id=principal.tenant_id,
user_id=principal.user.id,
owner_type=owner_type,
owner_id=target_owner,
path=normalized_path,
campaign_id=campaign_id,
archive_format=archive_format,
archive_sha256=digest,
)
extracted = extract_archive_upload(
session,
tenant_id=principal.tenant_id,
owner_type=owner_type,
owner_id=target_owner,
user_id=principal.user.id,
archive_data=archive_path,
filename=filename,
folder=normalized_path,
campaign_id=campaign_id,
selected_paths=selected_paths,
password=password,
conflict_strategy=conflict_strategy,
conflict_resolutions=upload_resolutions,
metadata=metadata,
is_admin=_is_admin(principal),
encryption_vault_id=encryption_vault_id,
max_entries=settings.file_archive_max_entries,
max_file_bytes=settings.file_upload_max_bytes,
max_expanded_bytes=settings.file_archive_max_expanded_bytes,
max_expansion_ratio=settings.file_archive_max_expansion_ratio,
)
uploaded_assets = [item.asset for item in extracted]
_audit_connector_imports(session, principal, uploaded_assets)
session.commit()
return FileUploadResponse(
files=[
_asset_response(session, asset, include_shares=True)
for asset in uploaded_assets
]
)
except ConnectorPolicyDenied as exc:
session.rollback()
raise _connector_policy_error(exc) from exc
except (
FileStorageError,
TransientPayloadError,
UnsafeFilePathError,
ValueError,
json.JSONDecodeError,
) as exc:
session.rollback()
raise _http_error(exc) from exc
finally:
if archive_path:
_cleanup_temp_file(archive_path)
@router.post("/upload", response_model=FileUploadResponse)
def upload_files(
files: list[UploadFile] = FastAPIFile(...),
owner_type: Literal["user", "group"] = Form(default="user"),
owner_id: str | None = Form(default=None),
path: str = Form(default=""),
campaign_id: str | None = Form(default=None),
unpack_zip: bool = Form(default=False),
conflict_strategy: Literal["reject", "overwrite", "rename"] = Form(
default="reject"
),
conflict_resolutions_json: str | None = Form(default=None),
source_provenance_json: str | None = Form(default=None),
source_revision: str | None = Form(default=None),
connector_policy_json: str | None = Form(default=None),
encryption_vault_id: str | None = Form(default=None),
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(require_scope("files:file:upload")),
):
target_owner = owner_id or principal.user.id
uploaded_assets: list[FileAsset] = []
try:
raw_resolutions = (
json.loads(conflict_resolutions_json) if conflict_resolutions_json else []
)
upload_resolutions = _conflict_resolutions(
[ConflictResolutionRequest(**item) for item in raw_resolutions]
)
_enforce_connector_policy(
source_provenance_json, connector_policy_json, operation="import"
)
metadata = _source_metadata_from_form(source_provenance_json, source_revision)
for upload in files:
filename = upload.filename or "file"
content_type = upload.content_type or None
upload_limit = (
settings.file_upload_zip_max_bytes
if unpack_zip and filename.lower().endswith(".zip")
else settings.file_upload_max_bytes
)
if unpack_zip and filename.lower().endswith(".zip"):
zip_path = _spool_limited_upload_to_temp(
upload, max_bytes=upload_limit, suffix=".zip"
)
try:
extracted = extract_zip_upload(
session,
tenant_id=principal.tenant_id,
owner_type=owner_type,
owner_id=target_owner,
user_id=principal.user.id,
zip_data=zip_path,
folder=path,
campaign_id=campaign_id,
conflict_strategy=conflict_strategy,
conflict_resolutions=upload_resolutions,
metadata=metadata,
is_admin=_is_admin(principal),
encryption_vault_id=encryption_vault_id,
max_file_bytes=settings.file_upload_max_bytes,
max_total_bytes=settings.file_upload_zip_max_bytes,
)
finally:
_cleanup_temp_file(zip_path)
uploaded_assets.extend(item.asset for item in extracted)
continue
data = _read_limited_upload(upload, max_bytes=upload_limit)
stored = create_file_asset(
session,
tenant_id=principal.tenant_id,
owner_type=owner_type,
owner_id=target_owner,
user_id=principal.user.id,
filename=filename,
data=data,
folder=path,
content_type=content_type,
campaign_id=campaign_id,
conflict_strategy=conflict_strategy,
conflict_resolutions=upload_resolutions,
metadata=metadata,
is_admin=_is_admin(principal),
encryption_vault_id=encryption_vault_id,
)
uploaded_assets.append(stored.asset)
_audit_connector_imports(session, principal, uploaded_assets)
session.commit()
except ConnectorPolicyDenied as exc:
session.rollback()
raise _connector_policy_error(exc) from exc
except (FileStorageError, UnsafeFilePathError, ValueError) as exc:
session.rollback()
raise _http_error(exc) from exc
return FileUploadResponse(
files=[
_asset_response(session, asset, include_shares=True)
for asset in uploaded_assets
]
)
@router.post("/upload-zip", response_model=FileUploadResponse)
def upload_zip(
file: UploadFile = FastAPIFile(...),
owner_type: Literal["user", "group"] = Form(default="user"),
owner_id: str | None = Form(default=None),
path: str = Form(default=""),
campaign_id: str | None = Form(default=None),
conflict_strategy: Literal["reject", "overwrite", "rename"] = Form(
default="reject"
),
conflict_resolutions_json: str | None = Form(default=None),
source_provenance_json: str | None = Form(default=None),
source_revision: str | None = Form(default=None),
connector_policy_json: str | None = Form(default=None),
encryption_vault_id: str | None = Form(default=None),
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(require_scope("files:file:upload")),
):
target_owner = owner_id or principal.user.id
zip_path: str | None = None
try:
raw_resolutions = (
json.loads(conflict_resolutions_json) if conflict_resolutions_json else []
)
upload_resolutions = _conflict_resolutions(
[ConflictResolutionRequest(**item) for item in raw_resolutions]
)
_enforce_connector_policy(
source_provenance_json, connector_policy_json, operation="import"
)
metadata = _source_metadata_from_form(source_provenance_json, source_revision)
zip_path = _spool_limited_upload_to_temp(
file, max_bytes=settings.file_upload_zip_max_bytes, suffix=".zip"
)
extracted = extract_zip_upload(
session,
tenant_id=principal.tenant_id,
owner_type=owner_type,
owner_id=target_owner,
user_id=principal.user.id,
zip_data=zip_path,
folder=path,
campaign_id=campaign_id,
conflict_strategy=conflict_strategy,
conflict_resolutions=upload_resolutions,
metadata=metadata,
is_admin=_is_admin(principal),
encryption_vault_id=encryption_vault_id,
max_file_bytes=settings.file_upload_max_bytes,
max_total_bytes=settings.file_upload_zip_max_bytes,
)
_audit_connector_imports(session, principal, [item.asset for item in extracted])
session.commit()
except ConnectorPolicyDenied as exc:
session.rollback()
raise _connector_policy_error(exc) from exc
except (FileStorageError, UnsafeFilePathError, ValueError) as exc:
session.rollback()
raise _http_error(exc) from exc
finally:
if zip_path:
_cleanup_temp_file(zip_path)
return FileUploadResponse(
files=[
_asset_response(session, item.asset, include_shares=True)
for item in extracted
]
)
+107
View File
@@ -1,5 +1,6 @@
from __future__ import annotations
from datetime import datetime
from typing import Any, Literal
from pydantic import BaseModel, Field
@@ -73,11 +74,93 @@ class FileConnectorSpacesResponse(BaseModel):
class FileShareResponse(BaseModel):
id: str
file_asset_id: str
target_type: str
target_id: str
permission: str
created_by_user_id: str | None = None
created_at: str
expires_at: str | None = None
revoked_at: str | None = None
revoked_by_user_id: str | None = None
active: bool
class FileSharesResponse(BaseModel):
shares: list[FileShareResponse] = Field(default_factory=list)
class FileIntegrityScanCreateRequest(BaseModel):
verify_checksums: bool = True
batch_size: int = Field(default=100, ge=1, le=1000)
class FileIntegrityScanResponse(BaseModel):
id: str
tenant_id: str
storage_backend: str
storage_prefix: str
status: str
revision: int
phase: str
verify_checksums: bool
batch_size: int
scanned_blob_count: int
verified_blob_count: int
quarantined_blob_count: int
scanned_object_count: int
orphan_object_count: int
created_by_user_id: str | None = None
started_at: str | None = None
completed_at: str | None = None
last_error: str | None = None
created_at: str
updated_at: str
class FileIntegrityScansResponse(BaseModel):
scans: list[FileIntegrityScanResponse] = Field(default_factory=list)
class FileIntegrityFindingResponse(BaseModel):
id: str
scan_id: str
tenant_id: str
kind: str
state: str
revision: int
blob_id: str | None = None
storage_key: str
expected_size_bytes: int | None = None
observed_size_bytes: int | None = None
expected_checksum_sha256: str | None = None
observed_checksum_sha256: str | None = None
resolved_at: str | None = None
resolved_by_user_id: str | None = None
created_at: str
updated_at: str
class FileIntegrityFindingsResponse(BaseModel):
findings: list[FileIntegrityFindingResponse] = Field(default_factory=list)
class FileIntegrityActionRequest(BaseModel):
dry_run: bool = True
expected_revision: int = Field(ge=1)
class FileIntegrityScanRunRequest(BaseModel):
expected_revision: int = Field(ge=1)
class FileIntegrityActionResponse(BaseModel):
action: str
changed: bool
dry_run: bool
finding: FileIntegrityFindingResponse
inspection_kind: str | None = None
inspection_valid: bool | None = None
class FileSourceProvenance(BaseModel):
@@ -154,6 +237,7 @@ class FileFolderDeleteResponse(BaseModel):
class FileListResponse(BaseModel):
files: list[FileAssetResponse]
total: int
cursor: str | None = None
next_cursor: str | None = None
watermark: str | None = None
@@ -172,6 +256,27 @@ class FileUploadResponse(BaseModel):
files: list[FileAssetResponse]
class ArchiveEntryResponse(BaseModel):
path: str
kind: Literal["file", "directory"]
size_bytes: int
compressed_size_bytes: int | None = None
encrypted: bool = False
class ArchivePreviewResponse(BaseModel):
preview_token: str
archive_format: str
entries: list[ArchiveEntryResponse] = Field(default_factory=list)
file_count: int
directory_count: int
expanded_size_bytes: int
compressed_size_bytes: int
requires_password: bool
password_verified: bool
expires_at: str
class FileConnectorPolicySource(BaseModel):
scope_type: Literal["system", "tenant", "user", "group", "campaign"] = "system"
scope_id: str | None = None
@@ -452,6 +557,7 @@ class FileShareRequest(BaseModel):
target_type: Literal["user", "group", "campaign", "tenant"]
target_id: str
permission: Literal["read", "write", "manage"] = "read"
expires_at: datetime | None = None
class BulkFileShareRequest(BaseModel):
@@ -459,6 +565,7 @@ class BulkFileShareRequest(BaseModel):
target_type: Literal["user", "group", "campaign", "tenant"]
target_id: str
permission: Literal["read", "write", "manage"] = "read"
expires_at: datetime | None = None
class BulkFileShareResponse(BaseModel):
+329
View File
@@ -0,0 +1,329 @@
from __future__ import annotations
from collections.abc import Mapping, Sequence
from urllib.parse import quote
from sqlalchemy import func, or_, select
from sqlalchemy.orm import Session
from govoplan_core.auth import ApiPrincipal
from govoplan_core.core.events import PlatformEvent
from govoplan_core.core.modules import ModuleContext
from govoplan_core.core.search import (
SearchAuthorizationRequest,
SearchBackfillPage,
SearchBackfillRequest,
SearchDocument,
SearchIndexChange,
SearchResourceReference,
SearchResourceType,
)
from govoplan_files.backend.db.models import FileAsset, FileFolder, FileShare
from govoplan_files.backend.storage.share_state import effective_file_share_clause
PROVIDER_ID = "files.objects"
RESOURCE_MODELS = {
"file": FileAsset,
"folder": FileFolder,
}
READ_SCOPE = "files:file:read"
ADMIN_SCOPE = "files:file:admin"
class FilesSearchSource:
def resource_types(self) -> Sequence[SearchResourceType]:
return (
SearchResourceType(
provider_id=PROVIDER_ID,
module_id="files",
resource_type="file",
label="Files",
requires_authorization_recheck=True,
),
SearchResourceType(
provider_id=PROVIDER_ID,
module_id="files",
resource_type="folder",
label="File folders",
requires_authorization_recheck=True,
),
)
def backfill(
self,
session: object,
*,
request: SearchBackfillRequest,
) -> SearchBackfillPage:
db = _session(session)
model = _model(request.provider_id, request.resource_type)
statement = select(model).where(
model.tenant_id == request.tenant_id,
model.deleted_at.is_(None),
)
if request.cursor:
statement = statement.where(model.id > request.cursor)
rows = list(
db.scalars(
statement.order_by(model.id).limit(request.limit + 1)
)
)
has_more = len(rows) > request.limit
selected = rows[: request.limit]
shares = (
_shares_by_asset(db, selected)
if request.resource_type == "file"
else {}
)
high_watermark = db.scalar(
select(func.max(model.updated_at)).where(
model.tenant_id == request.tenant_id,
model.deleted_at.is_(None),
)
)
return SearchBackfillPage(
documents=tuple(
_document(
row,
resource_type=request.resource_type,
shares=shares.get(row.id, ()),
)
for row in selected
),
next_cursor=selected[-1].id if has_more and selected else None,
complete=not has_more,
high_watermark=(
high_watermark.isoformat()
if high_watermark is not None
else None
),
)
def authorize(
self,
session: object,
principal: object,
*,
requests: Sequence[SearchAuthorizationRequest],
) -> Mapping[str, bool]:
decisions = {item.reference.key: False for item in requests}
if not isinstance(principal, ApiPrincipal) or not (
principal.has(READ_SCOPE) or principal.has(ADMIN_SCOPE)
):
return decisions
db = _session(session)
for request in requests:
reference = request.reference
if (
reference.tenant_id != principal.tenant_id
or reference.module_id != "files"
or reference.resource_type not in RESOURCE_MODELS
):
continue
decisions[reference.key] = _can_read(
db,
principal,
reference=reference,
)
return decisions
def index_changes_for_event(
self,
session: object,
*,
event: PlatformEvent,
delivery_key: str,
) -> Sequence[SearchIndexChange]:
if (
event.module_id != "files"
or event.tenant is None
or event.resource is None
or event.resource.id is None
or event.resource.type not in RESOURCE_MODELS
):
return ()
db = _session(session)
reference = SearchResourceReference(
tenant_id=event.tenant.id,
module_id="files",
resource_type=event.resource.type,
resource_id=event.resource.id,
)
model = RESOURCE_MODELS[event.resource.type]
row = db.get(model, event.resource.id)
deleted = row is None or row.tenant_id != event.tenant.id or row.deleted_at is not None
cursor = event.event_id
document = None
if not deleted:
shares = (
tuple(_active_shares(db, row.id))
if event.resource.type == "file"
else ()
)
document = _document(
row,
resource_type=event.resource.type,
shares=shares,
change_cursor=cursor,
)
return (
SearchIndexChange(
change_id=f"{delivery_key}:{PROVIDER_ID}:{event.resource.type}",
provider_id=PROVIDER_ID,
kind="delete" if deleted else "upsert",
reference=reference,
source_revision=(
document.source_revision if document is not None else cursor
),
cursor=cursor,
document=document,
occurred_at=event.occurred_at,
),
)
def create_files_search_source(_context: ModuleContext) -> FilesSearchSource:
return FilesSearchSource()
def _model(provider_id: str, resource_type: str):
if provider_id != PROVIDER_ID or resource_type not in RESOURCE_MODELS:
raise ValueError("Unsupported Files search source.")
return RESOURCE_MODELS[resource_type]
def _document(
row: FileAsset | FileFolder,
*,
resource_type: str,
shares: Sequence[FileShare] = (),
change_cursor: str | None = None,
) -> SearchDocument:
is_file = isinstance(row, FileAsset)
title = row.filename if is_file else (row.path.rsplit("/", 1)[-1] or row.path)
path = row.display_path if is_file else row.path
owner_id = row.owner_user_id if row.owner_type == "user" else row.owner_group_id
tokens = [f"scope:{READ_SCOPE}", f"scope:{ADMIN_SCOPE}"]
if owner_id:
tokens.append(
f"membership:{owner_id}"
if row.owner_type == "user"
else f"group:{owner_id}"
)
for share in shares:
prefix = "membership" if share.target_type == "user" else share.target_type
if prefix in {"membership", "group", "tenant"}:
tokens.append(f"{prefix}:{share.target_id}")
updated_at = row.updated_at or row.created_at
revision = (
f"{row.current_version_id or 'none'}:{updated_at.isoformat()}"
if is_file
else updated_at.isoformat()
)
return SearchDocument(
tenant_id=row.tenant_id,
module_id="files",
provider_id=PROVIDER_ID,
resource_type=resource_type,
resource_id=row.id,
title=title,
url=f"/files?{resource_type}Id={quote(row.id, safe='')}",
summary=((row.description or "") if is_file else path)[:4000] or None,
body=" ".join(
value for value in (path, row.description if is_file else None) if value
)[:200_000],
keywords=(path[:200], row.owner_type[:200]),
visibility="restricted",
acl_tokens=tuple(dict.fromkeys(tokens)),
metadata={
"path": path,
"owner_type": row.owner_type,
"current_version_id": row.current_version_id if is_file else None,
},
source_revision=revision,
change_cursor=change_cursor,
source_updated_at=updated_at,
requires_authorization_recheck=True,
)
def _can_read(
session: Session,
principal: ApiPrincipal,
*,
reference: SearchResourceReference,
) -> bool:
model = RESOURCE_MODELS[reference.resource_type]
row = session.get(model, reference.resource_id)
if row is None or row.tenant_id != principal.tenant_id or row.deleted_at is not None:
return False
if principal.has(ADMIN_SCOPE):
return True
user_id = str(getattr(principal.user, "id", "") or principal.membership_id or "")
if row.owner_type == "user" and row.owner_user_id == user_id:
return True
if row.owner_type == "group" and row.owner_group_id in principal.group_ids:
return True
if reference.resource_type != "file":
return False
target_clauses = [
(FileShare.target_type == "user") & (FileShare.target_id == user_id),
(FileShare.target_type == "tenant")
& (FileShare.target_id == principal.tenant_id),
]
if principal.group_ids:
target_clauses.append(
(FileShare.target_type == "group")
& (FileShare.target_id.in_(tuple(principal.group_ids)))
)
return session.scalar(
select(FileShare.id).where(
FileShare.tenant_id == principal.tenant_id,
FileShare.file_asset_id == row.id,
effective_file_share_clause(),
or_(*target_clauses),
).limit(1)
) is not None
def _active_shares(session: Session, asset_id: str) -> Sequence[FileShare]:
return tuple(
session.scalars(
select(FileShare).where(
FileShare.file_asset_id == asset_id,
effective_file_share_clause(),
)
)
)
def _shares_by_asset(
session: Session,
rows: Sequence[FileAsset | FileFolder],
) -> dict[str, tuple[FileShare, ...]]:
asset_ids = [row.id for row in rows if isinstance(row, FileAsset)]
grouped: dict[str, list[FileShare]] = {asset_id: [] for asset_id in asset_ids}
if not asset_ids:
return {}
for share in session.scalars(
select(FileShare).where(
FileShare.file_asset_id.in_(asset_ids),
effective_file_share_clause(),
)
):
grouped.setdefault(share.file_asset_id, []).append(share)
return {key: tuple(value) for key, value in grouped.items()}
def _session(value: object) -> Session:
if not isinstance(value, Session):
raise TypeError("Files search requires a SQLAlchemy session.")
return value
__all__ = [
"FilesSearchSource",
"PROVIDER_ID",
"create_files_search_source",
]
@@ -0,0 +1 @@
"""Query and response assembly services used by Files routes."""
@@ -0,0 +1,292 @@
from __future__ import annotations
from typing import Literal
from sqlalchemy.orm import Session
from govoplan_core.auth import ApiPrincipal
from govoplan_core.api.v1.schemas import DeltaDeletedItem
from govoplan_core.core.change_sequence import (
ChangeSequenceEntry,
)
from govoplan_files.backend.change_tracking import (
FILES_CONNECTOR_CREDENTIALS_COLLECTION,
FILES_CONNECTOR_POLICIES_COLLECTION,
FILES_CONNECTOR_PROFILES_COLLECTION,
FILES_CONNECTOR_SPACES_COLLECTION,
)
from govoplan_files.backend.schemas import (
FileConnectorCredentialResponse,
FileConnectorSettingsDeltaResponse,
FileConnectorPolicyResponse,
FileConnectorProfileResponse,
)
from govoplan_files.backend.db.models import FileConnectorSpace
from govoplan_files.backend.storage.connector_credential_store import (
ConnectorCredential,
)
from govoplan_files.backend.storage.connector_profiles import ConnectorProfile
from govoplan_files.backend.storage.connector_policy_store import (
connector_policy_response,
)
from govoplan_files.backend.route_support import (
FILES_CONNECTOR_CREDENTIAL_RESOURCE,
FILES_CONNECTOR_PROFILE_RESOURCE,
FILES_CONNECTOR_SPACE_RESOURCE,
_can_read_disabled_connector_profiles,
_connector_deleted_item,
_connector_space_response,
_ensure_campaign_file_access,
_file_connector_settings_response_watermark,
_file_connector_settings_watermark,
_visible_connector_credentials,
_visible_connector_profiles,
_visible_connector_spaces,
)
def _full_file_connector_settings_delta_response(
session: Session,
principal: ApiPrincipal,
*,
scope_type: str,
scope_id: str | None,
provider: str | None,
campaign_id: str | None,
include_disabled: bool,
include_inactive: bool,
owner_type: Literal["user", "group"] | None,
owner_id: str | None,
) -> FileConnectorSettingsDeltaResponse:
if campaign_id:
_ensure_campaign_file_access(session, principal, campaign_id)
profiles = _visible_connector_profiles(
session,
principal,
provider=provider,
campaign_id=campaign_id,
include_disabled=include_disabled
and _can_read_disabled_connector_profiles(principal),
include_admin_scopes=_can_read_disabled_connector_profiles(principal),
include_effective_policy=False,
)
credentials = _visible_connector_credentials(
session,
principal,
provider=provider,
include_disabled=include_disabled,
)
spaces = _visible_connector_spaces(
session,
principal,
owner_type=owner_type,
owner_id=owner_id,
include_inactive=include_inactive,
)
return FileConnectorSettingsDeltaResponse(
profiles=[
FileConnectorProfileResponse(**profile.to_response())
for profile in profiles
],
credentials=[
FileConnectorCredentialResponse(**credential.to_response())
for credential in credentials
],
spaces=[_connector_space_response(space) for space in spaces],
policy=FileConnectorPolicyResponse(
**connector_policy_response(
session,
tenant_id=principal.tenant_id,
scope_type=scope_type,
scope_id=scope_id,
)
),
changed_sections=["profiles", "credentials", "spaces", "policy"],
deleted=[],
watermark=_file_connector_settings_watermark(
session, tenant_id=principal.tenant_id
),
has_more=False,
full=True,
)
def _changed_file_connector_setting_ids(
entries: list[ChangeSequenceEntry],
) -> tuple[set[str], set[str], set[str], bool]:
changed_profile_ids = {
entry.resource_id
for entry in entries
if entry.collection == FILES_CONNECTOR_PROFILES_COLLECTION
and entry.resource_type == FILES_CONNECTOR_PROFILE_RESOURCE
}
changed_credential_ids = {
entry.resource_id
for entry in entries
if entry.collection == FILES_CONNECTOR_CREDENTIALS_COLLECTION
and entry.resource_type == FILES_CONNECTOR_CREDENTIAL_RESOURCE
}
changed_space_ids = {
entry.resource_id
for entry in entries
if entry.collection == FILES_CONNECTOR_SPACES_COLLECTION
and entry.resource_type == FILES_CONNECTOR_SPACE_RESOURCE
}
policy_changed = any(
entry.collection == FILES_CONNECTOR_POLICIES_COLLECTION for entry in entries
)
return (
changed_profile_ids,
changed_credential_ids,
changed_space_ids,
policy_changed,
)
def _file_connector_settings_changed_sections(
*,
changed_profile_ids: set[str],
changed_credential_ids: set[str],
changed_space_ids: set[str],
policy_changed: bool,
profiles: list[ConnectorProfile],
) -> list[str]:
changed_sections = []
if changed_profile_ids or any(
profile.credential_profile_id
and profile.credential_profile_id in changed_credential_ids
for profile in profiles
):
changed_sections.append("profiles")
if changed_credential_ids:
changed_sections.append("credentials")
if changed_space_ids:
changed_sections.append("spaces")
if policy_changed:
changed_sections.append("policy")
return changed_sections
def _file_connector_settings_deleted_items(
entries: list[ChangeSequenceEntry],
*,
visible_profiles: dict[str, ConnectorProfile],
visible_credentials: dict[str, ConnectorCredential],
visible_spaces: dict[str, FileConnectorSpace],
) -> list[DeltaDeletedItem]:
return [
_connector_deleted_item(entry)
for entry in entries
if (
entry.resource_type == FILES_CONNECTOR_PROFILE_RESOURCE
and entry.resource_id not in visible_profiles
)
or (
entry.resource_type == FILES_CONNECTOR_CREDENTIAL_RESOURCE
and entry.resource_id not in visible_credentials
)
or (
entry.resource_type == FILES_CONNECTOR_SPACE_RESOURCE
and entry.resource_id not in visible_spaces
)
]
def _incremental_file_connector_settings_delta_response(
session: Session,
principal: ApiPrincipal,
*,
entries: list[ChangeSequenceEntry],
has_more: bool,
scope_type: str,
scope_id: str | None,
provider: str | None,
campaign_id: str | None,
include_disabled: bool,
include_inactive: bool,
owner_type: Literal["user", "group"] | None,
owner_id: str | None,
) -> FileConnectorSettingsDeltaResponse:
changed_profile_ids, changed_credential_ids, changed_space_ids, policy_changed = (
_changed_file_connector_setting_ids(entries)
)
profiles = _visible_connector_profiles(
session,
principal,
provider=provider,
campaign_id=campaign_id,
include_disabled=include_disabled
and _can_read_disabled_connector_profiles(principal),
include_admin_scopes=_can_read_disabled_connector_profiles(principal),
include_effective_policy=False,
)
visible_profiles = {
profile.id: profile
for profile in profiles
if profile.id in changed_profile_ids
or (
profile.credential_profile_id
and profile.credential_profile_id in changed_credential_ids
)
}
credentials = _visible_connector_credentials(
session,
principal,
provider=provider,
include_disabled=include_disabled,
)
visible_credentials = {
credential.id: credential
for credential in credentials
if credential.id in changed_credential_ids
}
spaces = _visible_connector_spaces(
session,
principal,
owner_type=owner_type,
owner_id=owner_id,
include_inactive=include_inactive,
)
visible_spaces = {
space.id: space for space in spaces if space.id in changed_space_ids
}
return FileConnectorSettingsDeltaResponse(
profiles=[
FileConnectorProfileResponse(**profile.to_response())
for profile in visible_profiles.values()
],
credentials=[
FileConnectorCredentialResponse(**credential.to_response())
for credential in visible_credentials.values()
],
spaces=[_connector_space_response(space) for space in visible_spaces.values()],
policy=FileConnectorPolicyResponse(
**connector_policy_response(
session,
tenant_id=principal.tenant_id,
scope_type=scope_type,
scope_id=scope_id,
)
)
if policy_changed
else None,
changed_sections=_file_connector_settings_changed_sections(
changed_profile_ids=changed_profile_ids,
changed_credential_ids=changed_credential_ids,
changed_space_ids=changed_space_ids,
policy_changed=policy_changed,
profiles=profiles,
),
deleted=_file_connector_settings_deleted_items(
entries,
visible_profiles=visible_profiles,
visible_credentials=visible_credentials,
visible_spaces=visible_spaces,
),
watermark=_file_connector_settings_response_watermark(
session, tenant_id=principal.tenant_id, entries=entries, has_more=has_more
),
has_more=has_more,
full=False,
)
@@ -0,0 +1,620 @@
from __future__ import annotations
from datetime import datetime
from typing import Literal
from fastapi import HTTPException, status
from sqlalchemy.orm import Session
from govoplan_core.auth import ApiPrincipal
from govoplan_core.api.v1.schemas import DeltaDeletedItem
from govoplan_core.core.change_sequence import (
ChangeSequenceEntry,
decode_sequence_watermark,
encode_sequence_watermark,
max_sequence_id,
sequence_entries_since,
sequence_watermark_is_expired,
)
from govoplan_core.core.pagination import (
KeysetCursorError,
decode_keyset_cursor,
encode_keyset_cursor,
keyset_query_fingerprint,
)
from govoplan_files.backend.change_tracking import (
FILES_ASSETS_COLLECTION,
FILES_FOLDERS_COLLECTION,
FILES_MODULE_ID,
)
from govoplan_files.backend.schemas import (
FileDeltaResponse,
)
from govoplan_files.backend.db.models import FileAsset, FileFolder, FileShare
from govoplan_files.backend.storage.access import user_group_ids
from govoplan_files.backend.storage.files import (
list_assets_for_user,
)
from govoplan_files.backend.storage.folders import list_folders_for_user
from govoplan_files.backend.storage.share_state import effective_file_share_clause
from govoplan_files.backend.route_support import (
_asset_list_response,
_folder_response,
_is_admin,
)
_FILES_DELTA_COLLECTIONS = (FILES_ASSETS_COLLECTION, FILES_FOLDERS_COLLECTION)
FILES_LIST_CURSOR_SCOPE = "files.list.v1"
FOLDERS_LIST_CURSOR_SCOPE = "files.folders.list.v1"
DEFAULT_FILE_LIST_PAGE_SIZE = 500
def _cursor_http_error(exc: Exception) -> HTTPException:
return HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc))
def _cursor_page_size(
scope: str, cursor: str | None, explicit_page_size: int | None
) -> int | None:
if explicit_page_size is not None:
return explicit_page_size
if not cursor:
return DEFAULT_FILE_LIST_PAGE_SIZE
try:
values = decode_keyset_cursor(scope, cursor)
except KeysetCursorError as exc:
raise _cursor_http_error(exc) from exc
raw_page_size = values.get("page_size")
if not isinstance(raw_page_size, int) or raw_page_size < 1 or raw_page_size > 1000:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid pagination cursor"
)
return raw_page_size
def _files_list_fingerprint(
principal: ApiPrincipal,
*,
owner_type: Literal["user", "group"] | None,
owner_id: str | None,
campaign_id: str | None,
path_prefix: str | None,
campaign_usage: Literal["linked", "unlinked"] | None,
audit_relevant: bool | None,
page_size: int,
) -> str:
return keyset_query_fingerprint(
FILES_LIST_CURSOR_SCOPE,
{
"tenant_id": principal.tenant_id,
"actor": "admin" if _is_admin(principal) else principal.user.id,
"owner_type": owner_type,
"owner_id": owner_id,
"campaign_id": campaign_id,
"path_prefix": path_prefix or "",
"campaign_usage": campaign_usage or "",
"audit_relevant": audit_relevant,
"sort": "display_path.asc,updated_at.desc,id.asc",
"page_size": page_size,
},
)
def _folders_list_fingerprint(
principal: ApiPrincipal,
*,
owner_type: Literal["user", "group"],
owner_id: str,
page_size: int,
) -> str:
return keyset_query_fingerprint(
FOLDERS_LIST_CURSOR_SCOPE,
{
"tenant_id": principal.tenant_id,
"actor": "admin" if _is_admin(principal) else principal.user.id,
"owner_type": owner_type,
"owner_id": owner_id,
"sort": "path.asc,id.asc",
"page_size": page_size,
},
)
def _file_cursor_values(
cursor: str | None, *, fingerprint: str
) -> tuple[str | None, datetime | None, str | None]:
try:
values = decode_keyset_cursor(
FILES_LIST_CURSOR_SCOPE, cursor, fingerprint=fingerprint
)
except KeysetCursorError as exc:
raise _cursor_http_error(exc) from exc
if values is None:
return None, None, None
display_path = values.get("display_path")
updated_at = values.get("updated_at")
asset_id = values.get("id")
if (
not isinstance(display_path, str)
or not isinstance(updated_at, str)
or not isinstance(asset_id, str)
):
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid pagination cursor"
)
try:
parsed_updated_at = datetime.fromisoformat(updated_at)
except ValueError as exc:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid pagination cursor"
) from exc
return display_path, parsed_updated_at, asset_id
def _folder_cursor_values(
cursor: str | None, *, fingerprint: str
) -> tuple[str | None, str | None]:
try:
values = decode_keyset_cursor(
FOLDERS_LIST_CURSOR_SCOPE, cursor, fingerprint=fingerprint
)
except KeysetCursorError as exc:
raise _cursor_http_error(exc) from exc
if values is None:
return None, None
folder_path = values.get("path")
folder_id = values.get("id")
if not isinstance(folder_path, str) or not isinstance(folder_id, str):
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid pagination cursor"
)
return folder_path, folder_id
def _next_file_list_cursor(
principal: ApiPrincipal,
assets: list[FileAsset],
*,
owner_type: Literal["user", "group"] | None,
owner_id: str | None,
campaign_id: str | None,
path_prefix: str | None,
campaign_usage: Literal["linked", "unlinked"] | None,
audit_relevant: bool | None,
page_size: int,
has_more: bool,
) -> str | None:
if not has_more or not assets:
return None
last = assets[-1]
return encode_keyset_cursor(
FILES_LIST_CURSOR_SCOPE,
fingerprint=_files_list_fingerprint(
principal,
owner_type=owner_type,
owner_id=owner_id,
campaign_id=campaign_id,
path_prefix=path_prefix,
campaign_usage=campaign_usage,
audit_relevant=audit_relevant,
page_size=page_size,
),
values={
"display_path": last.display_path,
"updated_at": last.updated_at,
"id": last.id,
"page_size": page_size,
},
)
def _next_folder_list_cursor(
principal: ApiPrincipal,
folders: list[FileFolder],
*,
owner_type: Literal["user", "group"],
owner_id: str,
page_size: int,
has_more: bool,
) -> str | None:
if not has_more or not folders:
return None
last = folders[-1]
return encode_keyset_cursor(
FOLDERS_LIST_CURSOR_SCOPE,
fingerprint=_folders_list_fingerprint(
principal, owner_type=owner_type, owner_id=owner_id, page_size=page_size
),
values={"path": last.path, "id": last.id, "page_size": page_size},
)
def _files_delta_watermark(session: Session, tenant_id: str) -> str:
return encode_sequence_watermark(
max_sequence_id(
session,
tenant_id=tenant_id,
module_id=FILES_MODULE_ID,
collections=_FILES_DELTA_COLLECTIONS,
)
)
def _full_file_delta_response(
session: Session,
*,
principal: ApiPrincipal,
owner_type: Literal["user", "group"] | None,
owner_id: str | None,
campaign_id: str | None,
path_prefix: str | None,
) -> FileDeltaResponse:
assets = list_assets_for_user(
session,
tenant_id=principal.tenant_id,
user_id=principal.user.id,
owner_type=owner_type,
owner_id=owner_id,
campaign_id=campaign_id,
path_prefix=path_prefix,
is_admin=_is_admin(principal),
)
folders = _visible_folders_for_delta(
session,
principal=principal,
owner_type=owner_type,
owner_id=owner_id,
path_prefix=path_prefix,
)
return FileDeltaResponse(
files=_asset_list_response(session, assets, include_shares=True),
folders=[_folder_response(folder) for folder in folders],
deleted=[],
watermark=_files_delta_watermark(session, principal.tenant_id),
has_more=False,
full=True,
)
def _visible_folders_for_delta(
session: Session,
*,
principal: ApiPrincipal,
owner_type: Literal["user", "group"] | None,
owner_id: str | None,
path_prefix: str | None,
) -> list[FileFolder]:
if not owner_type or not owner_id:
return []
folders = list_folders_for_user(
session,
tenant_id=principal.tenant_id,
user_id=principal.user.id,
owner_type=owner_type,
owner_id=owner_id,
is_admin=_is_admin(principal),
)
if not path_prefix:
return folders
normalized = path_prefix.strip().strip("/")
if not normalized:
return folders
return [
folder
for folder in folders
if folder.path == normalized or folder.path.startswith(f"{normalized}/")
]
def _entry_path_matches(
entry_payload: dict[str, object], path_prefix: str | None
) -> bool:
if not path_prefix:
return True
normalized = path_prefix.strip().strip("/")
if not normalized:
return True
for key in ("path", "previous_path"):
value = entry_payload.get(key)
if isinstance(value, str) and (
value == normalized or value.startswith(f"{normalized}/")
):
return True
return False
def _entry_owner_matches(
entry_payload: dict[str, object],
owner_type: Literal["user", "group"] | None,
owner_id: str | None,
) -> bool:
if not owner_type or not owner_id:
return True
return (
entry_payload.get("owner_type") == owner_type
and entry_payload.get("owner_id") == owner_id
)
def _entry_campaign_matches(session: Session, entry, campaign_id: str | None) -> bool:
if not campaign_id:
return True
payload = entry.payload or {}
if (
payload.get("share_target_type") == "campaign"
and payload.get("share_target_id") == campaign_id
):
return True
if entry.resource_type != "file":
return False
return (
session.query(FileShare)
.filter(
FileShare.tenant_id == entry.tenant_id,
FileShare.file_asset_id == entry.resource_id,
FileShare.target_type == "campaign",
FileShare.target_id == campaign_id,
effective_file_share_clause(),
)
.first()
is not None
)
def _principal_group_ids_for_delta(
session: Session, principal: ApiPrincipal
) -> set[str]:
cache = session.info.setdefault("files_delta_group_ids", {})
key = (principal.tenant_id, principal.user.id)
if key not in cache:
cache[key] = set(
user_group_ids(
session, tenant_id=principal.tenant_id, user_id=principal.user.id
)
)
return cache[key]
def _entry_subject_matches_principal(
session: Session, principal: ApiPrincipal, entry_payload: dict[str, object]
) -> bool:
if _is_admin(principal):
return True
owner_type = entry_payload.get("owner_type")
owner_id = entry_payload.get("owner_id")
if owner_type == "user" and owner_id == principal.user.id:
return True
if owner_type == "group" and isinstance(owner_id, str):
if owner_id in _principal_group_ids_for_delta(session, principal):
return True
share_target_type = entry_payload.get("share_target_type")
share_target_id = entry_payload.get("share_target_id")
if share_target_type == "user" and share_target_id == principal.user.id:
return True
if share_target_type == "tenant" and share_target_id == principal.tenant_id:
return True
if share_target_type == "group" and isinstance(share_target_id, str):
if share_target_id in _principal_group_ids_for_delta(session, principal):
return True
return False
def _entry_matches_delta_scope(
session: Session,
principal: ApiPrincipal,
entry,
*,
owner_type: Literal["user", "group"] | None,
owner_id: str | None,
campaign_id: str | None,
path_prefix: str | None,
) -> bool:
payload = entry.payload or {}
if (
not owner_type
and not campaign_id
and not _entry_subject_matches_principal(session, principal, payload)
):
return False
return (
_entry_owner_matches(payload, owner_type, owner_id)
and _entry_path_matches(payload, path_prefix)
and _entry_campaign_matches(session, entry, campaign_id)
)
def _decode_files_delta_watermark(since: str) -> int:
try:
return decode_sequence_watermark(since)
except ValueError as exc:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)
) from exc
def _changed_delta_resource_ids(
entries: list[ChangeSequenceEntry],
) -> tuple[list[str], list[str]]:
file_ids = list(
dict.fromkeys(
entry.resource_id for entry in entries if entry.resource_type == "file"
)
)
folder_ids = list(
dict.fromkeys(
entry.resource_id for entry in entries if entry.resource_type == "folder"
)
)
return file_ids, folder_ids
def _visible_assets_for_delta(
session: Session,
*,
principal: ApiPrincipal,
owner_type: Literal["user", "group"] | None,
owner_id: str | None,
campaign_id: str | None,
path_prefix: str | None,
changed_file_ids: list[str],
) -> dict[str, FileAsset]:
return {
asset.id: asset
for asset in list_assets_for_user(
session,
tenant_id=principal.tenant_id,
user_id=principal.user.id,
owner_type=owner_type,
owner_id=owner_id,
campaign_id=campaign_id,
path_prefix=path_prefix,
is_admin=_is_admin(principal),
)
if asset.id in changed_file_ids
}
def _changed_visible_folders_for_delta(
session: Session,
*,
principal: ApiPrincipal,
owner_type: Literal["user", "group"] | None,
owner_id: str | None,
path_prefix: str | None,
changed_folder_ids: list[str],
) -> dict[str, FileFolder]:
return {
folder.id: folder
for folder in _visible_folders_for_delta(
session,
principal=principal,
owner_type=owner_type,
owner_id=owner_id,
path_prefix=path_prefix,
)
if folder.id in changed_folder_ids
}
def _deleted_delta_items(
session: Session,
*,
principal: ApiPrincipal,
entries: list[ChangeSequenceEntry],
visible_assets: dict[str, FileAsset],
visible_folders: dict[str, FileFolder],
owner_type: Literal["user", "group"] | None,
owner_id: str | None,
campaign_id: str | None,
path_prefix: str | None,
) -> list[DeltaDeletedItem]:
deleted: dict[tuple[str, str], DeltaDeletedItem] = {}
for entry in entries:
if entry.resource_type == "file" and entry.resource_id in visible_assets:
continue
if entry.resource_type == "folder" and entry.resource_id in visible_folders:
continue
if not _entry_matches_delta_scope(
session,
principal,
entry,
owner_type=owner_type,
owner_id=owner_id,
campaign_id=campaign_id,
path_prefix=path_prefix,
):
continue
deleted[(entry.resource_type, entry.resource_id)] = DeltaDeletedItem(
id=entry.resource_id,
resource_type=entry.resource_type,
revision=encode_sequence_watermark(entry.id),
deleted_at=entry.created_at if entry.operation == "deleted" else None,
)
return list(deleted.values())
def _files_delta_response(
session: Session,
*,
principal: ApiPrincipal,
owner_type: Literal["user", "group"] | None,
owner_id: str | None,
campaign_id: str | None,
path_prefix: str | None,
since: str,
limit: int,
) -> FileDeltaResponse:
since_sequence = _decode_files_delta_watermark(since)
if sequence_watermark_is_expired(
session,
since=since_sequence,
tenant_id=principal.tenant_id,
module_id=FILES_MODULE_ID,
collections=_FILES_DELTA_COLLECTIONS,
):
return _full_file_delta_response(
session,
principal=principal,
owner_type=owner_type,
owner_id=owner_id,
campaign_id=campaign_id,
path_prefix=path_prefix,
)
entries_plus_one = sequence_entries_since(
session,
since=since_sequence,
tenant_id=principal.tenant_id,
module_id=FILES_MODULE_ID,
collections=_FILES_DELTA_COLLECTIONS,
limit=limit + 1,
)
has_more = len(entries_plus_one) > limit
entries = entries_plus_one[:limit]
changed_file_ids, changed_folder_ids = _changed_delta_resource_ids(entries)
visible_assets = _visible_assets_for_delta(
session,
principal=principal,
owner_type=owner_type,
owner_id=owner_id,
campaign_id=campaign_id,
path_prefix=path_prefix,
changed_file_ids=changed_file_ids,
)
visible_folders = _changed_visible_folders_for_delta(
session,
principal=principal,
owner_type=owner_type,
owner_id=owner_id,
path_prefix=path_prefix,
changed_folder_ids=changed_folder_ids,
)
deleted = _deleted_delta_items(
session,
principal=principal,
entries=entries,
visible_assets=visible_assets,
visible_folders=visible_folders,
owner_type=owner_type,
owner_id=owner_id,
campaign_id=campaign_id,
path_prefix=path_prefix,
)
watermark = (
encode_sequence_watermark(entries[-1].id)
if has_more and entries
else _files_delta_watermark(session, principal.tenant_id)
)
return FileDeltaResponse(
files=_asset_list_response(
session, list(visible_assets.values()), include_shares=True
),
folders=[_folder_response(folder) for folder in visible_folders.values()],
deleted=deleted,
watermark=watermark,
has_more=has_more,
full=False,
)
+589 -58
View File
@@ -1,55 +1,91 @@
from __future__ import annotations
import mimetypes
import re
import stat
import tarfile
import zipfile
from dataclasses import dataclass
from io import BytesIO
from os import PathLike
from pathlib import Path
from typing import Any, Iterable
from pathlib import Path, PurePosixPath
from typing import Any, BinaryIO, Iterable, Literal
import pyzipper
from sqlalchemy.orm import Session
from govoplan_files.backend.db.models import FileAsset
from govoplan_files.backend.storage.common import FileConflictResolution, FileStorageError, UploadedStoredFile
from govoplan_files.backend.storage.backends import StorageBackendError, get_storage_backend
from govoplan_files.backend.storage.files import create_file_asset, current_versions_and_blobs
from govoplan_files.backend.storage.paths import filename_from_path, normalize_folder, normalize_logical_path
from govoplan_files.backend.storage.backends import (
StorageBackendError,
get_storage_backend,
)
from govoplan_files.backend.storage.common import (
FileConflictResolution,
FileStorageError,
UploadedStoredFile,
)
from govoplan_files.backend.storage.files import (
create_file_asset,
current_versions_and_blobs,
)
from govoplan_files.backend.storage.paths import (
filename_from_path,
normalize_folder,
normalize_logical_path,
)
_ZIP_READ_CHUNK_SIZE = 1024 * 1024
ZIP_UPLOAD_MAX_FILES = 1000
_ARCHIVE_READ_CHUNK_SIZE = 1024 * 1024
_WINDOWS_DRIVE_RE = re.compile(r"^[A-Za-z]:")
ARCHIVE_UPLOAD_MAX_ENTRIES = 10_000
# Kept for callers and documentation using the former ZIP-specific name.
ZIP_UPLOAD_MAX_FILES = ARCHIVE_UPLOAD_MAX_ENTRIES
SUPPORTED_ARCHIVE_SUFFIXES = (
".tar.bz2",
".tar.gz",
".tar.xz",
".tbz2",
".tgz",
".txz",
".tar",
".zip",
)
def _read_zip_member(
archive: zipfile.ZipFile,
info: zipfile.ZipInfo,
*,
max_file_bytes: int,
max_total_bytes: int,
current_total: int,
) -> tuple[bytes, int]:
parts: list[bytes] = []
actual_size = 0
with archive.open(info) as source:
while True:
read_size = min(_ZIP_READ_CHUNK_SIZE, max_file_bytes + 1 - actual_size)
chunk = source.read(read_size)
if not chunk:
break
actual_size += len(chunk)
if actual_size > max_file_bytes:
raise FileStorageError(f"ZIP member {info.filename!r} exceeds per-file limit")
if current_total + actual_size > max_total_bytes:
raise FileStorageError("ZIP is too large after extraction")
parts.append(chunk)
return b"".join(parts), current_total + actual_size
class ArchivePasswordError(FileStorageError):
pass
def create_zip_file(session: Session, assets: Iterable[FileAsset], output_path: str | Path) -> None:
@dataclass(frozen=True, slots=True)
class ArchiveEntry:
path: str
kind: Literal["file", "directory"]
size_bytes: int
compressed_size_bytes: int | None = None
encrypted: bool = False
@dataclass(frozen=True, slots=True)
class ArchiveInspection:
archive_format: str
entries: tuple[ArchiveEntry, ...]
file_count: int
directory_count: int
expanded_size_bytes: int
compressed_size_bytes: int
requires_password: bool
password_verified: bool
def create_zip_file(
session: Session, assets: Iterable[FileAsset], output_path: str | Path
) -> None:
backend = get_storage_backend()
asset_list = list(assets)
version_blobs = current_versions_and_blobs(session, asset_list)
with zipfile.ZipFile(output_path, mode="w", compression=zipfile.ZIP_DEFLATED) as archive:
with zipfile.ZipFile(
output_path, mode="w", compression=zipfile.ZIP_DEFLATED
) as archive:
for asset in asset_list:
_version, blob = version_blobs[asset.id]
info = zipfile.ZipInfo(asset.display_path)
@@ -63,6 +99,147 @@ def create_zip_file(session: Session, assets: Iterable[FileAsset], output_path:
raise FileStorageError(str(exc)) from exc
def archive_format_for_filename(filename: str) -> str:
lowered = filename.strip().casefold()
if lowered.endswith(".zip"):
return "zip"
if lowered.endswith((".tar.gz", ".tgz")):
return "tar.gz"
if lowered.endswith((".tar.bz2", ".tbz2")):
return "tar.bz2"
if lowered.endswith((".tar.xz", ".txz")):
return "tar.xz"
if lowered.endswith(".tar"):
return "tar"
raise FileStorageError(
"Unsupported archive format. Use ZIP, TAR, TAR.GZ, TAR.BZ2, or TAR.XZ."
)
def is_supported_archive_filename(filename: str) -> bool:
try:
archive_format_for_filename(filename)
except FileStorageError:
return False
return True
def inspect_archive(
archive_data: bytes | str | PathLike[str],
*,
filename: str,
password: str | None = None,
max_entries: int = ARCHIVE_UPLOAD_MAX_ENTRIES,
max_expanded_bytes: int = 2 * 1024 * 1024 * 1024,
max_expansion_ratio: int = 100,
) -> ArchiveInspection:
archive_format = archive_format_for_filename(filename)
compressed_size = _archive_size(archive_data)
if archive_format == "zip":
entries, requires_password, password_verified = _inspect_zip(
archive_data,
password=password,
)
else:
entries = _inspect_tar(archive_data)
requires_password = False
password_verified = True
_validate_archive_limits(
entries,
compressed_size=compressed_size,
max_entries=max_entries,
max_expanded_bytes=max_expanded_bytes,
max_expansion_ratio=max_expansion_ratio,
)
complete_entries = _with_derived_directories(entries)
return ArchiveInspection(
archive_format=archive_format,
entries=tuple(complete_entries),
file_count=sum(entry.kind == "file" for entry in complete_entries),
directory_count=sum(
entry.kind == "directory" for entry in complete_entries
),
expanded_size_bytes=sum(
entry.size_bytes for entry in entries if entry.kind == "file"
),
compressed_size_bytes=compressed_size,
requires_password=requires_password,
password_verified=password_verified,
)
def extract_archive_upload(
session: Session,
*,
tenant_id: str,
owner_type: str,
owner_id: str,
user_id: str,
archive_data: bytes | str | PathLike[str],
filename: str,
folder: str | None,
campaign_id: str | None,
selected_paths: Iterable[str] | None = None,
password: str | None = None,
conflict_strategy: str = "reject",
conflict_resolutions: Iterable[FileConflictResolution] | None = None,
metadata: dict[str, Any] | None = None,
is_admin: bool = False,
encryption_vault_id: str | None = None,
max_entries: int = ARCHIVE_UPLOAD_MAX_ENTRIES,
max_file_bytes: int = 50 * 1024 * 1024,
max_expanded_bytes: int = 2 * 1024 * 1024 * 1024,
max_expansion_ratio: int = 100,
) -> list[UploadedStoredFile]:
inspection = inspect_archive(
archive_data,
filename=filename,
password=password,
max_entries=max_entries,
max_expanded_bytes=max_expanded_bytes,
max_expansion_ratio=max_expansion_ratio,
)
if inspection.requires_password and not inspection.password_verified:
raise ArchivePasswordError("Archive password is required")
selected_files = _selected_file_paths(inspection.entries, selected_paths)
if not selected_files:
raise FileStorageError("Select at least one archive file to import")
actual_total_limit = min(
max_expanded_bytes,
inspection.compressed_size_bytes * max_expansion_ratio,
)
if inspection.archive_format == "zip":
members = _read_selected_zip_members(
archive_data,
selected_files=selected_files,
password=password,
max_file_bytes=max_file_bytes,
max_total_bytes=actual_total_limit,
)
else:
members = _read_selected_tar_members(
archive_data,
selected_files=selected_files,
max_file_bytes=max_file_bytes,
max_total_bytes=actual_total_limit,
)
return _store_archive_members(
session,
members=members,
tenant_id=tenant_id,
owner_type=owner_type,
owner_id=owner_id,
user_id=user_id,
folder=folder,
campaign_id=campaign_id,
conflict_strategy=conflict_strategy,
conflict_resolutions=conflict_resolutions,
metadata=metadata,
is_admin=is_admin,
encryption_vault_id=encryption_vault_id,
)
def extract_zip_upload(
session: Session,
*,
@@ -77,37 +254,348 @@ def extract_zip_upload(
conflict_resolutions: Iterable[FileConflictResolution] | None = None,
metadata: dict[str, Any] | None = None,
is_admin: bool = False,
encryption_vault_id: str | None = None,
max_files: int = ZIP_UPLOAD_MAX_FILES,
max_file_bytes: int = 50 * 1024 * 1024,
max_total_bytes: int = 250 * 1024 * 1024,
) -> list[UploadedStoredFile]:
uploaded: list[UploadedStoredFile] = []
total = 0
base_folder = normalize_folder(folder)
"""Backward-compatible wrapper for the original immediate ZIP endpoint."""
return extract_archive_upload(
session,
tenant_id=tenant_id,
owner_type=owner_type,
owner_id=owner_id,
user_id=user_id,
archive_data=zip_data,
filename="archive.zip",
folder=folder,
campaign_id=campaign_id,
conflict_strategy=conflict_strategy,
conflict_resolutions=conflict_resolutions,
metadata=metadata,
is_admin=is_admin,
encryption_vault_id=encryption_vault_id,
max_entries=max_files,
max_file_bytes=max_file_bytes,
max_expanded_bytes=max_total_bytes,
)
def _inspect_zip(
archive_data: bytes | str | PathLike[str],
*,
password: str | None,
) -> tuple[list[ArchiveEntry], bool, bool]:
try:
source = BytesIO(zip_data) if isinstance(zip_data, bytes) else zip_data
with zipfile.ZipFile(source) as archive:
infos = [info for info in archive.infolist() if not info.is_dir()]
if len(infos) > max_files:
raise FileStorageError(f"ZIP contains too many files (limit {max_files})")
for info in infos:
if info.flag_bits & 0x1:
raise FileStorageError("Encrypted ZIP uploads are not supported")
if info.file_size < 0:
raise FileStorageError("Invalid ZIP member")
if info.file_size > max_file_bytes:
raise FileStorageError(f"ZIP member {info.filename!r} exceeds per-file limit")
if total + info.file_size > max_total_bytes:
raise FileStorageError("ZIP is too large after extraction")
inner_path = normalize_logical_path(info.filename)
target_path = f"{base_folder}/{inner_path}" if base_folder else inner_path
data, total = _read_zip_member(
archive,
info,
with pyzipper.AESZipFile(_archive_source(archive_data)) as archive:
infos = archive.infolist()
entries = [_zip_entry(info) for info in infos]
encrypted_info = next(
(
info
for info in infos
if not info.is_dir() and bool(info.flag_bits & 0x1)
),
None,
)
if encrypted_info is None:
return entries, False, True
if not password:
return entries, True, False
try:
with archive.open(
encrypted_info, pwd=password.encode("utf-8")
) as member:
member.read(1)
except (RuntimeError, ValueError, zipfile.BadZipFile) as exc:
raise ArchivePasswordError("Archive password is incorrect") from exc
return entries, True, True
except ArchivePasswordError:
raise
except (OSError, ValueError, zipfile.BadZipFile) as exc:
raise FileStorageError("Invalid ZIP upload") from exc
def _inspect_tar(
archive_data: bytes | str | PathLike[str],
) -> list[ArchiveEntry]:
try:
with _open_tar(archive_data) as archive:
entries: list[ArchiveEntry] = []
for member in archive.getmembers():
path = _safe_member_path(member.name)
if member.isdir():
entries.append(
ArchiveEntry(path=path, kind="directory", size_bytes=0)
)
continue
if not member.isfile():
raise FileStorageError(
f"Archive member {member.name!r} is not a regular file or directory"
)
entries.append(
ArchiveEntry(
path=path,
kind="file",
size_bytes=max(0, int(member.size)),
)
)
return entries
except FileStorageError:
raise
except (OSError, tarfile.TarError) as exc:
raise FileStorageError("Invalid TAR upload") from exc
def _zip_entry(info: zipfile.ZipInfo) -> ArchiveEntry:
path = _safe_member_path(info.filename)
unix_mode = (info.external_attr >> 16) & 0xFFFF
file_type = stat.S_IFMT(unix_mode)
if file_type and not (
stat.S_ISREG(unix_mode) or stat.S_ISDIR(unix_mode)
):
raise FileStorageError(
f"Archive member {info.filename!r} is not a regular file or directory"
)
if info.file_size < 0 or info.compress_size < 0:
raise FileStorageError(f"Archive member {info.filename!r} has invalid size metadata")
return ArchiveEntry(
path=path,
kind="directory" if info.is_dir() else "file",
size_bytes=0 if info.is_dir() else int(info.file_size),
compressed_size_bytes=(
None if info.is_dir() else int(info.compress_size)
),
encrypted=bool(info.flag_bits & 0x1),
)
def _validate_archive_limits(
entries: list[ArchiveEntry],
*,
compressed_size: int,
max_entries: int,
max_expanded_bytes: int,
max_expansion_ratio: int,
) -> None:
if len(entries) > max_entries:
raise FileStorageError(
f"Archive contains too many entries (limit {max_entries})"
)
seen: dict[str, str] = {}
expanded_size = 0
for entry in entries:
previous_kind = seen.get(entry.path)
if previous_kind is not None:
raise FileStorageError(
f"Archive contains duplicate path {entry.path!r}"
)
seen[entry.path] = entry.kind
if entry.kind == "file":
expanded_size += entry.size_bytes
if expanded_size > max_expanded_bytes:
raise FileStorageError(
"Archive is too large after extraction "
f"(limit {max_expanded_bytes} bytes)"
)
if expanded_size and (
compressed_size <= 0
or expanded_size > compressed_size * max_expansion_ratio
):
raise FileStorageError(
"Archive expansion ratio exceeds "
f"{max_expansion_ratio}:1"
)
def _with_derived_directories(
entries: list[ArchiveEntry],
) -> list[ArchiveEntry]:
by_path = {entry.path: entry for entry in entries}
for entry in entries:
parent = PurePosixPath(entry.path).parent
while str(parent) not in {"", "."}:
path = str(parent)
existing = by_path.get(path)
if existing and existing.kind != "directory":
raise FileStorageError(
f"Archive path {path!r} is both a file and a directory"
)
by_path.setdefault(
path,
ArchiveEntry(path=path, kind="directory", size_bytes=0),
)
parent = parent.parent
return sorted(
by_path.values(),
key=lambda entry: (
tuple(entry.path.casefold().split("/")),
entry.kind != "directory",
),
)
def _selected_file_paths(
entries: tuple[ArchiveEntry, ...],
selected_paths: Iterable[str] | None,
) -> set[str]:
file_paths = {entry.path for entry in entries if entry.kind == "file"}
if selected_paths is None:
return file_paths
known = {entry.path: entry for entry in entries}
normalized = {_safe_member_path(path) for path in selected_paths}
unknown = normalized - known.keys()
if unknown:
raise FileStorageError(
f"Archive selection contains unknown path {sorted(unknown)[0]!r}"
)
selected_files: set[str] = set()
for path in normalized:
entry = known[path]
if entry.kind == "file":
selected_files.add(path)
continue
prefix = f"{path}/"
selected_files.update(
file_path
for file_path in file_paths
if file_path.startswith(prefix)
)
return selected_files
def _read_selected_zip_members(
archive_data: bytes | str | PathLike[str],
*,
selected_files: set[str],
password: str | None,
max_file_bytes: int,
max_total_bytes: int,
) -> list[tuple[str, bytes]]:
result: list[tuple[str, bytes]] = []
total = 0
try:
with pyzipper.AESZipFile(_archive_source(archive_data)) as archive:
for info in archive.infolist():
if info.is_dir():
continue
path = _safe_member_path(info.filename)
if path not in selected_files:
continue
pwd = password.encode("utf-8") if password else None
try:
with archive.open(info, pwd=pwd) as source:
data, total = _read_member(
source,
path=path,
max_file_bytes=max_file_bytes,
max_total_bytes=max_total_bytes,
current_total=total,
)
except (RuntimeError, ValueError, zipfile.BadZipFile) as exc:
if info.flag_bits & 0x1:
raise ArchivePasswordError(
"Archive password is incorrect"
) from exc
raise
result.append((path, data))
except (FileStorageError, ArchivePasswordError):
raise
except (OSError, ValueError, zipfile.BadZipFile) as exc:
raise FileStorageError("ZIP extraction failed") from exc
return result
def _read_selected_tar_members(
archive_data: bytes | str | PathLike[str],
*,
selected_files: set[str],
max_file_bytes: int,
max_total_bytes: int,
) -> list[tuple[str, bytes]]:
result: list[tuple[str, bytes]] = []
total = 0
try:
with _open_tar(archive_data) as archive:
for member in archive.getmembers():
if not member.isfile():
continue
path = _safe_member_path(member.name)
if path not in selected_files:
continue
source = archive.extractfile(member)
if source is None:
raise FileStorageError(
f"Archive member {path!r} could not be read"
)
with source:
data, total = _read_member(
source,
path=path,
max_file_bytes=max_file_bytes,
max_total_bytes=max_total_bytes,
current_total=total,
)
result.append((path, data))
except FileStorageError:
raise
except (OSError, tarfile.TarError) as exc:
raise FileStorageError("TAR extraction failed") from exc
return result
def _read_member(
source: BinaryIO,
*,
path: str,
max_file_bytes: int,
max_total_bytes: int,
current_total: int,
) -> tuple[bytes, int]:
parts: list[bytes] = []
actual_size = 0
while True:
read_size = min(
_ARCHIVE_READ_CHUNK_SIZE,
max_file_bytes + 1 - actual_size,
)
chunk = source.read(read_size)
if not chunk:
break
actual_size += len(chunk)
if actual_size > max_file_bytes:
raise FileStorageError(
f"Archive member {path!r} exceeds per-file limit"
)
if current_total + actual_size > max_total_bytes:
raise FileStorageError("Archive is too large after extraction")
parts.append(chunk)
return b"".join(parts), current_total + actual_size
def _store_archive_members(
session: Session,
*,
members: Iterable[tuple[str, bytes]],
tenant_id: str,
owner_type: str,
owner_id: str,
user_id: str,
folder: str | None,
campaign_id: str | None,
conflict_strategy: str,
conflict_resolutions: Iterable[FileConflictResolution] | None,
metadata: dict[str, Any] | None,
is_admin: bool,
encryption_vault_id: str | None,
) -> list[UploadedStoredFile]:
uploaded: list[UploadedStoredFile] = []
base_folder = normalize_folder(folder)
for inner_path, data in members:
target_path = (
f"{base_folder}/{inner_path}" if base_folder else inner_path
)
uploaded.append(
create_file_asset(
session,
@@ -118,14 +606,57 @@ def extract_zip_upload(
filename=filename_from_path(inner_path),
data=data,
display_path=target_path,
content_type=mimetypes.guess_type(inner_path)[0] or "application/octet-stream",
content_type=mimetypes.guess_type(inner_path)[0]
or "application/octet-stream",
metadata=metadata,
campaign_id=campaign_id,
conflict_strategy=conflict_strategy,
conflict_resolutions=conflict_resolutions,
is_admin=is_admin,
encryption_vault_id=encryption_vault_id,
)
)
except zipfile.BadZipFile as exc:
raise FileStorageError("Invalid ZIP upload") from exc
return uploaded
def _safe_member_path(value: str) -> str:
raw = str(value or "").replace("\\", "/").strip()
if (
not raw
or "\x00" in raw
or raw.startswith("/")
or _WINDOWS_DRIVE_RE.match(raw)
):
raise FileStorageError(f"Unsafe archive member path {value!r}")
if any(part == ".." for part in raw.split("/")):
raise FileStorageError(f"Unsafe archive member path {value!r}")
try:
return normalize_logical_path(raw.rstrip("/"))
except ValueError as exc:
raise FileStorageError(f"Unsafe archive member path {value!r}") from exc
def _archive_size(archive_data: bytes | str | PathLike[str]) -> int:
if isinstance(archive_data, bytes):
return len(archive_data)
try:
return Path(archive_data).stat().st_size
except OSError as exc:
raise FileStorageError("Archive upload could not be read") from exc
def _archive_source(
archive_data: bytes | str | PathLike[str],
) -> BytesIO | str | PathLike[str]:
return BytesIO(archive_data) if isinstance(archive_data, bytes) else archive_data
def _open_tar(
archive_data: bytes | str | PathLike[str],
) -> tarfile.TarFile:
if isinstance(archive_data, bytes):
return tarfile.open(fileobj=BytesIO(archive_data), mode="r:*")
try:
return tarfile.open(name=archive_data, mode="r:*")
except OSError as exc:
raise FileStorageError("Archive upload could not be read") from exc
+24 -199
View File
@@ -1,207 +1,32 @@
from __future__ import annotations
from dataclasses import dataclass, field
from pathlib import Path
from typing import Iterable, Protocol
from govoplan_core.security.outbound_http import (
OutboundHttpError,
response_limit,
validate_unpinned_sdk_http_url,
from govoplan_core.core.object_storage import (
LocalFilesystemStorageBackend,
S3StorageBackend,
StorageBackend,
StorageBackendError,
StorageObjectInfo,
StorageObjectMissing,
StorageObjectPage,
configured_storage_backend,
)
from govoplan_files.backend.runtime import settings
class StorageBackendError(RuntimeError):
pass
class StorageBackend(Protocol):
name: str
def put_bytes(self, key: str, data: bytes, *, content_type: str | None = None) -> None: ...
def get_bytes(self, key: str) -> bytes: ...
def iter_bytes(self, key: str, *, chunk_size: int = 1024 * 1024) -> Iterable[bytes]: ...
def delete(self, key: str) -> None: ...
def exists(self, key: str) -> bool: ...
@dataclass(slots=True)
class LocalFilesystemStorageBackend:
root: Path
fallback_roots: tuple[Path, ...] = field(default_factory=tuple)
name: str = "local"
def __post_init__(self) -> None:
self.root = self.root.expanduser().resolve()
self.fallback_roots = tuple(root.expanduser().resolve() for root in self.fallback_roots if root)
self.root.mkdir(parents=True, exist_ok=True)
def _path_for_root(self, root: Path, key: str) -> Path:
path = (root / key).resolve()
if not path.is_relative_to(root):
raise StorageBackendError("Storage key escapes local storage root")
return path
def _path(self, key: str) -> Path:
return self._path_for_root(self.root, key)
def _readable_path(self, key: str) -> Path:
primary = self._path(key)
if primary.exists() and primary.is_file():
return primary
for root in self.fallback_roots:
candidate = self._path_for_root(root, key)
if candidate.exists() and candidate.is_file():
return candidate
raise StorageBackendError("Stored object does not exist")
def put_bytes(self, key: str, data: bytes, *, content_type: str | None = None) -> None:
path = self._path(key)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(data)
def get_bytes(self, key: str) -> bytes:
return self._readable_path(key).read_bytes()
def iter_bytes(self, key: str, *, chunk_size: int = 1024 * 1024) -> Iterable[bytes]:
path = self._readable_path(key)
with path.open("rb") as handle:
while True:
chunk = handle.read(chunk_size)
if not chunk:
break
yield chunk
def delete(self, key: str) -> None:
path = self._path(key)
if path.exists() and path.is_file():
path.unlink()
def exists(self, key: str) -> bool:
try:
self._readable_path(key)
except StorageBackendError:
return False
return True
@dataclass(slots=True)
class S3StorageBackend:
bucket: str
endpoint_url: str
region_name: str
access_key_id: str
secret_access_key: str
name: str = "s3"
@property
def client(self):
try:
endpoint_url = validate_unpinned_sdk_http_url(
self.endpoint_url,
label="File storage S3 endpoint",
)
except OutboundHttpError as exc:
raise StorageBackendError(str(exc)) from exc
try:
import boto3
except ModuleNotFoundError as exc:
raise StorageBackendError("boto3 is required for the S3 storage backend") from exc
return boto3.client(
"s3",
endpoint_url=endpoint_url,
region_name=self.region_name,
aws_access_key_id=self.access_key_id,
aws_secret_access_key=self.secret_access_key,
)
def put_bytes(self, key: str, data: bytes, *, content_type: str | None = None) -> None:
max_bytes = response_limit("file")
if len(data) > max_bytes:
raise StorageBackendError(f"Stored object exceeds the deployment limit of {max_bytes} bytes")
kwargs = {"Bucket": self.bucket, "Key": key, "Body": data}
if content_type:
kwargs["ContentType"] = content_type
self.client.put_object(**kwargs)
def get_bytes(self, key: str) -> bytes:
try:
obj = self.client.get_object(Bucket=self.bucket, Key=key)
max_bytes = response_limit("file")
body = obj["Body"]
try:
_reject_declared_object_size(obj, max_bytes=max_bytes)
data = body.read(max_bytes + 1)
if len(data) > max_bytes:
raise StorageBackendError(f"Stored object exceeds the deployment limit of {max_bytes} bytes")
return data
finally:
if hasattr(body, "close"):
body.close()
except Exception as exc: # pragma: no cover - depends on S3 backend
raise StorageBackendError(str(exc)) from exc
def iter_bytes(self, key: str, *, chunk_size: int = 1024 * 1024) -> Iterable[bytes]:
try:
obj = self.client.get_object(Bucket=self.bucket, Key=key)
max_bytes = response_limit("file")
body = obj["Body"]
try:
_reject_declared_object_size(obj, max_bytes=max_bytes)
total = 0
while True:
chunk = body.read(chunk_size)
if not chunk:
break
total += len(chunk)
if total > max_bytes:
raise StorageBackendError(f"Stored object exceeds the deployment limit of {max_bytes} bytes")
yield chunk
finally:
if hasattr(body, "close"):
body.close()
except Exception as exc: # pragma: no cover - depends on S3 backend
raise StorageBackendError(str(exc)) from exc
def delete(self, key: str) -> None:
self.client.delete_object(Bucket=self.bucket, Key=key)
def exists(self, key: str) -> bool:
try:
self.client.head_object(Bucket=self.bucket, Key=key)
return True
except Exception:
return False
def _reject_declared_object_size(obj: object, *, max_bytes: int) -> None:
if not isinstance(obj, dict):
return
try:
declared_size = int(obj.get("ContentLength"))
except (TypeError, ValueError):
return
if declared_size > max_bytes:
raise StorageBackendError(f"Stored object exceeds the deployment limit of {max_bytes} bytes")
def _fallback_roots() -> tuple[Path, ...]:
raw = getattr(settings, "file_storage_local_fallback_roots", "") or ""
return tuple(Path(item.strip()) for item in str(raw).split(",") if item.strip())
def get_storage_backend() -> StorageBackend:
backend = settings.file_storage_backend.lower().strip()
if backend in {"local", "filesystem", "fs"}:
return LocalFilesystemStorageBackend(Path(settings.file_storage_local_root), fallback_roots=_fallback_roots())
if backend in {"s3", "garage"}:
return S3StorageBackend(
bucket=settings.file_storage_s3_bucket or settings.s3_bucket,
endpoint_url=settings.file_storage_s3_endpoint_url or settings.s3_endpoint_url,
region_name=settings.file_storage_s3_region or settings.s3_region,
access_key_id=settings.file_storage_s3_access_key_id or settings.s3_access_key_id,
secret_access_key=settings.file_storage_s3_secret_access_key or settings.s3_secret_access_key,
)
raise StorageBackendError(f"Unsupported file storage backend: {settings.file_storage_backend}")
"""Return the deployment-owned backend for Files-managed objects."""
return configured_storage_backend(settings)
__all__ = [
"LocalFilesystemStorageBackend",
"S3StorageBackend",
"StorageBackend",
"StorageBackendError",
"StorageObjectInfo",
"StorageObjectMissing",
"StorageObjectPage",
"get_storage_backend",
]
@@ -13,12 +13,16 @@ from typing import Any, Iterator
from sqlalchemy.orm import Session
from govoplan_files.backend.db.models import FileAsset, FileBlob, FileShare, FileVersion
from govoplan_files.backend.storage.backends import StorageBackend, StorageBackendError, get_storage_backend
from govoplan_files.backend.storage.common import FileStorageError
from govoplan_files.backend.storage.backends import StorageBackend, get_storage_backend
from govoplan_files.backend.storage.files import current_versions_and_blobs, get_asset_for_user, list_assets_for_user, share_files
from govoplan_files.backend.storage.access import ensure_owner_access
from govoplan_files.backend.storage.paths import normalize_folder, normalize_logical_path, safe_storage_component
from govoplan_files.backend.storage.provenance import source_provenance_from_metadata, source_revision_from_metadata
from govoplan_files.backend.storage.share_state import effective_file_share_clause
from govoplan_files.backend.storage.integrity import (
ensure_blob_is_readable,
read_verified_blob_bytes,
)
MANAGED_SOURCE_PREFIX = "managed:"
@@ -165,7 +169,7 @@ def _campaign_linked_asset_ids(
FileShare.file_asset_id.in_(chunk),
FileShare.target_type == "campaign",
FileShare.target_id == campaign_id,
FileShare.revoked_at.is_(None),
effective_file_share_clause(),
)
.all()
)
@@ -301,11 +305,9 @@ def _materialize_managed_asset(
target = _safe_local_target(local_root, relative_path)
target.parent.mkdir(parents=True, exist_ok=True)
version, blob = version_blobs[asset.id]
ensure_blob_is_readable(blob)
if include_bytes:
try:
data = backend.get_bytes(blob.storage_key) if backend else b""
except StorageBackendError as exc:
raise FileStorageError(str(exc)) from exc
data = read_verified_blob_bytes(blob, backend=backend) if backend else b""
target.write_bytes(data)
else:
target.touch()
@@ -15,8 +15,8 @@ from defusedxml import ElementTree as SafeElementTree
from govoplan_core.security.outbound_http import (
OutboundHttpError,
validate_unpinned_sdk_host,
validate_unpinned_sdk_http_url,
validate_outbound_host,
validate_outbound_http_url,
)
from govoplan_files.backend.storage.http_client import ConnectorHttpError, request_connector_bytes
@@ -27,6 +27,12 @@ from govoplan_files.backend.storage.connector_deployment import (
validate_connector_tls_metadata,
)
from govoplan_files.backend.storage.connector_profiles import ConnectorProfile
from govoplan_files.backend.storage.sdk_peer_pinning import (
SdkPeerPinningError,
create_pinned_s3_client,
install_pinned_smb_transport,
pinned_smb_connection_cache,
)
class ConnectorBrowseError(RuntimeError):
@@ -246,6 +252,28 @@ def _browse_smb(profile: ConnectorProfile, *, path: str) -> list[ConnectorBrowse
def _browse_s3(profile: ConnectorProfile, *, path: str, library_id: str | None, continuation_token: str | None) -> list[ConnectorBrowseItem]:
client = _s3_client(profile)
try:
return _browse_s3_with_client(
client,
profile=profile,
path=path,
library_id=library_id,
continuation_token=continuation_token,
)
finally:
close = getattr(client, "close", None)
if callable(close):
close()
def _browse_s3_with_client(
client: Any,
*,
profile: ConnectorProfile,
path: str,
library_id: str | None,
continuation_token: str | None,
) -> list[ConnectorBrowseItem]:
bucket = _s3_bucket(profile, library_id)
if not bucket:
try:
@@ -313,23 +341,22 @@ def _s3_client(profile: ConnectorProfile) -> Any:
raise ConnectorBrowseError("Secret-ref S3 credentials need a runtime secret resolver before live browsing")
if profile.endpoint_url:
try:
endpoint_url = validate_unpinned_sdk_http_url(
endpoint_url = validate_outbound_http_url(
profile.endpoint_url,
label="S3 connector endpoint",
)
except OutboundHttpError as exc:
raise ConnectorBrowseError(str(exc)) from exc
else:
raise ConnectorBrowseError(
"S3 connector endpoint discovery uses an SDK transport that cannot guarantee connection-time DNS/IP "
"pinning; live S3 access is disabled until that transport supports pinning"
)
endpoint_url = None
try:
boto3 = import_module("boto3")
config_module = import_module("botocore.config")
unsigned = import_module("botocore").UNSIGNED
except ImportError as exc:
raise ConnectorBrowseUnsupported("S3 connector browsing requires the optional boto3 dependency") from exc
kwargs: dict[str, object] = {"endpoint_url": endpoint_url}
kwargs: dict[str, object] = {}
if endpoint_url:
kwargs["endpoint_url"] = endpoint_url
region = _metadata_string(profile, "region") or _metadata_string(profile, "aws_region")
if region:
kwargs["region_name"] = region
@@ -346,10 +373,21 @@ def _s3_client(profile: ConnectorProfile) -> Any:
if verify is not None:
kwargs["verify"] = verify
addressing_style = _s3_addressing_style(profile)
config_values: dict[str, object] = {
"proxies": {},
"retries": {"mode": "standard", "max_attempts": 4},
}
if addressing_style:
kwargs["config"] = config_module.Config(s3={"addressing_style": addressing_style})
config_values["s3"] = {"addressing_style": addressing_style}
if bool(access_key) != bool(secret_key):
raise ConnectorBrowseError("S3 connectors require both an access key and a secret key")
if not access_key:
config_values["signature_version"] = unsigned
kwargs["config"] = config_module.Config(**config_values)
try:
return boto3.client("s3", **kwargs)
return create_pinned_s3_client(**kwargs)
except SdkPeerPinningError as exc:
raise ConnectorBrowseError(str(exc)) from exc
except Exception as exc: # pragma: no cover - concrete exception types are dependency-version specific
raise ConnectorBrowseError(f"S3 connector could not be initialized: {exc}") from exc
@@ -783,7 +821,7 @@ def _smb_location(profile: ConnectorProfile) -> _SmbLocation:
raise ConnectorBrowseError("SMB connector endpoint_url must include a server")
port = parsed.port or _int(profile.metadata.get("port")) or 445
try:
validate_unpinned_sdk_host(server, port=port, label="SMB connector endpoint")
validate_outbound_host(server, port=port, label="SMB connector endpoint")
except OutboundHttpError as exc:
raise ConnectorBrowseError(str(exc)) from exc
path_parts = [part for part in unquote(parsed.path or "").strip("/").split("/") if part]
@@ -810,6 +848,7 @@ def _smb_unc_path(location: _SmbLocation, path: str) -> str:
def _smb_client_kwargs(profile: ConnectorProfile, location: _SmbLocation) -> dict[str, object]:
kwargs: dict[str, object] = {
"port": location.port,
"connection_cache": pinned_smb_connection_cache(),
"require_signing": _metadata_bool(profile, "require_signing", default=True),
"auth_protocol": _metadata_string(profile, "auth_protocol") or "ntlm",
}
@@ -845,9 +884,11 @@ def _profile_token(profile: ConnectorProfile) -> str | None:
def _smbclient_module() -> Any:
try:
return import_module("smbclient")
return install_pinned_smb_transport(import_module("smbclient"))
except ImportError as exc:
raise ConnectorBrowseUnsupported("SMB connector browsing requires the optional smbprotocol dependency") from exc
except SdkPeerPinningError as exc:
raise ConnectorBrowseUnsupported(str(exc)) from exc
def _smb_entry_stat(entry: object) -> object | None:
@@ -5,8 +5,17 @@ from dataclasses import dataclass
from typing import Any
from sqlalchemy.orm import Session
from sqlalchemy import inspect
from govoplan_core.core.policy import normalize_policy_scope_type, policy_source_path
from govoplan_core.security.credential_envelopes import (
CredentialAccessContext,
CredentialEnvelope,
CredentialEnvelopeError,
ResolvedCredentialEnvelope,
list_credential_envelopes,
resolve_credential_envelope,
)
from govoplan_core.security.secrets import decrypt_secret, encrypt_secret
from govoplan_files.backend.db.models import FileConnectorCredential
from govoplan_files.backend.storage.common import FileStorageError
@@ -14,6 +23,8 @@ from govoplan_files.backend.storage.connector_deployment import reject_api_contr
from govoplan_files.backend.storage.connector_policy import ConnectorPolicySource, connector_policy_sources_from_payload
from govoplan_files.backend.storage.connector_profiles import supported_connector_providers
CORE_CREDENTIAL_ENVELOPE_PREFIX = "credential-envelope:"
@dataclass(frozen=True, slots=True)
class ConnectorCredential:
@@ -33,6 +44,7 @@ class ConnectorCredential:
policy_sources: tuple[ConnectorPolicySource, ...] = ()
metadata: Mapping[str, Any] | None = None
source_kind: str = "database"
has_secret: bool = False
@property
def source_path(self) -> str:
@@ -58,7 +70,7 @@ class ConnectorCredential:
return True
# Environment references are deliberately unavailable to API-managed
# credential rows. Legacy rows remain visible but fail closed.
return bool(self.secret_ref or self.password_value or self.token_value)
return bool(self.has_secret or self.secret_ref or self.password_value or self.token_value)
def to_response(self) -> dict[str, Any]:
return {
@@ -85,7 +97,159 @@ def list_database_connector_credentials(
tenant_id: str,
include_disabled: bool = False,
) -> list[ConnectorCredential]:
return [connector_credential_from_row(row) for row in list_connector_credential_rows(session, tenant_id=tenant_id, include_disabled=include_disabled)]
local = [
connector_credential_from_row(row)
for row in list_connector_credential_rows(
session,
tenant_id=tenant_id,
include_disabled=include_disabled,
)
]
return [
*local,
*list_reusable_connector_credentials(
session,
tenant_id=tenant_id,
include_disabled=include_disabled,
),
]
def reusable_credential_reference(credential_id: str) -> str:
return f"{CORE_CREDENTIAL_ENVELOPE_PREFIX}{credential_id}"
def reusable_credential_id(credential_ref: str | None) -> str | None:
if not credential_ref or not credential_ref.startswith(CORE_CREDENTIAL_ENVELOPE_PREFIX):
return None
value = credential_ref.removeprefix(CORE_CREDENTIAL_ENVELOPE_PREFIX).strip()
return value or None
def file_credential_context(
*,
tenant_id: str,
profile_id: str | None = None,
scope_type: str = "tenant",
scope_id: str | None = None,
administrative: bool = False,
) -> CredentialAccessContext:
return CredentialAccessContext(
tenant_id=tenant_id,
user_id=scope_id if scope_type == "user" else None,
group_ids=frozenset({scope_id}) if scope_type == "group" and scope_id else frozenset(),
target_scope_type=scope_type,
target_scope_id=scope_id or tenant_id,
module_id="files",
server_ref=f"files:{profile_id}" if profile_id else None,
administrative=administrative,
)
def list_reusable_connector_credentials(
session: Session,
*,
tenant_id: str,
include_disabled: bool = False,
) -> list[ConnectorCredential]:
if not inspect(session.get_bind()).has_table(CredentialEnvelope.__tablename__):
return []
context = file_credential_context(tenant_id=tenant_id, administrative=True)
return [
connector_credential_from_envelope(row)
for row in list_credential_envelopes(
session,
context=context,
include_inactive=include_disabled,
)
]
def resolve_reusable_connector_credential(
session: Session,
*,
tenant_id: str,
credential_ref: str,
profile_id: str,
scope_type: str,
scope_id: str | None,
) -> ConnectorCredential:
credential_id = reusable_credential_id(credential_ref)
if credential_id is None:
raise FileStorageError("Reusable credential reference is invalid")
try:
resolved = resolve_credential_envelope(
session,
credential_id=credential_id,
context=file_credential_context(
tenant_id=tenant_id,
profile_id=profile_id,
scope_type=scope_type,
scope_id=scope_id,
),
)
except CredentialEnvelopeError as exc:
raise FileStorageError("Reusable credential is unavailable to this file connection") from exc
return connector_credential_from_resolved_envelope(
resolved,
scope_type=scope_type,
scope_id=scope_id,
)
def connector_credential_from_envelope(row: CredentialEnvelope) -> ConnectorCredential:
return ConnectorCredential(
id=reusable_credential_reference(row.id),
label=row.name,
scope_type=row.scope_type,
scope_id=row.scope_id,
enabled=row.is_active,
credential_mode=_envelope_credential_mode(row.credential_kind, row.secret_keys),
username=_clean_public_value(row.public_data, "username"),
source_kind="credential_envelope",
has_secret=bool(row.secret_data_encrypted),
metadata={
"credential_envelope_id": row.id,
"credential_kind": row.credential_kind,
"allowed_modules": list(row.allowed_modules or []),
"allowed_server_refs": list(row.allowed_server_refs or []),
"inherit_to_lower_scopes": bool(row.inherit_to_lower_scopes),
"revision": row.revision,
},
)
def connector_credential_from_resolved_envelope(
row: ResolvedCredentialEnvelope,
*,
scope_type: str,
scope_id: str | None,
) -> ConnectorCredential:
return ConnectorCredential(
id=reusable_credential_reference(row.id),
label=row.name,
scope_type=scope_type,
scope_id=scope_id,
enabled=True,
credential_mode=_envelope_credential_mode(row.credential_kind, row.secret_data),
username=_clean_public_value(row.public_data, "username"),
password_value=_first_secret(row.secret_data, "password", "secret"),
token_value=_first_secret(
row.secret_data,
"access_token",
"bearer_token",
"token",
"api_key",
),
source_kind="credential_envelope",
has_secret=bool(row.secret_data),
metadata={
"credential_envelope_id": row.id,
"credential_kind": row.credential_kind,
"inherit_to_lower_scopes": True,
"revision": row.revision,
},
)
def list_connector_credential_rows(
@@ -348,6 +512,27 @@ def _policy_source_response(source: ConnectorPolicySource) -> dict[str, Any]:
}
def _envelope_credential_mode(kind: str, secret_values: Mapping[str, Any] | list[str]) -> str:
keys = {str(key) for key in secret_values}
if kind in {"token", "oauth2", "api_key"} or keys.intersection(
{"access_token", "bearer_token", "token", "api_key"}
):
return "token"
return "basic"
def _clean_public_value(values: Mapping[str, Any] | None, key: str) -> str | None:
return _clean((values or {}).get(key))
def _first_secret(values: Mapping[str, Any], *keys: str) -> str | None:
for key in keys:
value = _clean(values.get(key))
if value is not None:
return value
return None
def _clean(value: object) -> str | None:
if value is None:
return None
@@ -314,6 +314,7 @@ def _read_s3_file(profile: ConnectorProfile, *, library_id: str, path: str, max_
if not key:
raise ConnectorImportError("S3 import requires an object key")
client = _s3_import_client(profile)
try:
detail = _s3_object_detail(client, bucket=bucket, key=key, max_bytes=max_bytes)
version_id = _clean(detail.get("VersionId"))
response, data = _download_s3_object(
@@ -323,6 +324,10 @@ def _read_s3_file(profile: ConnectorProfile, *, library_id: str, path: str, max_
version_id=version_id,
max_bytes=max_bytes,
)
finally:
close = getattr(client, "close", None)
if callable(close):
close()
content_type = _clean(response.get("ContentType") if isinstance(response, dict) else None) or _clean(detail.get("ContentType")) or mimetypes.guess_type(key)[0]
etag = _clean(response.get("ETag") if isinstance(response, dict) else None) or _clean(detail.get("ETag"))
filename = filename_from_path(key)
@@ -246,11 +246,11 @@ def _applied_fields(policy: Mapping[str, Any]) -> tuple[str, ...]:
def _matches_field(request: ConnectorAccessRequest, field: str, patterns: list[str]) -> bool:
if field == "connectors":
return _matches_exact(request.connector_id, patterns)
return _matches_reference(request.connector_id, patterns)
if field == "credentials":
return _matches_exact(request.credential_id, patterns)
return _matches_reference(request.credential_id, patterns)
if field == "providers":
return _matches_exact(request.provider, patterns)
return _matches_reference(request.provider, patterns)
if field == "external_ids":
return _matches_glob(request.external_id, patterns)
if field == "external_paths":
@@ -260,11 +260,11 @@ def _matches_field(request: ConnectorAccessRequest, field: str, patterns: list[s
return False
def _matches_exact(value: str | None, patterns: list[str]) -> bool:
def _matches_reference(value: str | None, patterns: list[str]) -> bool:
if value is None:
return False
clean = value.casefold()
return any(pattern == "*" or clean == pattern.casefold() for pattern in patterns)
return any(fnmatchcase(clean, pattern.casefold()) for pattern in patterns)
def _matches_glob(value: str | None, patterns: list[str]) -> bool:
@@ -10,7 +10,13 @@ from govoplan_core.security.secrets import decrypt_secret, encrypt_secret
from govoplan_files.backend.db.models import FileConnectorCredential, FileConnectorProfile
from govoplan_files.backend.storage.common import FileStorageError
from govoplan_files.backend.storage.connector_deployment import reject_api_controlled_deployment_references
from govoplan_files.backend.storage.connector_credential_store import connector_credential_from_row, credential_rows_by_id
from govoplan_files.backend.storage.connector_credential_store import (
ConnectorCredential,
connector_credential_from_row,
credential_rows_by_id,
resolve_reusable_connector_credential,
reusable_credential_id,
)
from govoplan_files.backend.storage.connector_policy import connector_policy_sources_from_payload
from govoplan_files.backend.storage.connector_profiles import ConnectorProfile, supported_connector_providers
@@ -50,10 +56,28 @@ def select_database_connector_profiles(
profile_ids = {row.id for row in rows}
if row_visible is not None:
rows = [row for row in rows if row_visible(row)]
credential_ids = {_clean(row.credential_profile_id) for row in rows if _clean(row.credential_profile_id)}
credential_ids = {
_clean(row.credential_profile_id)
for row in rows
if _clean(row.credential_profile_id) and not reusable_credential_id(row.credential_profile_id)
}
credentials = credential_rows_by_id(session, tenant_id=tenant_id, credential_ids={item for item in credential_ids if item}, include_disabled=include_disabled)
return (
[connector_profile_from_row(row, credential_row=credentials.get(row.credential_profile_id or "")) for row in rows],
[
connector_profile_from_row(
row,
credential_row=(
_resolve_profile_reusable_credential(
session,
tenant_id=tenant_id,
row=row,
)
if reusable_credential_id(row.credential_profile_id)
else credentials.get(row.credential_profile_id or "")
),
)
for row in rows
],
profile_ids,
)
@@ -73,7 +97,10 @@ def list_connector_profile_rows(
return query.order_by(FileConnectorProfile.scope_type.asc(), FileConnectorProfile.label.asc()).all()
def connector_profile_from_row(row: FileConnectorProfile, credential_row: FileConnectorCredential | None = None) -> ConnectorProfile:
def connector_profile_from_row(
row: FileConnectorProfile,
credential_row: FileConnectorCredential | ConnectorCredential | None = None,
) -> ConnectorProfile:
policy_sources = []
if row.policy:
policy_sources = connector_policy_sources_from_payload({
@@ -82,9 +109,18 @@ def connector_profile_from_row(row: FileConnectorProfile, credential_row: FileCo
"label": row.label,
"policy": row.policy,
})
credential = connector_credential_from_row(credential_row) if credential_row is not None else None
credential = (
credential_row
if isinstance(credential_row, ConnectorCredential)
else connector_credential_from_row(credential_row)
if credential_row is not None
else None
)
if credential:
policy_sources.extend(credential.policy_sources)
metadata = dict(row.metadata_ or {})
if reusable_credential_id(row.credential_profile_id) and credential is None:
metadata["credential_unavailable"] = True
return ConnectorProfile(
id=row.id,
label=row.label,
@@ -105,11 +141,30 @@ def connector_profile_from_row(row: FileConnectorProfile, credential_row: FileCo
token_value=credential.token_value if credential else decrypt_secret(row.token_encrypted),
capabilities=tuple(_string_list(row.capabilities)),
policy_sources=tuple(policy_sources),
metadata=dict(row.metadata_ or {}),
metadata=metadata,
source_kind="database",
)
def _resolve_profile_reusable_credential(
session: Session,
*,
tenant_id: str,
row: FileConnectorProfile,
) -> ConnectorCredential | None:
try:
return resolve_reusable_connector_credential(
session,
tenant_id=tenant_id,
credential_ref=row.credential_profile_id or "",
profile_id=row.id,
scope_type=row.scope_type,
scope_id=row.scope_id,
)
except FileStorageError:
return None
def get_connector_profile_row(
session: Session,
*,
@@ -110,7 +110,7 @@ def connector_provider_descriptors() -> tuple[ConnectorProviderDescriptor, ...]:
conflict_strategy="Managed import/sync uses existing files conflict_strategy handling after download.",
preview_strategy="Previews are generated from the frozen managed file after import or sync, not directly from the share.",
audit_events=("files.connector.imported", "files.connector.synced", "files.connector.accessed"),
notes="Browse/import logic is implemented, but live smbprotocol access fails closed until the SDK supports connection-time DNS/IP pinning for initial connections and DFS referrals.",
notes="Initial sessions, reconnects, aliases, and DFS referral targets use the Files-owned pinned smbprotocol transport and the deployment-wide private-network policy.",
),
ConnectorProviderDescriptor(
provider="s3",
@@ -125,7 +125,7 @@ def connector_provider_descriptors() -> tuple[ConnectorProviderDescriptor, ...]:
conflict_strategy="Managed import/sync uses existing files conflict_strategy handling after object download.",
preview_strategy="Previews are generated from the frozen managed file after import or sync, not directly from the bucket.",
audit_events=("files.connector.imported", "files.connector.synced", "files.connector.accessed"),
notes="Browse/import logic is implemented, but live boto3 access fails closed until its HTTP transport supports connection-time DNS/IP pinning and redirect revalidation.",
notes="Every botocore connection pool uses the Files-owned pinned transport, including retries, redirects, endpoint discovery, and virtual-host aliases; outbound proxies and ambient credential discovery are disabled.",
),
ConnectorProviderDescriptor(
provider="sharepoint",
@@ -136,11 +136,6 @@ def connector_profile_usable_for_import(profile: ConnectorProfile) -> bool:
and descriptor.import_supported
):
return False
# These SDK paths intentionally fail closed until every connection peer and
# SDK-managed redirect/referral can be pinned and revalidated.
if profile.provider in {"s3", "smb"}:
return False
# Prove that the initial root browse performed by the current Files UI is
# policy-allowed. A later selected remote path/item is checked again.
return connector_policy_decision(
@@ -0,0 +1,93 @@
from __future__ import annotations
from govoplan_core.core.encryption import (
ContentProtectionRequest,
ContentUnprotectionRequest,
ProtectedContent,
encryption_content_cipher,
)
from govoplan_files.backend.runtime import get_registry
from govoplan_files.backend.storage.common import FileStorageError
FILES_PROTECTION_PROFILE = "files-server-envelope-v1"
def protect_blob_content(
session: object,
*,
tenant_id: str,
blob_id: str,
vault_id: str,
ciphertext_ref: str,
plaintext: bytes,
actor_id: str,
content_type: str | None,
) -> ProtectedContent:
capability = encryption_content_cipher(get_registry())
if capability is None:
raise FileStorageError(
"File encryption was requested, but the Encryption module is unavailable."
)
try:
return capability.protect_content(
session,
request=ContentProtectionRequest(
tenant_id=tenant_id,
owner_module="files",
resource_type="file_blob",
resource_id=blob_id,
profile_id=FILES_PROTECTION_PROFILE,
vault_id=vault_id,
ciphertext_ref=ciphertext_ref,
plaintext=plaintext,
policy_decision_ref="files:explicit-vault-selection:v1",
idempotency_key=f"file-blob:{blob_id}:content:v1",
actor_id=actor_id,
metadata={
"content_type": content_type or "application/octet-stream",
},
),
)
except Exception as exc:
raise FileStorageError(
"Managed file content could not be protected by the configured vault."
) from exc
def unprotect_blob_content(
session: object,
*,
tenant_id: str,
blob_id: str,
envelope_id: str,
ciphertext: bytes,
) -> bytes:
capability = encryption_content_cipher(get_registry())
if capability is None:
raise FileStorageError(
"This file is encrypted and cannot be read while Encryption is unavailable."
)
try:
return capability.unprotect_content(
session,
request=ContentUnprotectionRequest(
tenant_id=tenant_id,
owner_module="files",
resource_type="file_blob",
resource_id=blob_id,
envelope_id=envelope_id,
ciphertext=ciphertext,
),
)
except Exception as exc:
raise FileStorageError(
"Managed file content could not be opened with its protection envelope."
) from exc
__all__ = [
"FILES_PROTECTION_PROFILE",
"protect_blob_content",
"unprotect_blob_content",
]
+337 -24
View File
@@ -2,21 +2,32 @@ from __future__ import annotations
import hashlib
import mimetypes
from datetime import datetime
from pathlib import PurePosixPath
from typing import Any, Iterable
from uuid import uuid4
from sqlalchemy import and_, or_
from sqlalchemy import and_, exists, func, or_
from sqlalchemy.orm import Session
from govoplan_core.core.campaigns import CAPABILITY_CAMPAIGNS_ACCESS, CampaignAccessProvider
from govoplan_files.backend.db.models import CampaignAttachmentUse, FileAsset, FileBlob, FileShare, FileVersion
from govoplan_files.backend.runtime import get_registry, settings
from govoplan_files.backend.storage.access import ensure_owner_access, ensure_share_target_exists, user_group_ids
from govoplan_files.backend.storage.backends import StorageBackendError, get_storage_backend
from govoplan_files.backend.storage.backends import (
StorageBackendError,
StorageObjectMissing,
get_storage_backend,
)
from govoplan_files.backend.storage.common import FileConflictResolution, FileStorageError, UploadedStoredFile, utcnow
from govoplan_files.backend.storage.paths import filename_from_path, join_folder_filename, normalize_folder, normalize_logical_path, safe_storage_component
from govoplan_files.backend.storage.paths import filename_from_path, join_folder_filename, normalize_folder, normalize_logical_path
from govoplan_files.backend.storage.provenance import source_provenance_from_metadata
from govoplan_files.backend.storage.recovery import begin_blob_write_recovery
from govoplan_files.backend.storage.integrity import (
QUARANTINED_BLOB_STATUSES,
read_verified_blob_bytes,
)
from govoplan_files.backend.storage.share_state import effective_file_share_clause
def _campaign_access_provider() -> CampaignAccessProvider:
@@ -51,8 +62,9 @@ def _storage_backend_name() -> str:
return settings.file_storage_backend.lower().strip()
def _storage_key(*, tenant_id: str, checksum: str, filename: str) -> str:
return f"tenants/{tenant_id}/files/{checksum[:2]}/{uuid4().hex}-{safe_storage_component(filename)}"
def _storage_key(*, tenant_id: str, checksum: str) -> str:
# Object locators remain opaque so recovery evidence never persists names.
return f"tenants/{tenant_id}/files/{checksum[:2]}/{uuid4().hex}.blob"
def _get_or_create_blob(
@@ -62,40 +74,143 @@ def _get_or_create_blob(
data: bytes,
filename: str,
content_type: str | None,
actor_id: str,
encryption_vault_id: str | None = None,
) -> FileBlob:
checksum = hashlib.sha256(data).hexdigest()
size = len(data)
vault_id = str(encryption_vault_id or "").strip() or None
protection_discriminator = f"vault:{vault_id}" if vault_id else "plaintext"
blob = (
session.query(FileBlob)
.filter(FileBlob.tenant_id == tenant_id, FileBlob.checksum_sha256 == checksum, FileBlob.size_bytes == size)
.filter(FileBlob.tenant_id == tenant_id, FileBlob.checksum_sha256 == checksum, FileBlob.size_bytes == size, FileBlob.protection_discriminator == protection_discriminator)
.one_or_none()
)
if blob:
backend = get_storage_backend()
if not backend.exists(blob.storage_key):
repair_required = (
blob.integrity_status in QUARANTINED_BLOB_STATUSES
or blob.quarantined_at is not None
)
try:
backend.put_bytes(blob.storage_key, data, content_type=content_type)
backend.stat(blob.storage_key)
except StorageObjectMissing:
repair_required = True
except StorageBackendError as exc:
raise FileStorageError(str(exc)) from exc
if repair_required:
repair_token = hashlib.sha256(
repr(
(
blob.integrity_status,
blob.integrity_checked_at,
blob.quarantined_at,
blob.storage_checksum_sha256,
)
).encode("utf-8")
).hexdigest()
recovery = begin_blob_write_recovery(
session,
backend=backend,
tenant_id=tenant_id,
blob_id=blob.id,
storage_key=blob.storage_key,
semantic_checksum_sha256=checksum,
semantic_size_bytes=size,
protection_discriminator=protection_discriminator,
created_new=False,
repair_token=repair_token,
)
stored_data = data
expected_envelope_id = blob.encryption_envelope_id
if vault_id:
from govoplan_files.backend.storage.content_protection import protect_blob_content
protected = protect_blob_content(
session,
tenant_id=tenant_id,
blob_id=blob.id,
vault_id=vault_id,
ciphertext_ref=blob.storage_key,
plaintext=data,
actor_id=actor_id,
content_type=content_type,
)
if blob.encryption_envelope_id not in {None, protected.envelope.envelope_id}:
raise FileStorageError("The existing encrypted blob has another protection envelope.")
stored_data = protected.ciphertext
blob.encryption_envelope_id = protected.envelope.envelope_id
expected_envelope_id = protected.envelope.envelope_id
blob.storage_checksum_sha256 = hashlib.sha256(stored_data).hexdigest()
blob.storage_size_bytes = len(stored_data)
recovery.prepare_stored_bytes(
stored_data,
envelope_id=expected_envelope_id,
)
try:
backend.put_bytes(blob.storage_key, stored_data, content_type="application/octet-stream" if vault_id else content_type)
except StorageBackendError as exc:
raise FileStorageError(str(exc)) from exc
blob.integrity_status = "verified"
blob.integrity_checked_at = utcnow()
blob.integrity_failure = None
blob.quarantined_at = None
blob.ref_count += 1
session.add(blob)
return blob
storage_key = _storage_key(tenant_id=tenant_id, checksum=checksum, filename=filename)
blob_id = str(uuid4())
storage_key = _storage_key(tenant_id=tenant_id, checksum=checksum)
backend = get_storage_backend()
recovery = begin_blob_write_recovery(
session,
backend=backend,
tenant_id=tenant_id,
blob_id=blob_id,
storage_key=storage_key,
semantic_checksum_sha256=checksum,
semantic_size_bytes=size,
protection_discriminator=protection_discriminator,
created_new=True,
)
stored_data = data
envelope_id = None
if vault_id:
from govoplan_files.backend.storage.content_protection import protect_blob_content
protected = protect_blob_content(
session,
tenant_id=tenant_id,
blob_id=blob_id,
vault_id=vault_id,
ciphertext_ref=storage_key,
plaintext=data,
actor_id=actor_id,
content_type=content_type,
)
stored_data = protected.ciphertext
envelope_id = protected.envelope.envelope_id
recovery.prepare_stored_bytes(stored_data, envelope_id=envelope_id)
try:
backend.put_bytes(storage_key, data, content_type=content_type)
backend.put_bytes(storage_key, stored_data, content_type="application/octet-stream" if vault_id else content_type)
except StorageBackendError as exc:
raise FileStorageError(str(exc)) from exc
blob = FileBlob(
id=blob_id,
tenant_id=tenant_id,
storage_backend=_storage_backend_name(),
storage_bucket=_storage_bucket_name(),
storage_key=storage_key,
checksum_sha256=checksum,
size_bytes=size,
protection_discriminator=protection_discriminator,
encryption_envelope_id=envelope_id,
storage_checksum_sha256=hashlib.sha256(stored_data).hexdigest() if vault_id else None,
storage_size_bytes=len(stored_data) if vault_id else None,
content_type=content_type,
ref_count=1,
integrity_status="verified",
integrity_checked_at=utcnow(),
)
session.add(blob)
session.flush()
@@ -120,6 +235,7 @@ def create_file_asset(
conflict_strategy: str = "reject",
conflict_resolutions: Iterable[FileConflictResolution] | None = None,
is_admin: bool = False,
encryption_vault_id: str | None = None,
) -> UploadedStoredFile:
owner_type = owner_type.lower().strip()
ensure_owner_access(session, tenant_id=tenant_id, owner_type=owner_type, owner_id=owner_id, user_id=user_id, is_admin=is_admin)
@@ -147,7 +263,7 @@ def create_file_asset(
elif action == "rename":
logical_path = _next_available_logical_path(session, tenant_id=tenant_id, owner_type=owner_type, owner_id=owner_id, desired_path=logical_path)
blob = _get_or_create_blob(session, tenant_id=tenant_id, data=data, filename=safe_filename, content_type=content_type)
blob = _get_or_create_blob(session, tenant_id=tenant_id, data=data, filename=safe_filename, content_type=content_type, actor_id=user_id, encryption_vault_id=encryption_vault_id)
asset = FileAsset(
tenant_id=tenant_id,
owner_type=owner_type,
@@ -279,6 +395,7 @@ def update_file_asset_content(
data: bytes,
content_type: str | None,
metadata: dict[str, Any],
encryption_vault_id: str | None = None,
) -> tuple[UploadedStoredFile, str]:
if asset.tenant_id != tenant_id or asset.deleted_at is not None:
raise FileStorageError("File not found")
@@ -289,10 +406,23 @@ def update_file_asset_content(
checksum = hashlib.sha256(data).hexdigest()
asset.metadata_ = metadata
session.add(asset)
if current_blob.checksum_sha256 == checksum and current_blob.size_bytes == len(data):
inherited_vault_id = encryption_vault_id
if inherited_vault_id is None and current_blob.encryption_envelope_id:
prefix = "vault:"
if current_blob.protection_discriminator.startswith(prefix):
inherited_vault_id = current_blob.protection_discriminator[len(prefix) :]
inherited_vault_id = str(inherited_vault_id or "").strip() or None
target_protection = (
f"vault:{inherited_vault_id}" if inherited_vault_id else "plaintext"
)
if (
current_blob.checksum_sha256 == checksum
and current_blob.size_bytes == len(data)
and current_blob.protection_discriminator == target_protection
):
return UploadedStoredFile(asset=asset, version=current_version, blob=current_blob), "unchanged"
blob = _get_or_create_blob(session, tenant_id=tenant_id, data=data, filename=safe_filename, content_type=content_type)
blob = _get_or_create_blob(session, tenant_id=tenant_id, data=data, filename=safe_filename, content_type=content_type, actor_id=user_id, encryption_vault_id=inherited_vault_id)
version = FileVersion(
tenant_id=tenant_id,
file_asset_id=asset.id,
@@ -328,7 +458,7 @@ def get_asset_for_user(session: Session, *, tenant_id: str, user_id: str, asset_
.filter(
FileShare.tenant_id == tenant_id,
FileShare.file_asset_id == asset.id,
FileShare.revoked_at.is_(None),
effective_file_share_clause(),
FileShare.permission.in_(permission_values),
or_(
(FileShare.target_type == "user") & (FileShare.target_id == user_id),
@@ -343,6 +473,32 @@ def get_asset_for_user(session: Session, *, tenant_id: str, user_id: str, asset_
return asset
def get_asset_for_share_management(
session: Session,
*,
tenant_id: str,
user_id: str,
asset_id: str,
is_admin: bool = False,
) -> FileAsset:
asset = session.get(FileAsset, asset_id)
if not asset or asset.tenant_id != tenant_id or asset.deleted_at is not None:
raise FileStorageError("File not found")
if is_admin:
return asset
owns_asset = (
asset.owner_type == "user"
and asset.owner_user_id == user_id
) or (
asset.owner_type == "group"
and asset.owner_group_id
in user_group_ids(session, tenant_id=tenant_id, user_id=user_id)
)
if not owns_asset:
raise FileStorageError("Only file owners and administrators can manage shares")
return asset
def list_assets_for_user(
session: Session,
*,
@@ -352,6 +508,8 @@ def list_assets_for_user(
owner_id: str | None = None,
campaign_id: str | None = None,
path_prefix: str | None = None,
campaign_usage: str | None = None,
audit_relevant: bool | None = None,
include_deleted: bool = False,
is_admin: bool = False,
) -> list[FileAsset]:
@@ -363,6 +521,8 @@ def list_assets_for_user(
owner_id=owner_id,
campaign_id=campaign_id,
path_prefix=path_prefix,
campaign_usage=campaign_usage,
audit_relevant=audit_relevant,
include_deleted=include_deleted,
is_admin=is_admin,
)
@@ -378,6 +538,8 @@ def list_assets_for_user_window(
owner_id: str | None = None,
campaign_id: str | None = None,
path_prefix: str | None = None,
campaign_usage: str | None = None,
audit_relevant: bool | None = None,
include_deleted: bool = False,
is_admin: bool = False,
page_size: int,
@@ -393,6 +555,8 @@ def list_assets_for_user_window(
owner_id=owner_id,
campaign_id=campaign_id,
path_prefix=path_prefix,
campaign_usage=campaign_usage,
audit_relevant=audit_relevant,
include_deleted=include_deleted,
is_admin=is_admin,
)
@@ -417,6 +581,8 @@ def _asset_visibility_query_for_user(
owner_id: str | None = None,
campaign_id: str | None = None,
path_prefix: str | None = None,
campaign_usage: str | None = None,
audit_relevant: bool | None = None,
include_deleted: bool = False,
is_admin: bool = False,
):
@@ -430,30 +596,97 @@ def _asset_visibility_query_for_user(
if owner_type == "group" and owner_id:
query = query.filter(FileAsset.owner_group_id == owner_id)
if campaign_id:
query = query.join(FileShare, FileShare.file_asset_id == FileAsset.id).filter(
campaign_share = exists().where(
FileShare.tenant_id == tenant_id,
FileShare.file_asset_id == FileAsset.id,
FileShare.target_type == "campaign",
FileShare.target_id == campaign_id,
FileShare.revoked_at.is_(None),
effective_file_share_clause(),
)
query = query.filter(campaign_share)
elif not is_admin and not owner_type:
group_ids = user_group_ids(session, tenant_id=tenant_id, user_id=user_id)
query = query.outerjoin(FileShare, FileShare.file_asset_id == FileAsset.id).filter(
active_share = exists().where(
FileShare.tenant_id == tenant_id,
FileShare.file_asset_id == FileAsset.id,
effective_file_share_clause(),
or_(
(FileShare.target_type == "user") & (FileShare.target_id == user_id),
(FileShare.target_type == "group") & (FileShare.target_id.in_(group_ids)),
(FileShare.target_type == "tenant") & (FileShare.target_id == tenant_id),
)
)
query = query.filter(
or_(
(FileAsset.owner_type == "user") & (FileAsset.owner_user_id == user_id),
(FileAsset.owner_type == "group") & (FileAsset.owner_group_id.in_(group_ids)),
(FileShare.revoked_at.is_(None)) & (FileShare.target_type == "user") & (FileShare.target_id == user_id),
(FileShare.revoked_at.is_(None)) & (FileShare.target_type == "group") & (FileShare.target_id.in_(group_ids)),
(FileShare.revoked_at.is_(None)) & (FileShare.target_type == "tenant") & (FileShare.target_id == tenant_id),
active_share,
)
)
if path_prefix:
prefix = normalize_folder(path_prefix)
if prefix:
query = query.filter(FileAsset.display_path.like(f"{prefix}/%"))
campaign_share_exists = exists().where(
FileShare.tenant_id == tenant_id,
FileShare.file_asset_id == FileAsset.id,
FileShare.target_type == "campaign",
effective_file_share_clause(),
)
campaign_use_exists = exists().where(
CampaignAttachmentUse.tenant_id == tenant_id,
CampaignAttachmentUse.file_asset_id == FileAsset.id,
)
if campaign_usage == "linked":
query = query.filter(or_(campaign_share_exists, campaign_use_exists))
elif campaign_usage == "unlinked":
query = query.filter(~or_(campaign_share_exists, campaign_use_exists))
if audit_relevant is not None:
sent_use_exists = exists().where(
CampaignAttachmentUse.tenant_id == tenant_id,
CampaignAttachmentUse.file_asset_id == FileAsset.id,
CampaignAttachmentUse.use_stage == "sent",
)
query = query.filter(sent_use_exists if audit_relevant else ~sent_use_exists)
return query
def count_assets_for_user(
session: Session,
*,
tenant_id: str,
user_id: str,
owner_type: str | None = None,
owner_id: str | None = None,
campaign_id: str | None = None,
path_prefix: str | None = None,
campaign_usage: str | None = None,
audit_relevant: bool | None = None,
include_deleted: bool = False,
is_admin: bool = False,
) -> int:
query = _asset_visibility_query_for_user(
session,
tenant_id=tenant_id,
user_id=user_id,
owner_type=owner_type,
owner_id=owner_id,
campaign_id=campaign_id,
path_prefix=path_prefix,
campaign_usage=campaign_usage,
audit_relevant=audit_relevant,
include_deleted=include_deleted,
is_admin=is_admin,
)
return int(
query.order_by(None)
.with_entities(func.count(func.distinct(FileAsset.id)))
.scalar()
or 0
)
def current_version_and_blob(session: Session, asset: FileAsset) -> tuple[FileVersion, FileBlob]:
if not asset.current_version_id:
raise FileStorageError("File has no current version")
@@ -495,10 +728,7 @@ def current_versions_and_blobs(session: Session, assets: Iterable[FileAsset]) ->
def read_asset_bytes(session: Session, asset: FileAsset) -> tuple[bytes, FileVersion, FileBlob]:
version, blob = current_version_and_blob(session, asset)
backend = get_storage_backend()
try:
return backend.get_bytes(blob.storage_key), version, blob
except StorageBackendError as exc:
raise FileStorageError(str(exc)) from exc
return read_verified_blob_bytes(blob, backend=backend), version, blob
def share_file(
@@ -510,6 +740,7 @@ def share_file(
target_id: str,
permission: str,
user_id: str,
expires_at: datetime | None = None,
) -> FileShare:
target_type = target_type.lower().strip()
permission = permission.lower().strip()
@@ -519,6 +750,7 @@ def share_file(
raise FileStorageError("Unsupported share target")
if permission not in {"read", "write", "manage"}:
raise FileStorageError("Unsupported file permission")
_validate_share_expiry(expires_at)
if target_type in {"user", "group", "tenant"}:
ensure_share_target_exists(tenant_id=tenant_id, target_type=target_type, target_id=target_id)
if target_type == "campaign":
@@ -536,6 +768,7 @@ def share_file(
)
if existing:
existing.permission = permission
existing.expires_at = expires_at
session.add(existing)
return existing
share = FileShare(
@@ -545,6 +778,7 @@ def share_file(
target_id=target_id,
permission=permission,
created_by_user_id=user_id,
expires_at=expires_at,
)
session.add(share)
return share
@@ -559,6 +793,7 @@ def share_files(
target_id: str,
permission: str,
user_id: str,
expires_at: datetime | None = None,
) -> list[FileShare]:
target_type = target_type.lower().strip()
permission = permission.lower().strip()
@@ -566,6 +801,7 @@ def share_files(
raise FileStorageError("Unsupported share target")
if permission not in {"read", "write", "manage"}:
raise FileStorageError("Unsupported file permission")
_validate_share_expiry(expires_at)
if target_type in {"user", "group", "tenant"}:
ensure_share_target_exists(tenant_id=tenant_id, target_type=target_type, target_id=target_id)
if target_type == "campaign":
@@ -593,6 +829,7 @@ def share_files(
existing = existing_by_asset.get(asset.id)
if existing:
existing.permission = permission
existing.expires_at = expires_at
session.add(existing)
shares.append(existing)
continue
@@ -603,12 +840,88 @@ def share_files(
target_id=target_id,
permission=permission,
created_by_user_id=user_id,
expires_at=expires_at,
)
session.add(share)
shares.append(share)
return shares
def list_file_shares(
session: Session,
*,
tenant_id: str,
asset_id: str,
include_inactive: bool = False,
) -> list[FileShare]:
query = session.query(FileShare).filter(
FileShare.tenant_id == tenant_id,
FileShare.file_asset_id == asset_id,
)
if not include_inactive:
query = query.filter(effective_file_share_clause())
return query.order_by(FileShare.created_at.desc(), FileShare.id.desc()).all()
def revoke_file_share(
session: Session,
*,
tenant_id: str,
asset_id: str,
share_id: str,
user_id: str,
) -> tuple[FileShare, bool]:
share = (
session.query(FileShare)
.filter(
FileShare.id == share_id,
FileShare.tenant_id == tenant_id,
FileShare.file_asset_id == asset_id,
)
.one_or_none()
)
if share is None:
raise FileStorageError("File share not found")
if share.revoked_at is not None:
return share, False
share.revoked_at = utcnow()
share.revoked_by_user_id = user_id
session.add(share)
return share, True
def current_file_share_for_target(
session: Session,
*,
tenant_id: str,
asset_id: str,
target_type: str,
target_id: str,
) -> FileShare | None:
return (
session.query(FileShare)
.filter(
FileShare.tenant_id == tenant_id,
FileShare.file_asset_id == asset_id,
FileShare.target_type == target_type.lower().strip(),
FileShare.target_id == target_id,
FileShare.revoked_at.is_(None),
)
.order_by(FileShare.created_at.desc(), FileShare.id.desc())
.first()
)
def _validate_share_expiry(expires_at: datetime | None) -> None:
if expires_at is None:
return
from govoplan_files.backend.storage.share_state import file_share_is_active
candidate = FileShare(expires_at=expires_at)
if not file_share_is_active(candidate):
raise FileStorageError("File share expiry must be in the future")
def soft_delete_assets(session: Session, assets: Iterable[FileAsset]) -> int:
count = 0
@@ -185,14 +185,22 @@ class _OutboundPolicyHTTPTransport(httpx.BaseTransport):
self._connection_pool.close()
@contextmanager
def _stream_connector_request(method: str, url: str, **kwargs: Any) -> Iterator[httpx.Response]:
with httpx.Client(
_CONNECTOR_HTTP_CLIENT = httpx.Client(
transport=_OutboundPolicyHTTPTransport(),
follow_redirects=False,
timeout=kwargs.pop("timeout", 15.0),
) as client:
with client.stream(method, url, **kwargs) as response:
timeout=15.0,
)
@contextmanager
def _stream_connector_request(method: str, url: str, **kwargs: Any) -> Iterator[httpx.Response]:
timeout = kwargs.pop("timeout", 15.0)
with _CONNECTOR_HTTP_CLIENT.stream(
method,
url,
timeout=timeout,
**kwargs,
) as response:
yield response
@@ -0,0 +1,550 @@
from __future__ import annotations
import hashlib
from dataclasses import dataclass
from sqlalchemy.orm import Session, object_session
from govoplan_files.backend.db.models import (
FileBlob,
FileIntegrityFinding,
FileIntegrityScan,
)
from govoplan_files.backend.storage.backends import (
StorageBackend,
StorageBackendError,
StorageObjectMissing,
get_storage_backend,
)
from govoplan_files.backend.storage.recovery import (
begin_orphan_cleanup_recovery,
)
from govoplan_files.backend.storage.common import FileStorageError, utcnow
TERMINAL_SCAN_STATUSES = {"completed", "cancelled"}
QUARANTINED_BLOB_STATUSES = {
"missing",
"size_mismatch",
"checksum_mismatch",
}
@dataclass(frozen=True, slots=True)
class BlobInspection:
valid: bool
kind: str
observed_size_bytes: int | None = None
observed_checksum_sha256: str | None = None
@dataclass(frozen=True, slots=True)
class IntegrityActionResult:
action: str
changed: bool
dry_run: bool
finding: FileIntegrityFinding
inspection: BlobInspection | None = None
def storage_prefix_for_tenant(tenant_id: str) -> str:
return f"tenants/{tenant_id}/files/"
def create_integrity_scan(
session: Session,
*,
tenant_id: str,
user_id: str,
verify_checksums: bool = True,
batch_size: int = 100,
backend: StorageBackend | None = None,
) -> FileIntegrityScan:
active_backend = backend or get_storage_backend()
scan = FileIntegrityScan(
tenant_id=tenant_id,
storage_backend=active_backend.name,
storage_prefix=storage_prefix_for_tenant(tenant_id),
verify_checksums=verify_checksums,
batch_size=max(1, min(int(batch_size), 1000)),
created_by_user_id=user_id,
)
session.add(scan)
session.flush()
return scan
def run_integrity_scan_batch(
session: Session,
scan: FileIntegrityScan,
*,
backend: StorageBackend | None = None,
) -> FileIntegrityScan:
if scan.status in TERMINAL_SCAN_STATUSES:
return scan
active_backend = backend or get_storage_backend()
if active_backend.name != scan.storage_backend:
raise FileStorageError(
"The configured storage backend changed after this integrity scan started"
)
if scan.started_at is None:
scan.started_at = utcnow()
scan.status = "running"
scan.last_error = None
if scan.phase == "blobs":
_scan_blob_batch(session, scan, active_backend)
elif scan.phase == "objects":
_scan_object_batch(session, scan, active_backend)
else:
scan.phase = "completed"
scan.status = "completed"
scan.completed_at = utcnow()
scan.revision += 1
session.add(scan)
return scan
def mark_integrity_scan_failed(
scan: FileIntegrityScan,
*,
error: Exception,
) -> None:
scan.status = "failed"
scan.last_error = type(error).__name__[:255]
scan.revision += 1
def inspect_blob(
blob: FileBlob,
*,
backend: StorageBackend,
verify_checksum: bool = True,
) -> BlobInspection:
expected_size = blob.storage_size_bytes if blob.storage_size_bytes is not None else blob.size_bytes
expected_checksum = blob.storage_checksum_sha256 if blob.storage_checksum_sha256 is not None else blob.checksum_sha256
try:
info = backend.stat(blob.storage_key)
except StorageObjectMissing:
return BlobInspection(valid=False, kind="missing")
if info.size_bytes != expected_size:
return BlobInspection(
valid=False,
kind="size_mismatch",
observed_size_bytes=info.size_bytes,
)
if not verify_checksum:
return BlobInspection(
valid=True,
kind="verified",
observed_size_bytes=info.size_bytes,
)
digest = hashlib.sha256()
observed_size = 0
for chunk in backend.iter_bytes(blob.storage_key):
observed_size += len(chunk)
digest.update(chunk)
observed_checksum = digest.hexdigest()
if observed_size != expected_size:
return BlobInspection(
valid=False,
kind="size_mismatch",
observed_size_bytes=observed_size,
observed_checksum_sha256=observed_checksum,
)
if observed_checksum != expected_checksum:
return BlobInspection(
valid=False,
kind="checksum_mismatch",
observed_size_bytes=observed_size,
observed_checksum_sha256=observed_checksum,
)
return BlobInspection(
valid=True,
kind="verified",
observed_size_bytes=observed_size,
observed_checksum_sha256=observed_checksum,
)
def apply_blob_inspection(
session: Session,
blob: FileBlob,
inspection: BlobInspection,
*,
checked_at=None,
) -> None:
timestamp = checked_at or utcnow()
blob.integrity_checked_at = timestamp
if inspection.valid:
blob.integrity_status = "verified"
blob.integrity_failure = None
blob.quarantined_at = None
else:
blob.integrity_status = inspection.kind
blob.integrity_failure = inspection.kind
blob.quarantined_at = blob.quarantined_at or timestamp
session.add(blob)
def ensure_blob_is_readable(blob: FileBlob) -> None:
if blob.integrity_status in QUARANTINED_BLOB_STATUSES or blob.quarantined_at:
raise FileStorageError(
"Managed file content is quarantined because integrity verification failed"
)
def read_verified_blob_bytes(
blob: FileBlob,
*,
backend: StorageBackend,
) -> bytes:
ensure_blob_is_readable(blob)
try:
data = backend.get_bytes(blob.storage_key)
except StorageBackendError as exc:
raise FileStorageError("Managed file content is not available") from exc
expected_storage_size = blob.storage_size_bytes if blob.storage_size_bytes is not None else blob.size_bytes
expected_storage_checksum = blob.storage_checksum_sha256 if blob.storage_checksum_sha256 is not None else blob.checksum_sha256
if len(data) != expected_storage_size:
raise FileStorageError(
"Managed file content failed its recorded size verification"
)
if hashlib.sha256(data).hexdigest() != expected_storage_checksum:
raise FileStorageError(
"Managed file content failed its recorded checksum verification"
)
if blob.encryption_envelope_id:
session = object_session(blob)
if session is None:
raise FileStorageError("Encrypted managed content requires an attached database session")
from govoplan_files.backend.storage.content_protection import unprotect_blob_content
data = unprotect_blob_content(
session,
tenant_id=blob.tenant_id,
blob_id=blob.id,
envelope_id=blob.encryption_envelope_id,
ciphertext=data,
)
if len(data) != blob.size_bytes:
raise FileStorageError("Decrypted managed file content failed its recorded size verification")
if hashlib.sha256(data).hexdigest() != blob.checksum_sha256:
raise FileStorageError("Decrypted managed file content failed its recorded checksum verification")
return data
def recheck_integrity_finding(
session: Session,
finding: FileIntegrityFinding,
*,
user_id: str,
dry_run: bool,
backend: StorageBackend | None = None,
) -> IntegrityActionResult:
if finding.kind == "orphan_object" or not finding.blob_id:
raise FileStorageError("Only blob integrity findings can be rechecked")
blob = session.get(FileBlob, finding.blob_id)
if blob is None or blob.tenant_id != finding.tenant_id:
raise FileStorageError("Integrity finding blob no longer exists")
active_backend = backend or get_storage_backend()
scan = session.get(FileIntegrityScan, finding.scan_id)
if scan is None or scan.storage_backend != active_backend.name:
raise FileStorageError(
"The configured storage backend does not match the integrity finding"
)
inspection = inspect_blob(
blob,
backend=active_backend,
verify_checksum=True,
)
changed = False
if not dry_run:
previous = (
blob.integrity_status,
blob.quarantined_at,
finding.state,
)
apply_blob_inspection(session, blob, inspection)
_update_finding_from_inspection(finding, inspection)
if inspection.valid:
finding.state = "resolved"
finding.resolved_at = utcnow()
finding.resolved_by_user_id = user_id
session.add(finding)
finding.revision += 1
changed = previous != (
blob.integrity_status,
blob.quarantined_at,
finding.state,
)
return IntegrityActionResult(
action="recheck",
changed=changed,
dry_run=dry_run,
finding=finding,
inspection=inspection,
)
def cleanup_orphan_finding(
session: Session,
finding: FileIntegrityFinding,
*,
user_id: str,
dry_run: bool,
backend: StorageBackend | None = None,
) -> IntegrityActionResult:
if finding.kind != "orphan_object":
raise FileStorageError("Only orphan-object findings can be cleaned up")
scan = session.get(FileIntegrityScan, finding.scan_id)
if scan is None or scan.tenant_id != finding.tenant_id:
raise FileStorageError("Integrity scan not found")
if not finding.storage_key.startswith(scan.storage_prefix):
raise FileStorageError("Orphan cleanup is outside the scan storage scope")
referenced = (
session.query(FileBlob.id)
.filter(
FileBlob.tenant_id == finding.tenant_id,
FileBlob.storage_backend == scan.storage_backend,
FileBlob.storage_key == finding.storage_key,
)
.first()
)
if referenced:
if not dry_run and finding.state != "resolved":
finding.state = "resolved"
finding.resolved_at = utcnow()
finding.resolved_by_user_id = user_id
finding.revision += 1
session.add(finding)
return IntegrityActionResult(
action="retained_referenced",
changed=True,
dry_run=False,
finding=finding,
)
return IntegrityActionResult(
action="retained_referenced",
changed=False,
dry_run=dry_run,
finding=finding,
)
if finding.state == "deleted":
return IntegrityActionResult(
action="already_deleted",
changed=False,
dry_run=dry_run,
finding=finding,
)
if dry_run:
return IntegrityActionResult(
action="would_delete",
changed=False,
dry_run=True,
finding=finding,
)
active_backend = backend or get_storage_backend()
if active_backend.name != scan.storage_backend:
raise FileStorageError(
"The configured storage backend does not match the integrity finding"
)
begin_orphan_cleanup_recovery(
session,
finding,
backend=active_backend,
user_id=user_id,
)
try:
active_backend.stat(finding.storage_key)
except StorageObjectMissing:
action = "already_absent"
else:
active_backend.delete(finding.storage_key)
action = "deleted"
finding.state = "deleted"
finding.resolved_at = utcnow()
finding.resolved_by_user_id = user_id
finding.revision += 1
session.add(finding)
return IntegrityActionResult(
action=action,
changed=True,
dry_run=False,
finding=finding,
)
def _scan_blob_batch(
session: Session,
scan: FileIntegrityScan,
backend: StorageBackend,
) -> None:
query = session.query(FileBlob).filter(
FileBlob.tenant_id == scan.tenant_id,
FileBlob.storage_backend == scan.storage_backend,
)
if scan.blob_cursor:
query = query.filter(FileBlob.id > scan.blob_cursor)
blobs = query.order_by(FileBlob.id.asc()).limit(scan.batch_size).all()
for blob in blobs:
inspection = inspect_blob(
blob,
backend=backend,
verify_checksum=scan.verify_checksums,
)
apply_blob_inspection(session, blob, inspection)
scan.scanned_blob_count += 1
if inspection.valid:
scan.verified_blob_count += 1
_resolve_scan_blob_findings(session, scan, blob)
else:
scan.quarantined_blob_count += 1
_record_blob_finding(session, scan, blob, inspection)
scan.blob_cursor = blob.id
if len(blobs) < scan.batch_size:
scan.phase = "objects"
scan.object_cursor = None
def _scan_object_batch(
session: Session,
scan: FileIntegrityScan,
backend: StorageBackend,
) -> None:
page = backend.list_objects(
prefix=scan.storage_prefix,
after=scan.object_cursor,
limit=scan.batch_size,
)
keys = [item.key for item in page.objects]
referenced_keys = {
row[0]
for row in session.query(FileBlob.storage_key)
.filter(
FileBlob.tenant_id == scan.tenant_id,
FileBlob.storage_backend == scan.storage_backend,
FileBlob.storage_key.in_(keys),
)
.all()
} if keys else set()
for item in page.objects:
scan.scanned_object_count += 1
if item.key in referenced_keys:
continue
scan.orphan_object_count += 1
_record_orphan_finding(
session,
scan,
storage_key=item.key,
size_bytes=item.size_bytes,
)
scan.object_cursor = page.next_cursor
if page.next_cursor is None:
scan.phase = "completed"
scan.status = "completed"
scan.completed_at = utcnow()
def _record_blob_finding(
session: Session,
scan: FileIntegrityScan,
blob: FileBlob,
inspection: BlobInspection,
) -> FileIntegrityFinding:
finding = (
session.query(FileIntegrityFinding)
.filter(
FileIntegrityFinding.scan_id == scan.id,
FileIntegrityFinding.blob_id == blob.id,
FileIntegrityFinding.kind == inspection.kind,
)
.one_or_none()
)
if finding is None:
finding = FileIntegrityFinding(
scan_id=scan.id,
tenant_id=scan.tenant_id,
kind=inspection.kind,
blob_id=blob.id,
storage_key=blob.storage_key,
expected_size_bytes=blob.storage_size_bytes if blob.storage_size_bytes is not None else blob.size_bytes,
expected_checksum_sha256=blob.storage_checksum_sha256 if blob.storage_checksum_sha256 is not None else blob.checksum_sha256,
)
else:
finding.revision += 1
_update_finding_from_inspection(finding, inspection)
session.add(finding)
return finding
def _record_orphan_finding(
session: Session,
scan: FileIntegrityScan,
*,
storage_key: str,
size_bytes: int,
) -> FileIntegrityFinding:
finding = (
session.query(FileIntegrityFinding)
.filter(
FileIntegrityFinding.scan_id == scan.id,
FileIntegrityFinding.kind == "orphan_object",
FileIntegrityFinding.storage_key == storage_key,
)
.one_or_none()
)
if finding is None:
finding = FileIntegrityFinding(
scan_id=scan.id,
tenant_id=scan.tenant_id,
kind="orphan_object",
storage_key=storage_key,
observed_size_bytes=size_bytes,
)
session.add(finding)
return finding
def _resolve_scan_blob_findings(
session: Session,
scan: FileIntegrityScan,
blob: FileBlob,
) -> None:
for finding in (
session.query(FileIntegrityFinding)
.filter(
FileIntegrityFinding.scan_id == scan.id,
FileIntegrityFinding.blob_id == blob.id,
FileIntegrityFinding.state == "open",
)
.all()
):
finding.state = "resolved"
finding.resolved_at = utcnow()
finding.revision += 1
session.add(finding)
def _update_finding_from_inspection(
finding: FileIntegrityFinding,
inspection: BlobInspection,
) -> None:
finding.observed_size_bytes = inspection.observed_size_bytes
finding.observed_checksum_sha256 = inspection.observed_checksum_sha256
__all__ = [
"BlobInspection",
"IntegrityActionResult",
"QUARANTINED_BLOB_STATUSES",
"apply_blob_inspection",
"cleanup_orphan_finding",
"create_integrity_scan",
"ensure_blob_is_readable",
"inspect_blob",
"mark_integrity_scan_failed",
"read_verified_blob_bytes",
"recheck_integrity_finding",
"run_integrity_scan_batch",
"storage_prefix_for_tenant",
]
@@ -0,0 +1,691 @@
from __future__ import annotations
from dataclasses import dataclass
from datetime import UTC, datetime
import hashlib
from typing import Protocol
from sqlalchemy import event
from sqlalchemy.orm import Session
from govoplan_core.core.recovery import (
RecoveryGuaranteeError,
RecoveryMode,
RecoveryPlan,
RecoveryStatus,
)
from govoplan_core.core.recovery_runtime import (
DurableRecoveryOperation,
RecoveryOperationBusy,
RecoveryOperationStateConflict,
begin_durable_recovery_operation,
)
from govoplan_core.core.runtime_coordination import process_runtime_identity
from govoplan_core.db.session import get_database
from govoplan_files.backend.db.models import (
FileBlob,
FileIntegrityFinding,
FileIntegrityScan,
)
from govoplan_files.backend.storage.backends import (
StorageBackend,
StorageBackendError,
StorageObjectMissing,
)
from govoplan_files.backend.storage.common import FileStorageError
_PENDING_EFFECTS_KEY = "govoplan_files_pending_recovery_effects"
_HOOKS_INSTALLED_KEY = "govoplan_files_recovery_hooks_installed"
_ROLLBACK_ERRORS_KEY = "govoplan_files_recovery_rollback_errors"
class _PendingEffect(Protocol):
def settle(self, *, committed: bool) -> None: ...
@dataclass(slots=True)
class PendingBlobWrite:
operation: DurableRecoveryOperation
backend: StorageBackend
tenant_id: str
blob_id: str
storage_key: str
semantic_checksum_sha256: str
semantic_size_bytes: int
protection_discriminator: str
created_new: bool
expected_storage_checksum_sha256: str | None = None
expected_storage_size_bytes: int | None = None
expected_envelope_id: str | None = None
def prepare_stored_bytes(
self,
data: bytes,
*,
envelope_id: str | None,
) -> None:
"""Retain process-local verification evidence without recording content."""
self.expected_storage_checksum_sha256 = hashlib.sha256(data).hexdigest()
self.expected_storage_size_bytes = len(data)
self.expected_envelope_id = envelope_id
def settle(self, *, committed: bool) -> None:
evidence = _blob_write_evidence(self)
if _blob_write_complete(evidence):
self.operation.succeed(evidence=evidence)
return
if self.created_new and evidence.get("database_blob_present") is False:
self._settle_unreferenced_new_object(evidence)
return
if not self.created_new and _object_matches(evidence):
if _forward_complete_blob_repair(self):
completed = _blob_write_evidence(self)
if _blob_write_complete(completed):
self.operation.succeed(evidence=completed)
return
if evidence.get("database_blob_present") is True and not _object_matches(
evidence
):
_quarantine_blob_after_failed_verification(self, evidence)
evidence = _blob_write_evidence(self)
status = (
RecoveryStatus.OUTCOME_UNKNOWN
if not evidence.get("verified")
else RecoveryStatus.RECOVERY_REQUIRED
)
transaction = "committed" if committed else "rolled back"
self.operation.unresolved(
status=status,
summary=f"Managed Files blob write remained unresolved after the database transaction {transaction}",
evidence=evidence,
failure_summary=(
"The managed object and Files blob metadata require reconciliation"
),
)
def _settle_unreferenced_new_object(self, evidence: dict[str, object]) -> None:
object_present = evidence.get("object_present")
if object_present is False:
self.operation.reject(
summary="The Files blob write left no durable object or database row",
evidence=evidence,
)
return
if object_present is not True:
self.operation.unresolved(
status=RecoveryStatus.OUTCOME_UNKNOWN,
summary="The unreferenced Files object could not be probed",
evidence=evidence,
failure_summary="Object storage availability prevented upload compensation",
)
return
try:
self.backend.delete(self.storage_key)
except (StorageBackendError, OSError):
self.operation.unresolved(
status=RecoveryStatus.RECOVERY_REQUIRED,
summary="An unreferenced Files object could not be compensated",
evidence=evidence,
failure_summary="Delete the unreferenced managed object after verifying that no FileBlob references it",
)
return
recovered = _blob_write_evidence(self)
if (
recovered.get("verified") is True
and recovered.get("database_blob_present") is False
and recovered.get("object_present") is False
):
self.operation.compensate(
failure_summary="The Files database transaction did not retain the new blob",
failure_evidence=evidence,
recovery_evidence=recovered,
)
return
self.operation.unresolved(
status=RecoveryStatus.RECOVERY_REQUIRED,
summary="Files upload compensation could not be verified",
evidence=recovered,
failure_summary="The unreferenced managed object requires operator reconciliation",
)
@dataclass(slots=True)
class PendingOrphanCleanup:
operation: DurableRecoveryOperation
backend: StorageBackend
finding_id: str
tenant_id: str
storage_key: str
resolved_by_user_id: str
def settle(self, *, committed: bool) -> None:
del committed
evidence = _orphan_cleanup_evidence(self)
if _orphan_cleanup_complete(evidence):
self.operation.succeed(evidence=evidence)
return
if (
evidence.get("verified") is True
and evidence.get("object_present") is True
and evidence.get("finding_deleted") is False
):
self.operation.reject(
summary="The orphan object was retained and the cleanup finding stayed open",
evidence=evidence,
)
return
if evidence.get("object_present") is False:
if _forward_complete_orphan_finding(self):
completed = _orphan_cleanup_evidence(self)
if _orphan_cleanup_complete(completed):
self.operation.succeed(evidence=completed)
return
status = (
RecoveryStatus.OUTCOME_UNKNOWN
if not evidence.get("verified")
else RecoveryStatus.RECOVERY_REQUIRED
)
self.operation.unresolved(
status=status,
summary="Files orphan cleanup requires reconciliation",
evidence=evidence,
failure_summary="Recheck the object and integrity-finding state before another cleanup attempt",
)
def begin_blob_write_recovery(
session: Session,
*,
backend: StorageBackend,
tenant_id: str,
blob_id: str,
storage_key: str,
semantic_checksum_sha256: str,
semantic_size_bytes: int,
protection_discriminator: str,
created_new: bool,
repair_token: str | None = None,
) -> PendingBlobWrite:
disposition = "create" if created_new else "repair"
state_token = repair_token or blob_id
key_digest = hashlib.sha256(storage_key.encode("utf-8")).hexdigest()
idempotency_key = (
f"files-blob-{disposition}:{blob_id}:{state_token[:48]}"
)
try:
started = begin_durable_recovery_operation(
get_database().SessionLocal,
identity=process_runtime_identity(),
module_id="files",
operation_type=f"blob-{disposition}",
idempotency_key=idempotency_key,
request={
"tenant_id": tenant_id,
"blob_id": blob_id,
"storage_key": storage_key if created_new else None,
"storage_key_sha256": key_digest,
"semantic_checksum_sha256": semantic_checksum_sha256,
"semantic_size_bytes": semantic_size_bytes,
"protection_discriminator": protection_discriminator,
"disposition": disposition,
},
recovery_plan=RecoveryPlan(
mode=(
RecoveryMode.COMPENSATION
if created_new
else RecoveryMode.FORWARD_RECOVERY
),
preconditions=(
"the caller has Files write authority for the target owner",
"the storage key belongs to the tenant Files namespace",
"the request records digests rather than file contents",
),
compensation_steps=(
"verify that no FileBlob references the newly reserved key",
"delete only that unreferenced key and verify absence",
)
if created_new
else (),
forward_recovery_steps=(
"verify the expected bytes at the existing blob key",
"forward-complete matching integrity metadata or quarantine the blob",
)
if not created_new
else (),
verification_steps=(
"reload FileBlob metadata through an independent session",
"stream and hash the managed object independently",
),
),
precondition_evidence={
"blob_id": blob_id,
"storage_key_sha256": key_digest,
"semantic_checksum_sha256": semantic_checksum_sha256,
"semantic_size_bytes": semantic_size_bytes,
"created_new": created_new,
},
lease_resource_key=(
f"files:blob:{tenant_id}:{blob_id}"
),
lease_ttl_seconds=15 * 60,
resource_type="file_blob",
resource_id=blob_id,
metadata={
"resources": ["postgresql", "object-storage"],
"storage_backend": backend.name,
},
)
except (RecoveryOperationBusy, RecoveryOperationStateConflict) as exc:
raise FileStorageError(
"This managed blob is already owned by another recovery operation"
) from exc
except (RecoveryGuaranteeError, RuntimeError) as exc:
raise FileStorageError(
"The Files recovery ledger is unavailable; no object was written"
) from exc
if started.replayed or started.operation is None:
raise FileStorageError(
"The matching Files blob operation was already completed; reload before retrying"
)
pending = PendingBlobWrite(
operation=started.operation,
backend=backend,
tenant_id=tenant_id,
blob_id=blob_id,
storage_key=storage_key,
semantic_checksum_sha256=semantic_checksum_sha256,
semantic_size_bytes=semantic_size_bytes,
protection_discriminator=protection_discriminator,
created_new=created_new,
)
_register_pending_effect(session, pending)
return pending
def begin_orphan_cleanup_recovery(
session: Session,
finding: FileIntegrityFinding,
*,
backend: StorageBackend,
user_id: str,
) -> PendingOrphanCleanup:
key_digest = hashlib.sha256(finding.storage_key.encode("utf-8")).hexdigest()
try:
started = begin_durable_recovery_operation(
get_database().SessionLocal,
identity=process_runtime_identity(),
module_id="files",
operation_type="integrity-orphan-cleanup",
idempotency_key=f"files-orphan-cleanup:{finding.id}",
request={
"tenant_id": finding.tenant_id,
"finding_id": finding.id,
"storage_key_sha256": key_digest,
},
recovery_plan=RecoveryPlan(
mode=RecoveryMode.FORWARD_RECOVERY,
preconditions=(
"the integrity finding identifies an unreferenced object",
"the object key remains inside the completed scan scope",
"a fresh database reference check found no FileBlob",
),
forward_recovery_steps=(
"verify object absence independently",
"mark the durable finding deleted only after absence is proven",
),
verification_steps=(
"reload the finding and its scan through an independent session",
"probe the original storage key through the configured backend",
),
),
precondition_evidence={
"finding_id": finding.id,
"scan_id": finding.scan_id,
"storage_key_sha256": key_digest,
"finding_state": finding.state,
},
lease_resource_key=(
f"files:orphan-cleanup:{finding.tenant_id}:{key_digest[:40]}"
),
lease_ttl_seconds=15 * 60,
resource_type="file_integrity_finding",
resource_id=finding.id,
metadata={
"resources": ["postgresql", "object-storage"],
"storage_backend": backend.name,
},
)
except (RecoveryOperationBusy, RecoveryOperationStateConflict) as exc:
raise FileStorageError(
"This orphan object is already owned by another recovery operation"
) from exc
except (RecoveryGuaranteeError, RuntimeError) as exc:
raise FileStorageError(
"The Files recovery ledger is unavailable; no object was deleted"
) from exc
if started.replayed or started.operation is None:
raise FileStorageError(
"This orphan cleanup was already completed; reload the finding"
)
pending = PendingOrphanCleanup(
operation=started.operation,
backend=backend,
finding_id=finding.id,
tenant_id=finding.tenant_id,
storage_key=finding.storage_key,
resolved_by_user_id=user_id,
)
_register_pending_effect(session, pending)
return pending
def _register_pending_effect(session: Session, effect: _PendingEffect) -> None:
if not session.in_transaction():
session.begin()
pending = session.info.setdefault(_PENDING_EFFECTS_KEY, [])
pending.append(effect)
if session.info.get(_HOOKS_INSTALLED_KEY):
return
event.listen(session, "after_commit", _after_session_commit)
event.listen(session, "after_rollback", _after_session_rollback)
session.info[_HOOKS_INSTALLED_KEY] = True
def _after_session_commit(session: Session) -> None:
_settle_pending_effects(session, committed=True)
def _after_session_rollback(session: Session) -> None:
try:
_settle_pending_effects(session, committed=False)
except RecoveryGuaranteeError as exc:
session.info.setdefault(_ROLLBACK_ERRORS_KEY, []).append(str(exc))
def _settle_pending_effects(session: Session, *, committed: bool) -> None:
pending = list(session.info.pop(_PENDING_EFFECTS_KEY, []))
failures: list[Exception] = []
for effect in pending:
try:
effect.settle(committed=committed)
except Exception as exc: # preserve every effect's chance to settle
failures.append(exc)
operation = getattr(effect, "operation", None)
if operation is not None and not operation.closed:
try:
operation.release_unresolved()
except Exception:
pass
if failures:
raise RecoveryGuaranteeError(
f"{len(failures)} Files recovery operation(s) could not be finalized"
) from failures[0]
def _blob_write_evidence(effect: PendingBlobWrite) -> dict[str, object]:
database_blob_present: bool | None
database_matches: bool | None
database_integrity_verified: bool | None
try:
with effect.operation.session_factory() as session:
blob = session.get(FileBlob, effect.blob_id)
database_blob_present = blob is not None
database_matches = bool(
blob is not None
and blob.tenant_id == effect.tenant_id
and blob.storage_key == effect.storage_key
and blob.checksum_sha256 == effect.semantic_checksum_sha256
and blob.size_bytes == effect.semantic_size_bytes
and blob.protection_discriminator
== effect.protection_discriminator
and blob.encryption_envelope_id == effect.expected_envelope_id
and (
effect.expected_storage_checksum_sha256
== (
blob.storage_checksum_sha256
or blob.checksum_sha256
)
)
) if blob is not None else False
database_integrity_verified = bool(
blob is not None
and blob.integrity_status == "verified"
and blob.quarantined_at is None
) if blob is not None else False
except Exception:
database_blob_present = None
database_matches = None
database_integrity_verified = None
object_present: bool | None
observed_size: int | None = None
observed_checksum: str | None = None
try:
digest = hashlib.sha256()
observed_size = 0
for chunk in effect.backend.iter_bytes(effect.storage_key):
observed_size += len(chunk)
digest.update(chunk)
observed_checksum = digest.hexdigest()
object_present = True
except StorageObjectMissing:
object_present = False
except (StorageBackendError, OSError):
object_present = None
object_size_matches = (
observed_size == effect.expected_storage_size_bytes
if object_present is True and effect.expected_storage_size_bytes is not None
else False if object_present is False else None
)
object_checksum_matches = (
observed_checksum == effect.expected_storage_checksum_sha256
if object_present is True
and effect.expected_storage_checksum_sha256 is not None
else False if object_present is False else None
)
verified = database_blob_present is not None and object_present is not None
return {
"verified": verified,
"checks": {
"database_reloaded": database_blob_present is not None,
"object_probed": object_present is not None,
"database_matches_request": database_matches,
"database_integrity_verified": database_integrity_verified,
"object_size_matches": object_size_matches,
"object_checksum_matches": object_checksum_matches,
},
"blob_id": effect.blob_id,
"database_blob_present": database_blob_present,
"database_matches_request": database_matches,
"database_integrity_verified": database_integrity_verified,
"object_present": object_present,
"observed_size_bytes": observed_size,
"observed_checksum_sha256": observed_checksum,
"expected_storage_size_bytes": effect.expected_storage_size_bytes,
"expected_storage_checksum_sha256": effect.expected_storage_checksum_sha256,
}
def _blob_write_complete(evidence: dict[str, object]) -> bool:
return bool(
evidence.get("verified") is True
and evidence.get("database_matches_request") is True
and evidence.get("database_integrity_verified") is True
and _object_matches(evidence)
)
def _object_matches(evidence: dict[str, object]) -> bool:
checks = evidence.get("checks")
return bool(
isinstance(checks, dict)
and evidence.get("object_present") is True
and checks.get("object_size_matches") is True
and checks.get("object_checksum_matches") is True
)
def _forward_complete_blob_repair(effect: PendingBlobWrite) -> bool:
try:
with effect.operation.session_factory() as session:
blob = session.get(FileBlob, effect.blob_id)
if (
blob is None
or blob.tenant_id != effect.tenant_id
or blob.storage_key != effect.storage_key
or blob.checksum_sha256 != effect.semantic_checksum_sha256
or blob.size_bytes != effect.semantic_size_bytes
or blob.protection_discriminator
!= effect.protection_discriminator
or blob.encryption_envelope_id != effect.expected_envelope_id
):
return False
blob.storage_checksum_sha256 = (
effect.expected_storage_checksum_sha256
if effect.expected_envelope_id
else None
)
blob.storage_size_bytes = (
effect.expected_storage_size_bytes
if effect.expected_envelope_id
else None
)
blob.integrity_status = "verified"
blob.integrity_checked_at = datetime.now(UTC)
blob.integrity_failure = None
blob.quarantined_at = None
session.add(blob)
session.commit()
return True
except Exception:
return False
def _quarantine_blob_after_failed_verification(
effect: PendingBlobWrite,
evidence: dict[str, object],
) -> None:
try:
with effect.operation.session_factory() as session:
blob = session.get(FileBlob, effect.blob_id)
if blob is None or blob.storage_key != effect.storage_key:
return
blob.integrity_status = (
"missing"
if evidence.get("object_present") is False
else "checksum_mismatch"
)
blob.integrity_checked_at = datetime.now(UTC)
blob.integrity_failure = "recovery_verification_failed"
blob.quarantined_at = datetime.now(UTC)
session.add(blob)
session.commit()
except Exception:
return
def _orphan_cleanup_evidence(effect: PendingOrphanCleanup) -> dict[str, object]:
try:
object_present: bool | None = effect.backend.exists(effect.storage_key)
except (StorageBackendError, OSError):
object_present = None
finding_present: bool | None
finding_deleted: bool | None
still_unreferenced: bool | None
try:
with effect.operation.session_factory() as session:
finding = session.get(FileIntegrityFinding, effect.finding_id)
finding_present = finding is not None
finding_deleted = bool(
finding is not None and finding.state == "deleted"
) if finding is not None else False
referenced = (
session.query(FileBlob.id)
.filter(
FileBlob.tenant_id == effect.tenant_id,
FileBlob.storage_key == effect.storage_key,
)
.first()
)
still_unreferenced = referenced is None
except Exception:
finding_present = None
finding_deleted = None
still_unreferenced = None
verified = object_present is not None and finding_present is not None
return {
"verified": verified,
"checks": {
"database_reloaded": finding_present is not None,
"object_probed": object_present is not None,
"finding_marked_deleted": finding_deleted,
"object_absent": (
not object_present if object_present is not None else None
),
"still_unreferenced": still_unreferenced,
},
"finding_id": effect.finding_id,
"finding_present": finding_present,
"finding_deleted": finding_deleted,
"object_present": object_present,
"still_unreferenced": still_unreferenced,
}
def _orphan_cleanup_complete(evidence: dict[str, object]) -> bool:
return bool(
evidence.get("verified") is True
and evidence.get("finding_deleted") is True
and evidence.get("object_present") is False
and evidence.get("still_unreferenced") is True
)
def _forward_complete_orphan_finding(effect: PendingOrphanCleanup) -> bool:
try:
with effect.operation.session_factory() as session:
finding = session.get(FileIntegrityFinding, effect.finding_id)
if (
finding is None
or finding.tenant_id != effect.tenant_id
or finding.storage_key != effect.storage_key
):
return False
scan = session.get(FileIntegrityScan, finding.scan_id)
if scan is None or not finding.storage_key.startswith(
scan.storage_prefix
):
return False
referenced = (
session.query(FileBlob.id)
.filter(
FileBlob.tenant_id == effect.tenant_id,
FileBlob.storage_key == effect.storage_key,
)
.first()
)
if referenced:
return False
finding.state = "deleted"
finding.resolved_at = datetime.now(UTC)
finding.resolved_by_user_id = effect.resolved_by_user_id
session.add(finding)
session.commit()
return True
except Exception:
return False
__all__ = [
"PendingBlobWrite",
"PendingOrphanCleanup",
"begin_blob_write_recovery",
"begin_orphan_cleanup_recovery",
]
@@ -0,0 +1,331 @@
from __future__ import annotations
import inspect
import socket
import threading
from functools import lru_cache
from importlib import import_module
from typing import Any
from govoplan_core.security.outbound_http import (
OutboundHttpError,
create_outbound_connection,
)
class SdkPeerPinningError(RuntimeError):
"""Raised when an optional SDK cannot be bound to the pinned transport."""
_BOTOCORE_CLIENT_CREATION_LOCK = threading.Lock()
def create_pinned_s3_client(**kwargs: Any) -> Any:
"""Construct an S3 client whose first and every later socket is pinned.
Botocore does not expose the HTTP-session class through boto3's public
client API. Its endpoint creator does expose that seam, so this function
replaces the creator only for the bounded client-construction operation.
The lock avoids an unsafe interleaving with another GovOPlaN client
construction. Any unrelated botocore client created during the short
replacement window also receives the stricter transport.
"""
try:
boto3 = import_module("boto3")
botocore_args = import_module("botocore.args")
except ImportError as exc:
raise SdkPeerPinningError(
"S3 connector browsing requires the optional boto3 dependency"
) from exc
pinned_session_cls = _botocore_transport_types()[0]
original_creator = getattr(botocore_args, "EndpointCreator", None)
if original_creator is None or not hasattr(original_creator, "create_endpoint"):
raise SdkPeerPinningError(
"The installed botocore release does not expose the endpoint transport seam required for peer pinning"
)
class PinnedEndpointCreator(original_creator): # type: ignore[misc, valid-type]
def create_endpoint(self, *args: Any, **endpoint_kwargs: Any) -> Any:
endpoint_kwargs["http_session_cls"] = pinned_session_cls
return super().create_endpoint(*args, **endpoint_kwargs)
with _BOTOCORE_CLIENT_CREATION_LOCK:
current_creator = getattr(botocore_args, "EndpointCreator", None)
if current_creator is not original_creator:
raise SdkPeerPinningError(
"Botocore endpoint construction changed concurrently; refusing to create an unproven S3 client"
)
botocore_args.EndpointCreator = PinnedEndpointCreator
try:
session_type = getattr(getattr(boto3, "session", None), "Session", None)
if session_type is None:
raise SdkPeerPinningError(
"The installed boto3 release does not expose its isolated session constructor"
)
client = session_type().client("s3", **kwargs)
finally:
if getattr(botocore_args, "EndpointCreator", None) is PinnedEndpointCreator:
botocore_args.EndpointCreator = original_creator
http_session = getattr(getattr(client, "_endpoint", None), "http_session", None)
if http_session is None or not isinstance(http_session, pinned_session_cls):
close = getattr(client, "close", None)
if callable(close):
close()
raise SdkPeerPinningError(
"Botocore did not install the required pinned HTTP transport; the S3 client was discarded"
)
return client
@lru_cache(maxsize=1)
def _botocore_transport_types() -> tuple[type[Any], type[Any], type[Any]]:
try:
awsrequest = import_module("botocore.awsrequest")
httpsession = import_module("botocore.httpsession")
urllib3_exceptions = import_module("urllib3.exceptions")
except ImportError as exc:
raise SdkPeerPinningError(
"S3 connector browsing requires a compatible botocore HTTP transport"
) from exc
required = (
"AWSHTTPConnection",
"AWSHTTPSConnection",
"AWSHTTPConnectionPool",
"AWSHTTPSConnectionPool",
)
if any(not hasattr(awsrequest, name) for name in required) or not hasattr(
httpsession, "URLLib3Session"
):
raise SdkPeerPinningError(
"The installed botocore release is missing the connection classes required for peer pinning"
)
def pinned_new_connection(connection: Any) -> socket.socket:
hostname = str(getattr(connection, "_dns_host", "") or "").strip()
port = int(getattr(connection, "port", 0) or 0)
if not hostname or not port:
raise urllib3_exceptions.NewConnectionError(
connection, "Pinned S3 connection is missing its target authority"
)
try:
return create_outbound_connection(
hostname,
port,
timeout=getattr(connection, "timeout", None),
source_address=getattr(connection, "source_address", None),
socket_options=getattr(connection, "socket_options", None),
label="S3 connector peer",
)
except socket.timeout as exc:
raise urllib3_exceptions.ConnectTimeoutError(
connection,
f"Connection to {hostname} timed out while selecting an approved peer",
) from exc
except (OSError, OutboundHttpError, ValueError) as exc:
raise urllib3_exceptions.NewConnectionError(
connection,
f"S3 connector peer was rejected: {exc}",
) from exc
class PinnedAWSHTTPConnection(awsrequest.AWSHTTPConnection):
_new_conn = pinned_new_connection
class PinnedAWSHTTPSConnection(awsrequest.AWSHTTPSConnection):
_new_conn = pinned_new_connection
class PinnedAWSHTTPConnectionPool(awsrequest.AWSHTTPConnectionPool):
ConnectionCls = PinnedAWSHTTPConnection
class PinnedAWSHTTPSConnectionPool(awsrequest.AWSHTTPSConnectionPool):
ConnectionCls = PinnedAWSHTTPSConnection
class PinnedURLLib3Session(httpsession.URLLib3Session):
def __init__(self, *args: Any, **kwargs: Any) -> None:
# A proxy would select the final target outside this process. Until
# proxy peer delegation is modeled, connector traffic is direct.
kwargs["proxies"] = {}
super().__init__(*args, **kwargs)
self._pool_classes_by_scheme = {
"http": PinnedAWSHTTPConnectionPool,
"https": PinnedAWSHTTPSConnectionPool,
}
manager = getattr(self, "_manager", None)
if manager is None or not hasattr(manager, "pool_classes_by_scheme"):
raise SdkPeerPinningError(
"The installed botocore pool manager cannot enforce pinned connection classes"
)
manager.pool_classes_by_scheme = self._pool_classes_by_scheme
return PinnedURLLib3Session, PinnedAWSHTTPConnection, PinnedAWSHTTPSConnection
@lru_cache(maxsize=1)
def pinned_smb_connection_cache() -> dict[str, Any]:
"""Return the Files-owned cache; unproven process-global sessions are never reused."""
return {}
def install_pinned_smb_transport(smbclient: Any) -> Any:
"""Install a process-wide fail-closed smbclient session factory.
smbclient routes initial connections, reconnects, and DFS targets through
``smbclient._pool.register_session``. Replacing that single factory with a
behavior-compatible implementation ensures every target uses a socket
selected by Core's deployment-wide outbound policy.
"""
try:
pool = import_module("smbclient._pool")
connection_module = import_module("smbprotocol.connection")
session_module = import_module("smbprotocol.session")
transport_module = import_module("smbprotocol.transport")
except ImportError as exc:
raise SdkPeerPinningError(
"SMB connector browsing requires the optional smbprotocol dependency"
) from exc
current = getattr(pool, "register_session", None)
if getattr(current, "__govoplan_peer_pinned__", False):
expected_tcp = getattr(current, "__govoplan_pinned_tcp__", None)
if expected_tcp is None or getattr(connection_module, "Tcp", None) is not expected_tcp:
raise SdkPeerPinningError(
"The installed SMB transport changed after peer pinning; refusing to reuse the session factory"
)
return smbclient
required_parameters = {
"server",
"username",
"password",
"port",
"encrypt",
"connection_timeout",
"connection_cache",
"auth_protocol",
"require_signing",
}
if current is None or not required_parameters.issubset(
inspect.signature(current).parameters
):
raise SdkPeerPinningError(
"The installed smbprotocol release does not expose the session seam required for peer pinning"
)
class PinnedTcp(transport_module.Tcp):
def connect(self) -> None:
with self._sock_lock:
if self.connected:
return
try:
self._sock = create_outbound_connection(
self.server,
int(self.port),
timeout=self.timeout,
label="SMB connector peer",
)
except (OSError, OutboundHttpError, ValueError) as exc:
raise ValueError(
f"SMB connector peer '{self.server}:{self.port}' was rejected: {exc}"
) from exc
self._sock.settimeout(None)
self.connected = True
# Connection.connect() instantiates the module-level Tcp symbol on every
# reconnect. Replacing it process-wide is deliberate: an SMB connection
# created by another code path must become stricter, never bypass Files'
# peer boundary.
if not hasattr(connection_module, "Tcp"):
raise SdkPeerPinningError(
"The installed smbprotocol release cannot install the pinned TCP transport"
)
connection_module.Tcp = PinnedTcp
def pinned_register_session(
server: str,
username: str | None = None,
password: str | None = None,
port: int = 445,
encrypt: bool | None = None,
connection_timeout: float = 60,
connection_cache: dict[str, Any] | None = None,
auth_protocol: str = "negotiate",
require_signing: bool = True,
) -> Any:
cache = pinned_smb_connection_cache() if connection_cache is None else connection_cache
connection_key = f"{server.lower()}:{port}"
connection = cache.get(connection_key)
transport = getattr(connection, "transport", None)
if connection is not None and not isinstance(transport, PinnedTcp):
disconnect = getattr(connection, "disconnect", None)
if callable(disconnect):
try:
disconnect(close=True)
except Exception:
pass
cache.pop(connection_key, None)
connection = None
if connection is None or not getattr(connection.transport, "connected", False):
connection = connection_module.Connection(
pool.ClientConfig().client_guid,
server,
port,
require_signing=require_signing,
)
connection.transport = PinnedTcp(server, port)
connection.connect(timeout=connection_timeout)
if not isinstance(connection.transport, PinnedTcp):
disconnect = getattr(connection, "disconnect", None)
if callable(disconnect):
try:
disconnect(close=True)
except Exception:
pass
raise SdkPeerPinningError(
"smbprotocol replaced the required pinned TCP transport during connection setup"
)
cache[connection_key] = connection
session = next(
(
item
for item in connection.session_table.values()
if username is None or item.username == username
),
None,
)
if session is None:
session = session_module.Session(
connection,
username=username,
password=password,
require_encryption=(encrypt is True),
auth_protocol=auth_protocol,
)
session.connect()
elif encrypt is not None:
if session.encrypt_data and not encrypt:
raise ValueError(
"Cannot disable encryption on an already negotiated session."
)
if not session.encrypt_data and encrypt:
session.encrypt = True
return session
pinned_register_session.__govoplan_peer_pinned__ = True # type: ignore[attr-defined]
pinned_register_session.__govoplan_pinned_tcp__ = PinnedTcp # type: ignore[attr-defined]
pool.register_session = pinned_register_session
if hasattr(smbclient, "register_session"):
smbclient.register_session = pinned_register_session
return smbclient
__all__ = [
"SdkPeerPinningError",
"create_pinned_s3_client",
"install_pinned_smb_transport",
"pinned_smb_connection_cache",
]
@@ -0,0 +1,34 @@
from __future__ import annotations
from datetime import datetime, timezone
from sqlalchemy import and_, or_
from govoplan_files.backend.db.models import FileShare
from govoplan_files.backend.storage.common import utcnow
def effective_file_share_clause(*, at: datetime | None = None):
checked_at = at or utcnow()
return and_(
FileShare.revoked_at.is_(None),
or_(FileShare.expires_at.is_(None), FileShare.expires_at > checked_at),
)
def file_share_is_active(share: FileShare, *, at: datetime | None = None) -> bool:
if share.revoked_at is not None:
return False
if share.expires_at is None:
return True
checked_at = _aware_utc(at or utcnow())
return _aware_utc(share.expires_at) > checked_at
def _aware_utc(value: datetime) -> datetime:
if value.tzinfo is None:
return value.replace(tzinfo=timezone.utc)
return value.astimezone(timezone.utc)
__all__ = ["effective_file_share_clause", "file_share_is_active"]
+118 -2
View File
@@ -1,15 +1,18 @@
from __future__ import annotations
import unittest
from datetime import datetime, timedelta, timezone
from sqlalchemy import create_engine
from sqlalchemy import create_engine, text
from sqlalchemy.orm import sessionmaker
from govoplan_access.backend.db.models import Account, Group, User
from govoplan_core.core.access import PrincipalRef
from govoplan_core.core.change_sequence import ChangeSequenceEntry
from govoplan_core.db.base import Base
from govoplan_files.backend.capabilities import FilesAccessService, virtual_folder_resource_id
from govoplan_files.backend.db.models import FileAsset, FileFolder, FileShare
from govoplan_files.backend.storage.files import count_assets_for_user, list_assets_for_user
TENANT_ID = "tenant-1"
@@ -75,10 +78,123 @@ class FilesAccessProviderTests(unittest.TestCase):
self.assertTrue(any(item.kind == "owner" and item.id == GROUP_ID for item in virtual_items))
self.assertEqual("files.not_found", missing_items[0].source)
def test_file_property_filters_cover_campaign_and_audit_usage(self) -> None:
session = _session()
self.addCleanup(_close_session, session)
_seed_access_subjects(session)
assets = [
FileAsset(
id=f"file-{index}",
tenant_id=TENANT_ID,
owner_type="user",
owner_user_id=USER_ID,
display_path=f"{index}.pdf",
filename=f"{index}.pdf",
)
for index in range(1, 4)
]
session.add_all([
*assets,
FileShare(
id="share-campaign",
tenant_id=TENANT_ID,
file_asset_id=assets[0].id,
target_type="campaign",
target_id="campaign-1",
permission="read",
),
FileShare(
id="share-campaign-expired",
tenant_id=TENANT_ID,
file_asset_id=assets[2].id,
target_type="campaign",
target_id="campaign-1",
permission="read",
expires_at=datetime.now(timezone.utc) - timedelta(seconds=1),
),
])
session.commit()
session.execute(
text(
"INSERT INTO campaign_attachment_uses "
"(id, tenant_id, file_asset_id, use_stage) "
"VALUES (:id, :tenant_id, :file_asset_id, :use_stage)"
),
{
"id": "attachment-use-1",
"tenant_id": TENANT_ID,
"file_asset_id": assets[1].id,
"use_stage": "sent",
},
)
session.commit()
linked = list_assets_for_user(
session,
tenant_id=TENANT_ID,
user_id=USER_ID,
owner_type="user",
owner_id=USER_ID,
campaign_usage="linked",
)
unlinked = list_assets_for_user(
session,
tenant_id=TENANT_ID,
user_id=USER_ID,
owner_type="user",
owner_id=USER_ID,
campaign_usage="unlinked",
)
audit_relevant = list_assets_for_user(
session,
tenant_id=TENANT_ID,
user_id=USER_ID,
owner_type="user",
owner_id=USER_ID,
audit_relevant=True,
)
self.assertEqual([asset.id for asset in linked], ["file-1", "file-2"])
self.assertEqual([asset.id for asset in unlinked], ["file-3"])
self.assertEqual([asset.id for asset in audit_relevant], ["file-2"])
self.assertEqual(
count_assets_for_user(
session,
tenant_id=TENANT_ID,
user_id=USER_ID,
owner_type="user",
owner_id=USER_ID,
campaign_usage="unlinked",
),
1,
)
def _session():
engine = create_engine("sqlite:///:memory:", future=True)
Base.metadata.create_all(bind=engine, tables=[Account.__table__, User.__table__, Group.__table__, FileAsset.__table__, FileFolder.__table__, FileShare.__table__])
Base.metadata.create_all(
bind=engine,
tables=[
Account.__table__,
User.__table__,
Group.__table__,
ChangeSequenceEntry.__table__,
FileAsset.__table__,
FileFolder.__table__,
FileShare.__table__,
],
)
with engine.begin() as connection:
connection.execute(
text(
"CREATE TABLE campaign_attachment_uses ("
"id VARCHAR(36) PRIMARY KEY, "
"tenant_id VARCHAR(36) NOT NULL, "
"file_asset_id VARCHAR(36) NOT NULL, "
"use_stage VARCHAR(20) NOT NULL"
")"
)
)
return sessionmaker(bind=engine, future=True)()
+101
View File
@@ -0,0 +1,101 @@
from __future__ import annotations
import io
import hashlib
import unittest
import zipfile
from types import SimpleNamespace
from unittest.mock import patch
from fastapi import UploadFile
from govoplan_files.backend.routes.uploads import (
_validate_archive_preview_token,
preview_archive_upload,
)
from govoplan_files.backend.storage.common import FileStorageError
def _archive_bytes() -> bytes:
target = io.BytesIO()
with zipfile.ZipFile(target, "w", compression=zipfile.ZIP_DEFLATED) as archive:
archive.writestr("folder/report.txt", b"report")
return target.getvalue()
class ArchivePreviewTokenTests(unittest.TestCase):
def setUp(self) -> None:
self.settings = SimpleNamespace(
file_upload_zip_max_bytes=10 * 1024 * 1024,
file_archive_max_entries=10_000,
file_archive_max_expanded_bytes=2 * 1024 * 1024 * 1024,
file_archive_max_expansion_ratio=100,
file_archive_preview_ttl_seconds=30 * 60,
)
self.principal = SimpleNamespace(
tenant_id="tenant-1",
user=SimpleNamespace(id="user-1"),
)
def test_preview_token_is_bound_to_actor_archive_and_destination(self) -> None:
archive_data = _archive_bytes()
upload = UploadFile(
file=io.BytesIO(archive_data),
filename="monthly.zip",
)
with patch(
"govoplan_files.backend.routes.uploads.settings",
self.settings,
):
preview = preview_archive_upload(
file=upload,
owner_type="user",
owner_id="user-1",
path="incoming",
campaign_id=None,
password=None,
principal=self.principal,
)
_validate_archive_preview_token(
preview.preview_token,
tenant_id="tenant-1",
user_id="user-1",
owner_type="user",
owner_id="user-1",
path="incoming",
campaign_id=None,
archive_format="zip",
archive_sha256=hashlib.sha256(archive_data).hexdigest(),
)
with self.assertRaisesRegex(
FileStorageError, "does not match this upload destination"
):
_validate_archive_preview_token(
preview.preview_token,
tenant_id="tenant-1",
user_id="user-1",
owner_type="user",
owner_id="user-1",
path="elsewhere",
campaign_id=None,
archive_format="zip",
archive_sha256=hashlib.sha256(archive_data).hexdigest(),
)
with self.assertRaisesRegex(
FileStorageError, "contents changed"
):
_validate_archive_preview_token(
preview.preview_token,
tenant_id="tenant-1",
user_id="user-1",
owner_type="user",
owner_id="user-1",
path="incoming",
campaign_id=None,
archive_format="zip",
archive_sha256="0" * 64,
)
if __name__ == "__main__":
unittest.main()
+170
View File
@@ -0,0 +1,170 @@
from __future__ import annotations
import io
import tarfile
import unittest
import zipfile
from types import SimpleNamespace
from unittest.mock import patch
import pyzipper
from govoplan_files.backend.storage.archives import (
ArchivePasswordError,
extract_archive_upload,
inspect_archive,
)
from govoplan_files.backend.storage.common import FileStorageError
def _zip_bytes(entries: dict[str, bytes]) -> bytes:
target = io.BytesIO()
with zipfile.ZipFile(target, "w", compression=zipfile.ZIP_DEFLATED) as archive:
for path, data in entries.items():
archive.writestr(path, data)
return target.getvalue()
def _tar_bytes(entries: dict[str, bytes], mode: str = "w:gz") -> bytes:
target = io.BytesIO()
with tarfile.open(fileobj=target, mode=mode) as archive:
for path, data in entries.items():
info = tarfile.TarInfo(path)
info.size = len(data)
archive.addfile(info, io.BytesIO(data))
return target.getvalue()
def _encrypted_zip_bytes(password: str) -> bytes:
target = io.BytesIO()
with pyzipper.AESZipFile(
target,
"w",
compression=zipfile.ZIP_DEFLATED,
encryption=pyzipper.WZ_AES,
) as archive:
archive.setpassword(password.encode("utf-8"))
archive.writestr("secure/report.txt", b"classified")
return target.getvalue()
class ArchiveInspectionTests(unittest.TestCase):
def test_zip_preview_derives_folders_and_sizes(self) -> None:
inspection = inspect_archive(
_zip_bytes(
{
"department/one.txt": b"one",
"department/nested/two.csv": b"two",
}
),
filename="monthly.zip",
)
self.assertEqual("zip", inspection.archive_format)
self.assertEqual(2, inspection.file_count)
self.assertEqual(2, inspection.directory_count)
self.assertEqual(6, inspection.expanded_size_bytes)
self.assertEqual(
[
"department",
"department/nested",
"department/nested/two.csv",
"department/one.txt",
],
[entry.path for entry in inspection.entries],
)
def test_tar_compression_variants_are_inspected(self) -> None:
for filename, mode in (
("monthly.tar", "w"),
("monthly.tar.gz", "w:gz"),
("monthly.tar.bz2", "w:bz2"),
("monthly.tar.xz", "w:xz"),
):
with self.subTest(filename=filename):
inspection = inspect_archive(
_tar_bytes({"report.txt": b"report"}, mode=mode),
filename=filename,
)
self.assertEqual(1, inspection.file_count)
self.assertEqual(filename.removeprefix("monthly."), inspection.archive_format)
def test_unsafe_member_path_is_rejected(self) -> None:
with self.assertRaisesRegex(FileStorageError, "Unsafe archive member"):
inspect_archive(
_zip_bytes({"../escape.txt": b"no"}),
filename="unsafe.zip",
)
def test_archive_bomb_ratio_is_rejected(self) -> None:
with self.assertRaisesRegex(FileStorageError, "expansion ratio"):
inspect_archive(
_zip_bytes({"zeros.bin": b"\0" * 50_000}),
filename="bomb.zip",
max_expansion_ratio=2,
)
def test_encrypted_zip_reports_and_verifies_password(self) -> None:
archive = _encrypted_zip_bytes("correct horse")
missing = inspect_archive(archive, filename="secure.zip")
self.assertTrue(missing.requires_password)
self.assertFalse(missing.password_verified)
verified = inspect_archive(
archive,
filename="secure.zip",
password="correct horse",
)
self.assertTrue(verified.requires_password)
self.assertTrue(verified.password_verified)
with self.assertRaisesRegex(ArchivePasswordError, "incorrect"):
inspect_archive(
archive,
filename="secure.zip",
password="wrong",
)
def test_folder_selection_extracts_only_descendants(self) -> None:
archive = _zip_bytes(
{
"selected/one.txt": b"one",
"selected/two.txt": b"two",
"other/three.txt": b"three",
}
)
stored = SimpleNamespace(
asset=SimpleNamespace(id="asset"),
version=SimpleNamespace(id="version"),
blob=SimpleNamespace(id="blob"),
)
with patch(
"govoplan_files.backend.storage.archives.create_file_asset",
return_value=stored,
) as create_asset:
result = extract_archive_upload(
object(), # type: ignore[arg-type]
tenant_id="tenant-1",
owner_type="user",
owner_id="user-1",
user_id="user-1",
archive_data=archive,
filename="selection.zip",
folder="imports",
campaign_id=None,
selected_paths=["selected"],
)
self.assertEqual(2, len(result))
self.assertEqual(
{"imports/selected/one.txt", "imports/selected/two.txt"},
{
call.kwargs["display_path"]
for call in create_asset.call_args_list
},
)
if __name__ == "__main__":
unittest.main()
+89
View File
@@ -0,0 +1,89 @@
from __future__ import annotations
from types import SimpleNamespace
import unittest
from unittest.mock import patch
from govoplan_core.auth import ApiPrincipal
from govoplan_core.core.access import PrincipalRef
from govoplan_core.core.files import ManagedArtifactStore, ManagedArtifactWriteRequest
from govoplan_files.backend.capabilities import FilesArtifactStore
def principal() -> ApiPrincipal:
return ApiPrincipal(
principal=PrincipalRef(
account_id="account-1",
membership_id="membership-1",
tenant_id="tenant-1",
scopes=frozenset({"files:file:upload"}),
),
account=object(),
user=SimpleNamespace(id="user-1"),
)
class _Session:
def query(self):
raise AssertionError("not used")
def flush(self):
raise AssertionError("not used")
def stored():
return SimpleNamespace(
asset=SimpleNamespace(
id="file-1",
display_path="Generated/Templates/result.html",
owner_type="user",
),
version=SimpleNamespace(
id="version-1",
filename_at_upload="result.html",
content_type="text/html",
size_bytes=6,
checksum_sha256="0" * 64,
),
)
class FilesArtifactStoreTests(unittest.TestCase):
def test_plain_write_uses_files_authority_and_returns_provider_neutral_ref(self) -> None:
service = FilesArtifactStore()
self.assertIsInstance(service, ManagedArtifactStore)
request = ManagedArtifactWriteRequest(
filename="result.html",
payload=b"result",
content_type="text/html",
folder="Generated/Templates",
metadata={"producer_module": "templates"},
)
with patch("govoplan_files.backend.capabilities.create_file_asset", return_value=stored()) as create:
result = service.store_artifact(_Session(), principal(), request=request)
self.assertEqual("file-1", result.file_asset_id)
self.assertEqual("version-1", result.file_version_id)
self.assertEqual("tenant-1", create.call_args.kwargs["tenant_id"])
self.assertEqual("user-1", create.call_args.kwargs["owner_id"])
self.assertEqual(b"result", create.call_args.kwargs["data"])
def test_idempotent_write_uses_source_provenance_without_payload_metadata(self) -> None:
request = ManagedArtifactWriteRequest(
filename="result.html",
payload=b"secret body",
content_type="text/html",
idempotency_key="render-1",
metadata={"producer_module": "templates", "output_sha256": "a" * 64},
)
with patch(
"govoplan_files.backend.capabilities.sync_file_asset_from_source",
return_value=(stored(), "unchanged", None),
) as sync:
FilesArtifactStore().store_artifact(_Session(), principal(), request=request)
metadata = sync.call_args.kwargs["metadata"]
self.assertEqual("render-1", metadata["source_provenance"]["external_id"])
self.assertNotIn("secret body", str(metadata))
if __name__ == "__main__":
unittest.main()
+2 -1
View File
@@ -12,7 +12,8 @@ from govoplan_core.core.change_sequence import ChangeSequenceEntry
from govoplan_core.db.base import Base
from govoplan_core.security.secrets import encrypt_secret
from govoplan_files.backend.db.models import FileConnectorCredential, FileConnectorProfile
from govoplan_files.backend.router import deactivate_connector_credential, deactivate_connector_profile
from govoplan_files.backend.routes.connector_profiles import deactivate_connector_profile
from govoplan_files.backend.routes.connector_settings import deactivate_connector_credential
class Principal:
+5 -5
View File
@@ -10,7 +10,7 @@ from unittest.mock import MagicMock
from fastapi import HTTPException
from govoplan_files.backend.router import discover_connector_endpoint
from govoplan_files.backend.routes.connector_settings import discover_connector_endpoint
from govoplan_files.backend.schemas import FileConnectorDiscoveryRequest
from govoplan_files.backend.storage.connector_browse import ConnectorBrowseError, _profile_password, _s3_verify
from govoplan_files.backend.storage.connector_deployment import (
@@ -182,7 +182,7 @@ class ConnectorDiscoveryBoundaryTests(unittest.TestCase):
credential_mode="basic",
credentials={"username": "admin", "password_env": "MASTER_KEY_B64"},
)
with patch("govoplan_files.backend.router.browse_connector_profile") as browse, self.assertRaises(
with patch("govoplan_files.backend.routes.connector_settings.browse_connector_profile") as browse, self.assertRaises(
HTTPException
) as raised:
discover_connector_endpoint(payload, session=object(), principal=self._principal()) # type: ignore[arg-type]
@@ -212,10 +212,10 @@ class ConnectorDiscoveryBoundaryTests(unittest.TestCase):
events.append("io")
return []
with patch("govoplan_files.backend.router._ensure_connector_configuration_allowed", side_effect=ensure), patch(
"govoplan_files.backend.router._audit_connector_discovery_attempt",
with patch("govoplan_files.backend.routes.connector_settings._ensure_connector_configuration_allowed", side_effect=ensure), patch(
"govoplan_files.backend.routes.connector_settings._audit_connector_discovery_attempt",
side_effect=audit,
), patch("govoplan_files.backend.router.browse_connector_profile", side_effect=browse):
), patch("govoplan_files.backend.routes.connector_settings.browse_connector_profile", side_effect=browse):
response = discover_connector_endpoint(payload, session=object(), principal=self._principal()) # type: ignore[arg-type]
self.assertEqual("usable", response.status)
+66
View File
@@ -0,0 +1,66 @@
from __future__ import annotations
import unittest
from govoplan_files.backend.storage.connector_policy import (
ConnectorAccessRequest,
ConnectorPolicySource,
connector_policy_decision,
)
class ConnectorPolicyPatternTests(unittest.TestCase):
def test_resource_reference_allow_patterns_match(self) -> None:
decision = connector_policy_decision(
ConnectorAccessRequest(
connector_id="dev-smb",
credential_id="credential-primary",
provider="smb",
),
(
ConnectorPolicySource(
scope_type="tenant",
scope_id="tenant-1",
label="Tenant",
policy={
"allow": {
"connectors": ["dev-*"],
"credentials": ["credential-*"],
"providers": ["s?b"],
}
},
),
),
)
self.assertTrue(decision.allowed)
def test_resource_reference_deny_patterns_take_precedence(self) -> None:
decision = connector_policy_decision(
ConnectorAccessRequest(
connector_id="dev-smb",
credential_id="credential-legacy",
provider="smb",
),
(
ConnectorPolicySource(
scope_type="tenant",
scope_id="tenant-1",
label="Tenant",
policy={
"allow": {"connectors": ["dev-*"]},
"deny": {"credentials": ["*-legacy"]},
},
),
),
)
self.assertFalse(decision.allowed)
self.assertEqual(
("connector_policy_denylist",),
decision.requirements,
)
if __name__ == "__main__":
unittest.main()
+53 -1
View File
@@ -1,5 +1,6 @@
from __future__ import annotations
import json
import unittest
from unittest.mock import patch
@@ -8,6 +9,7 @@ from sqlalchemy.orm import sessionmaker
from govoplan_access.backend.db import models as access_models # noqa: F401 - resolve Files user foreign keys
from govoplan_core.db.base import Base
from govoplan_core.security.credential_envelopes import CredentialEnvelope
from govoplan_core.security.secrets import decrypt_secret, encrypt_secret
from govoplan_files.backend.db.models import FileConnectorProfile
from govoplan_files.backend.storage.common import FileStorageError
@@ -20,7 +22,13 @@ from govoplan_files.backend.storage.connector_profile_store import (
class ConnectorProfileStoreUpdateTests(unittest.TestCase):
def setUp(self) -> None:
self.engine = create_engine("sqlite:///:memory:")
Base.metadata.create_all(self.engine, tables=[FileConnectorProfile.__table__])
Base.metadata.create_all(
self.engine,
tables=[
FileConnectorProfile.__table__,
CredentialEnvelope.__table__,
],
)
self.session = sessionmaker(bind=self.engine)()
self.row = FileConnectorProfile(
id="profile-1",
@@ -171,6 +179,50 @@ class ConnectorProfileStoreUpdateTests(unittest.TestCase):
self.assertEqual([self.row.id], [profile.id for profile in profiles])
self.assertEqual("original-password", profiles[0].password_value)
def test_reusable_credential_resolves_and_revocation_keeps_profile_visible(self) -> None:
credential = CredentialEnvelope(
id="shared-credential",
tenant_id="tenant-1",
scope_type="tenant",
scope_id="tenant-1",
name="Shared WebDAV login",
credential_kind="username_password",
public_data={"username": "ada"},
secret_data_encrypted=encrypt_secret(json.dumps({"password": "secret"})),
secret_keys=["password"],
allowed_modules=["files"],
allowed_server_refs=[],
inherit_to_lower_scopes=True,
is_active=True,
revision="revision-1",
)
self.row.credential_profile_id = "credential-envelope:shared-credential"
self.row.username = None
self.row.password_encrypted = None
self.row.token_encrypted = None
self.session.add(credential)
self.session.commit()
profile = list_database_connector_profiles(
self.session,
tenant_id="tenant-1",
)[0]
self.assertEqual("ada", profile.username)
self.assertEqual("secret", profile.password_value)
self.assertTrue(profile.credentials_configured)
credential.is_active = False
self.session.commit()
profile = list_database_connector_profiles(
self.session,
tenant_id="tenant-1",
)[0]
self.assertIsNone(profile.password_value)
self.assertFalse(profile.credentials_configured)
self.assertTrue(profile.metadata["credential_unavailable"])
if __name__ == "__main__":
unittest.main()
+47 -37
View File
@@ -5,7 +5,7 @@ from datetime import UTC, datetime
from unittest.mock import patch
from govoplan_files.backend.storage.connector_browse import ConnectorBrowseError, _smb_location, browse_connector_profile
from govoplan_files.backend.storage.connector_imports import ConnectorImportError, read_connector_file
from govoplan_files.backend.storage.connector_imports import read_connector_file
from govoplan_files.backend.storage.connector_profiles import ConnectorProfile, connector_profiles_from_payload
from govoplan_files.backend.storage.connector_providers import connector_provider_descriptors
@@ -23,6 +23,10 @@ class FakeS3Client:
self.list_objects_request: dict[str, object] | None = None
self.head_request: dict[str, object] | None = None
self.get_request: dict[str, object] | None = None
self.closed = False
def close(self) -> None:
self.closed = True
def list_objects_v2(self, **kwargs: object) -> dict[str, object]:
self.list_objects_request = dict(kwargs)
@@ -75,42 +79,39 @@ def s3_profile(**overrides: object) -> ConnectorProfile:
class ConnectorProviderTests(unittest.TestCase):
def test_smb_sdk_transport_fails_closed_before_client_creation_in_all_modes(self) -> None:
def test_smb_browse_uses_the_files_owned_pinned_connection_cache(self) -> None:
profile = ConnectorProfile(
id="smb",
label="SMB",
provider="smb",
endpoint_url="smb://files.example.test/share",
)
for allow_private, address in ((False, "93.184.216.34"), (True, "10.0.0.5")):
with self.subTest(allow_private=allow_private), patch.dict(
with patch.dict(
"os.environ",
{
"APP_ENV": "production",
"GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS": str(allow_private).lower(),
},
{"APP_ENV": "production", "GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS": "false"},
), patch(
"govoplan_core.security.outbound_http.socket.getaddrinfo",
return_value=[(2, 1, 6, "", (address, 445))],
), patch("govoplan_files.backend.storage.connector_browse._smbclient_module") as sdk, self.assertRaisesRegex(
ConnectorBrowseError,
"redirects/referrals.*DNS/IP pinning",
):
browse_connector_profile(profile, path="")
sdk.assert_not_called()
return_value=[(2, 1, 6, "", ("93.184.216.34", 445))],
), patch("govoplan_files.backend.storage.connector_browse._smbclient_module") as sdk:
sdk.return_value.scandir.return_value.__enter__.return_value = iter(())
self.assertEqual([], browse_connector_profile(profile, path=""))
def test_smb_explicit_ip_still_fails_closed_because_the_sdk_may_follow_referrals(self) -> None:
kwargs = sdk.return_value.scandir.call_args.kwargs
self.assertIsInstance(kwargs["connection_cache"], dict)
self.assertTrue(kwargs["require_signing"])
def test_smb_endpoint_preflight_applies_private_network_policy(self) -> None:
profile = ConnectorProfile(id="smb", label="SMB", provider="smb", endpoint_url="smb://10.0.0.5/share")
with patch.dict(
"os.environ",
{"APP_ENV": "production", "GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS": "true"},
{"APP_ENV": "production", "GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS": "false"},
), patch(
"govoplan_core.security.outbound_http.socket.getaddrinfo",
return_value=[(2, 1, 6, "", ("10.0.0.5", 445))],
), self.assertRaisesRegex(ConnectorBrowseError, "redirects/referrals.*DNS/IP pinning"):
), self.assertRaisesRegex(ConnectorBrowseError, "non-public network"):
_smb_location(profile)
def test_smb_import_surfaces_fail_closed_policy_as_an_import_error(self) -> None:
def test_smb_import_uses_the_same_pinned_connection_cache(self) -> None:
profile = ConnectorProfile(id="smb", label="SMB", provider="smb", endpoint_url="smb://10.0.0.5/share")
with patch.dict(
"os.environ",
@@ -118,12 +119,13 @@ class ConnectorProviderTests(unittest.TestCase):
), patch(
"govoplan_core.security.outbound_http.socket.getaddrinfo",
return_value=[(2, 1, 6, "", ("10.0.0.5", 445))],
), patch("govoplan_files.backend.storage.connector_imports._smbclient_module") as sdk, self.assertRaisesRegex(
ConnectorImportError,
"redirects/referrals.*DNS/IP pinning",
):
read_connector_file(profile, library_id="", path="notice.txt", max_bytes=1024)
sdk.assert_not_called()
), patch("govoplan_files.backend.storage.connector_imports._smbclient_module") as sdk:
sdk.return_value.stat.return_value.st_size = 4
sdk.return_value.open_file.return_value.__enter__.return_value.read.return_value = b"test"
downloaded = read_connector_file(profile, library_id="", path="notice.txt", max_bytes=1024)
self.assertEqual(b"test", downloaded.data)
self.assertIsInstance(sdk.return_value.stat.call_args.kwargs["connection_cache"], dict)
def test_provider_descriptors_include_s3_and_reserved_microsoft_providers(self) -> None:
descriptors = {descriptor.provider: descriptor for descriptor in connector_provider_descriptors()}
@@ -172,6 +174,7 @@ class ConnectorProviderTests(unittest.TestCase):
self.assertEqual("govoplan:root/report.xlsx", items[1].external_id)
self.assertEqual("root/report.xlsx", items[1].metadata["key"])
self.assertEqual("next-page", items[1].metadata["next_continuation_token"])
self.assertTrue(client.closed)
def test_s3_browse_lists_buckets_when_profile_has_no_bucket(self) -> None:
client = FakeS3Client()
@@ -181,27 +184,33 @@ class ConnectorProviderTests(unittest.TestCase):
self.assertEqual(["archive"], [item.path for item in items])
def test_s3_sdk_transport_fails_closed_before_client_creation_in_private_mode(self) -> None:
def test_s3_client_is_constructed_through_the_pinned_transport(self) -> None:
with patch.dict(
"os.environ",
{"APP_ENV": "production", "GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS": "true"},
), patch(
"govoplan_core.security.outbound_http.socket.getaddrinfo",
return_value=[(2, 1, 6, "", ("127.0.0.1", 9000))],
), patch("govoplan_files.backend.storage.connector_browse.import_module") as importer, self.assertRaisesRegex(
ConnectorBrowseError,
"until that transport supports.*DNS/IP pinning",
):
), patch(
"govoplan_files.backend.storage.connector_browse.create_pinned_s3_client",
return_value=FakeS3Client(),
) as factory:
browse_connector_profile(s3_profile(), path="")
importer.assert_not_called()
def test_s3_sdk_endpoint_discovery_fails_closed(self) -> None:
with patch("govoplan_files.backend.storage.connector_browse.import_module") as importer, self.assertRaisesRegex(
ConnectorBrowseError,
"endpoint discovery.*cannot guarantee.*DNS/IP pinning",
):
kwargs = factory.call_args.kwargs
self.assertEqual("http://127.0.0.1:9000", kwargs["endpoint_url"])
self.assertEqual("access-key", kwargs["aws_access_key_id"])
self.assertEqual("secret-key", kwargs["aws_secret_access_key"])
self.assertEqual({}, kwargs["config"].proxies)
def test_s3_endpoint_discovery_uses_the_same_pinned_transport(self) -> None:
with patch(
"govoplan_files.backend.storage.connector_browse.create_pinned_s3_client",
return_value=FakeS3Client(),
) as factory:
browse_connector_profile(s3_profile(endpoint_url=None), path="")
importer.assert_not_called()
self.assertNotIn("endpoint_url", factory.call_args.kwargs)
def test_s3_import_downloads_object_and_preserves_remote_identity(self) -> None:
client = FakeS3Client()
@@ -217,6 +226,7 @@ class ConnectorProviderTests(unittest.TestCase):
self.assertEqual("govoplan:root/report.txt", downloaded.external_id)
self.assertEqual("s3://govoplan/root/report.txt", downloaded.external_url)
self.assertEqual("checksum", downloaded.metadata["checksum_sha256"])
self.assertTrue(client.closed)
if __name__ == "__main__":
+12 -5
View File
@@ -10,6 +10,9 @@ from govoplan_files.backend.storage.connector_visibility import (
connector_profile_usable_for_import,
visible_connector_profiles_for_actor,
)
from govoplan_files.backend.storage.connector_providers import (
connector_provider_descriptors,
)
def _profile(
@@ -168,11 +171,15 @@ class ConnectorVisibilityTests(unittest.TestCase):
self,
) -> None:
self.assertTrue(connector_profile_usable_for_import(_profile("webdav")))
self.assertFalse(
connector_profile_usable_for_import(_profile("s3", provider="s3"))
)
self.assertFalse(
connector_profile_usable_for_import(_profile("smb", provider="smb"))
descriptors = {
item.provider: item for item in connector_provider_descriptors()
}
for provider in ("s3", "smb"):
self.assertEqual(
descriptors[provider].installed,
connector_profile_usable_for_import(
_profile(provider, provider=provider)
),
)
self.assertFalse(
connector_profile_usable_for_import(
+149
View File
@@ -0,0 +1,149 @@
from __future__ import annotations
from datetime import UTC, datetime
import hashlib
import unittest
from sqlalchemy import create_engine
from sqlalchemy.orm import Session
from govoplan_core.core.encryption import (
CAPABILITY_ENCRYPTION_CONTENT_CIPHER,
ContentProtectionEnvelope,
ProtectedContent,
)
from govoplan_files.backend.db.models import FileBlob
from govoplan_files.backend.runtime import configure_runtime
from govoplan_files.backend.storage.common import FileStorageError
from govoplan_files.backend.storage.content_protection import protect_blob_content
from govoplan_files.backend.storage.integrity import read_verified_blob_bytes
class Registry:
def __init__(self, capability=None) -> None:
self.capability_value = capability
def has_capability(self, name: str) -> bool:
return (
name == CAPABILITY_ENCRYPTION_CONTENT_CIPHER
and self.capability_value is not None
)
def capability(self, _name: str):
return self.capability_value
class Cipher:
def protect_content(self, _session, *, request):
ciphertext = b"protected:" + request.plaintext
envelope = ContentProtectionEnvelope(
envelope_id="envelope-file-1",
tenant_id=request.tenant_id,
owner_module=request.owner_module,
resource_type=request.resource_type,
resource_id=request.resource_id,
profile_kind="server_envelope",
profile_id=request.profile_id,
provider_id="test",
vault_id=request.vault_id,
key_version=1,
algorithm_suite="AES-256-GCM",
ciphertext_ref=request.ciphertext_ref,
ciphertext_digest=f"sha256:{hashlib.sha256(ciphertext).hexdigest()}",
authenticated_context_digest="sha256:context",
state="active",
created_at=datetime.now(tz=UTC),
wrapped_key_refs=("wrapped:1",),
)
return ProtectedContent(envelope=envelope, ciphertext=ciphertext)
def unprotect_content(self, _session, *, request):
if not request.ciphertext.startswith(b"protected:"):
raise ValueError("invalid fixture ciphertext")
return request.ciphertext.removeprefix(b"protected:")
def execute_rewrap(self, *_args, **_kwargs):
raise NotImplementedError
def prepare_reencryption(self, *_args, **_kwargs):
raise NotImplementedError
class Backend:
def __init__(self, data: bytes) -> None:
self.data = data
def get_bytes(self, _key: str) -> bytes:
return self.data
class FileContentProtectionTests(unittest.TestCase):
def setUp(self) -> None:
self.engine = create_engine("sqlite+pysqlite:///:memory:")
FileBlob.__table__.create(self.engine)
self.session = Session(self.engine)
def tearDown(self) -> None:
self.session.close()
self.engine.dispose()
def test_encrypted_blob_is_opened_after_stored_object_integrity_check(self) -> None:
configure_runtime(registry=Registry(Cipher()), settings=object())
plaintext = b"monthly records"
protected = protect_blob_content(
self.session,
tenant_id="tenant-1",
blob_id="blob-1",
vault_id="vault-1",
ciphertext_ref="tenants/tenant-1/files/blob-1",
plaintext=plaintext,
actor_id="account-1",
content_type="text/plain",
)
blob = FileBlob(
id="blob-1",
tenant_id="tenant-1",
storage_backend="test",
storage_key="tenants/tenant-1/files/blob-1",
checksum_sha256=hashlib.sha256(plaintext).hexdigest(),
size_bytes=len(plaintext),
protection_discriminator="vault:vault-1",
encryption_envelope_id=protected.envelope.envelope_id,
storage_checksum_sha256=hashlib.sha256(protected.ciphertext).hexdigest(),
storage_size_bytes=len(protected.ciphertext),
content_type="text/plain",
ref_count=1,
integrity_status="verified",
)
self.session.add(blob)
self.session.flush()
self.assertEqual(
plaintext,
read_verified_blob_bytes(blob, backend=Backend(protected.ciphertext)),
)
def test_encrypted_blob_fails_closed_without_encryption_capability(self) -> None:
configure_runtime(registry=Registry(), settings=object())
ciphertext = b"protected:monthly records"
blob = FileBlob(
id="blob-1",
tenant_id="tenant-1",
storage_backend="test",
storage_key="tenants/tenant-1/files/blob-1",
checksum_sha256=hashlib.sha256(b"monthly records").hexdigest(),
size_bytes=len(b"monthly records"),
protection_discriminator="vault:vault-1",
encryption_envelope_id="envelope-file-1",
storage_checksum_sha256=hashlib.sha256(ciphertext).hexdigest(),
storage_size_bytes=len(ciphertext),
ref_count=1,
integrity_status="verified",
)
self.session.add(blob)
self.session.flush()
with self.assertRaisesRegex(FileStorageError, "Encryption is unavailable"):
read_verified_blob_bytes(blob, backend=Backend(ciphertext))
if __name__ == "__main__":
unittest.main()
+10 -1
View File
@@ -49,6 +49,10 @@ class FilesRuntimeDocumentationTests(unittest.TestCase):
settings = SimpleNamespace(
file_upload_max_bytes=7 * 1024 * 1024,
file_upload_zip_max_bytes=19 * 1024 * 1024,
file_archive_max_expanded_bytes=31 * 1024 * 1024,
file_archive_max_entries=321,
file_archive_max_expansion_ratio=42,
file_archive_preview_ttl_seconds=15 * 60,
)
with patch(
"govoplan_files.backend.documentation.visible_connector_profiles_for_actor",
@@ -73,8 +77,12 @@ class FilesRuntimeDocumentationTests(unittest.TestCase):
archive = topics["files.workflow.upload-and-unpack-zip"]
self.assertIn("19 MiB (19,922,944 bytes)", archive.body)
self.assertIn("31 MiB (32,505,856 bytes)", archive.body)
self.assertIn("7 MiB (7,340,032 bytes)", archive.body)
self.assertIn("1,000", archive.body)
self.assertIn("321", archive.body)
self.assertIn("42:1", archive.body)
self.assertIn("15 minutes", archive.body)
self.assertIn("passwords remain request-only", archive.body)
self.assertIn("Actual extracted bytes are counted", archive.body)
def test_connector_task_requires_authority_and_a_visible_usable_profile(
@@ -134,6 +142,7 @@ class FilesRuntimeDocumentationTests(unittest.TestCase):
base_path="/classified",
credential_mode="basic",
password_value="credential-secret",
secret_ref="runtime/connector-secret",
)
with patch(
"govoplan_files.backend.documentation.visible_connector_profiles_for_actor",
+211
View File
@@ -0,0 +1,211 @@
from __future__ import annotations
import hashlib
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch
from fastapi import HTTPException
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from govoplan_access.backend.db.models import Account, Group, User
from govoplan_core.db.base import Base
from govoplan_files.backend.db.models import (
FileBlob,
FileIntegrityFinding,
FileIntegrityScan,
)
from govoplan_files.backend.routes.integrity import _assert_expected_revision
from govoplan_files.backend.storage.backends import LocalFilesystemStorageBackend
from govoplan_files.backend.storage.common import FileStorageError
from govoplan_files.backend.storage.integrity import (
cleanup_orphan_finding,
create_integrity_scan,
read_verified_blob_bytes,
recheck_integrity_finding,
run_integrity_scan_batch,
)
TENANT_ID = "tenant-1"
USER_ID = "user-1"
class IntegrityReconciliationTests(unittest.TestCase):
def setUp(self) -> None:
self.temporary_directory = tempfile.TemporaryDirectory()
self.addCleanup(self.temporary_directory.cleanup)
self.backend = LocalFilesystemStorageBackend(
Path(self.temporary_directory.name)
)
self.engine = create_engine("sqlite:///:memory:", future=True)
Base.metadata.create_all(
bind=self.engine,
tables=[
Account.__table__,
User.__table__,
Group.__table__,
FileBlob.__table__,
FileIntegrityScan.__table__,
FileIntegrityFinding.__table__,
],
)
self.session = sessionmaker(bind=self.engine, future=True)()
self.addCleanup(self._close)
def _close(self) -> None:
self.session.close()
self.engine.dispose()
def test_scan_is_bounded_resumable_and_reconciles_both_orphan_directions(
self,
) -> None:
valid_data = b"valid"
restored_data = b"restore-me"
expected_corrupt_data = b"expected"
stored_corrupt_data = b"corrupt!"
valid = _blob("blob-1", "valid.bin", valid_data)
missing = _blob("blob-2", "missing.bin", restored_data)
corrupt = _blob("blob-3", "corrupt.bin", expected_corrupt_data)
self.session.add_all([valid, missing, corrupt])
self.session.commit()
self.backend.put_bytes(valid.storage_key, valid_data)
self.backend.put_bytes(corrupt.storage_key, stored_corrupt_data)
orphan_key = f"tenants/{TENANT_ID}/files/orphan.bin"
self.backend.put_bytes(orphan_key, b"orphan")
scan = create_integrity_scan(
self.session,
tenant_id=TENANT_ID,
user_id=USER_ID,
batch_size=1,
backend=self.backend,
)
self.session.commit()
self.assertEqual(1, scan.revision)
invocations = 0
while scan.status != "completed":
run_integrity_scan_batch(
self.session,
scan,
backend=self.backend,
)
self.session.commit()
invocations += 1
scan = self.session.get(FileIntegrityScan, scan.id)
self.assertIsNotNone(scan)
self.session.expire_all()
if invocations > 12:
self.fail("Integrity scan did not complete")
self.assertGreater(invocations, 3)
self.assertEqual(1 + invocations, scan.revision)
self.assertEqual(3, scan.scanned_blob_count)
self.assertEqual(1, scan.verified_blob_count)
self.assertEqual(2, scan.quarantined_blob_count)
self.assertEqual(3, scan.scanned_object_count)
self.assertEqual(1, scan.orphan_object_count)
findings = (
self.session.query(FileIntegrityFinding)
.filter(FileIntegrityFinding.scan_id == scan.id)
.all()
)
self.assertEqual(
{"missing", "checksum_mismatch", "orphan_object"},
{finding.kind for finding in findings},
)
missing = self.session.get(FileBlob, missing.id)
corrupt = self.session.get(FileBlob, corrupt.id)
self.assertEqual("missing", missing.integrity_status)
self.assertEqual("checksum_mismatch", corrupt.integrity_status)
with self.assertRaisesRegex(FileStorageError, "quarantined"):
read_verified_blob_bytes(missing, backend=self.backend)
missing_finding = next(
finding for finding in findings if finding.kind == "missing"
)
self.backend.put_bytes(missing.storage_key, restored_data)
preview = recheck_integrity_finding(
self.session,
missing_finding,
user_id=USER_ID,
dry_run=True,
backend=self.backend,
)
self.assertTrue(preview.inspection.valid)
self.assertEqual("open", missing_finding.state)
repaired = recheck_integrity_finding(
self.session,
missing_finding,
user_id=USER_ID,
dry_run=False,
backend=self.backend,
)
self.session.commit()
self.assertTrue(repaired.changed)
self.assertEqual("resolved", missing_finding.state)
self.assertEqual(
restored_data,
read_verified_blob_bytes(missing, backend=self.backend),
)
orphan_finding = next(
finding for finding in findings if finding.kind == "orphan_object"
)
preview_cleanup = cleanup_orphan_finding(
self.session,
orphan_finding,
user_id=USER_ID,
dry_run=True,
backend=self.backend,
)
self.assertEqual("would_delete", preview_cleanup.action)
self.assertTrue(self.backend.exists(orphan_key))
with patch(
"govoplan_files.backend.storage.integrity.begin_orphan_cleanup_recovery"
):
cleanup = cleanup_orphan_finding(
self.session,
orphan_finding,
user_id=USER_ID,
dry_run=False,
backend=self.backend,
)
repeated = cleanup_orphan_finding(
self.session,
orphan_finding,
user_id=USER_ID,
dry_run=False,
backend=self.backend,
)
self.assertEqual("deleted", cleanup.action)
self.assertFalse(self.backend.exists(orphan_key))
self.assertEqual("already_deleted", repeated.action)
self.assertFalse(repeated.changed)
def test_stale_integrity_action_revision_is_rejected(self) -> None:
with self.assertRaises(HTTPException) as captured:
_assert_expected_revision(3, 2)
self.assertEqual(409, captured.exception.status_code)
self.assertIn("reload", str(captured.exception.detail).lower())
def _blob(blob_id: str, filename: str, expected_data: bytes) -> FileBlob:
return FileBlob(
id=blob_id,
tenant_id=TENANT_ID,
storage_backend="local",
storage_key=f"tenants/{TENANT_ID}/files/{filename}",
checksum_sha256=hashlib.sha256(expected_data).hexdigest(),
size_bytes=len(expected_data),
ref_count=1,
)
if __name__ == "__main__":
unittest.main()
+33 -4
View File
@@ -4,12 +4,15 @@ import unittest
STATIC_TOPIC_IDS = {
"files.search.managed-content",
"files.workflow.organize-managed-files",
"files.workflow.find-and-download-files",
"files.workflow.share-managed-files",
"files.workflow.delete-managed-files",
"files.governed-connectors-and-provenance",
"files.reference.integrity-recovery-and-fail-closed-transports",
"files.reference.shared-storage-profile",
"files.reference.generated-artifact-store",
"files.reference.snapshot-provenance-and-capabilities",
"files.assurance.process-and-release-readiness",
}
@@ -103,8 +106,8 @@ class FilesManifestDocumentationTests(unittest.TestCase):
def test_share_and_delete_tasks_state_current_boundaries(self) -> None:
share = self.topic("files.workflow.share-managed-files")
self.assertEqual("available", share.layer)
self.assertIn("does not yet provide a general share editor", share.body)
self.assertIn("no share-revocation route", share.body)
self.assertIn("Expired and revoked grants stop authorizing", share.body)
self.assertIn("revocation is idempotent", share.body)
self.assertIn(
"/api/v1/files/{file_id}/shares", {link.href for link in share.links}
)
@@ -125,6 +128,14 @@ class FilesManifestDocumentationTests(unittest.TestCase):
self.assertEqual("reference", topic.metadata["kind"])
self.assertEqual("File connections", topic.metadata["screen"])
self.assertTrue(topic.metadata["section"])
self.assertEqual(
{
"files.connectors",
"files.connector.credentials",
"files.connector.policy",
},
set(topic.metadata["help_contexts"]),
)
self.assertIn("deny rules win", topic.body)
self.assertIn("redact secret values", topic.body)
self.assertIn("same transaction", topic.body)
@@ -142,7 +153,7 @@ class FilesManifestDocumentationTests(unittest.TestCase):
"/api/v1/files/connectors/credentials", {link.href for link in topic.links}
)
def test_operator_topic_covers_recovery_and_fail_closed_s3_smb(self) -> None:
def test_operator_topic_covers_recovery_and_pinned_s3_smb(self) -> None:
topic = self.topic(
"files.reference.integrity-recovery-and-fail-closed-transports"
)
@@ -153,9 +164,27 @@ class FilesManifestDocumentationTests(unittest.TestCase):
self.assertIn("SMB", topic.body)
self.assertIn("fail closed", topic.body)
self.assertIn("DFS referrals", topic.body)
self.assertIn("ambient credential discovery", topic.body)
self.assertIn("does not remove backend blob objects", topic.body)
self.assertIn("bounded resumable integrity scan", topic.body)
self.assertIn("quarantined", topic.body)
self.assertIn("MASTER_KEY_B64", topic.metadata["recovery_unit"])
self.assertIn("Encryption envelope and wrapped-key rows", topic.metadata["recovery_unit"])
self.assertIn("lease-fenced Core recovery", topic.body)
self.assertIn("Ops", topic.body)
self.assertTrue(topic.metadata["verification"])
self.assertIn(
"/api/v1/files/integrity/scans",
{link.href for link in topic.links},
)
self.assertIn(
"/admin?section=tenant-file-integrity",
{link.href for link in topic.links},
)
self.assertIn(
"files.admin.tenant-integrity",
{surface.id for surface in self.manifest.frontend.view_surfaces},
)
self.assertIn(
"GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS", topic.configuration_keys
)
@@ -184,7 +213,7 @@ class FilesManifestDocumentationTests(unittest.TestCase):
self.assertIn("process_owner", topic.audience)
self.assertIn("release_manager", topic.audience)
self.assertIn("versions align", topic.body)
self.assertIn("no general share-management UI or share revocation", topic.body)
self.assertIn("Share grant, change, expiry, and revocation", topic.body)
self.assertIn("no enforced retention or legal hold", topic.body)
for key in ("prerequisites", "steps", "outcome", "verification"):
self.assertTrue(topic.metadata[key])
+42
View File
@@ -0,0 +1,42 @@
from __future__ import annotations
from unittest.mock import patch
from govoplan_files.backend.operational_checks import managed_storage_roundtrip_check
from govoplan_files.backend.storage.backends import LocalFilesystemStorageBackend
def test_managed_storage_roundtrip_removes_probe(tmp_path) -> None:
backend = LocalFilesystemStorageBackend(tmp_path)
with patch(
"govoplan_files.backend.operational_checks.get_storage_backend",
return_value=backend,
):
result = managed_storage_roundtrip_check()
assert result.state == "ok"
assert result.metrics["backend"] == "local"
assert list(tmp_path.rglob("*.bin")) == []
def test_managed_storage_roundtrip_fails_closed() -> None:
class BrokenBackend:
name = "broken"
def put_bytes(self, *_args, **_kwargs):
raise OSError("private provider detail")
def delete(self, *_args, **_kwargs):
return None
with patch(
"govoplan_files.backend.operational_checks.get_storage_backend",
return_value=BrokenBackend(),
):
result = managed_storage_roundtrip_check()
assert result.state == "error"
assert result.readiness_critical is True
assert "private provider detail" not in result.detail
+97
View File
@@ -0,0 +1,97 @@
from __future__ import annotations
import unittest
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from govoplan_access.backend.db import models as access_models # noqa: F401
from govoplan_core.core.provider_governance import ExternalProviderStateContext
from govoplan_core.db.base import Base
from govoplan_files.backend.db.models import FileConnectorProfile, FileConnectorSpace
from govoplan_files.backend.manifest import manifest
from govoplan_files.backend.provider_state import (
REMOTE_STORAGE_PROVIDER_ID,
remote_storage_provider_states,
)
class FilesProviderStateTests(unittest.TestCase):
def setUp(self) -> None:
self.engine = create_engine("sqlite+pysqlite:///:memory:", future=True)
Base.metadata.create_all(
self.engine,
tables=(FileConnectorProfile.__table__, FileConnectorSpace.__table__),
)
self.session = sessionmaker(bind=self.engine, expire_on_commit=False)()
self.profile = FileConnectorProfile(
id="profile-1",
tenant_id="tenant-1",
scope_type="tenant",
scope_id="tenant-1",
label="Remote WebDAV",
provider="webdav",
endpoint_url="https://files.example.test/dav/",
enabled=True,
)
self.session.add_all(
(
self.profile,
FileConnectorSpace(
id="space-1",
tenant_id="tenant-1",
owner_type="tenant",
label="Documents",
connector_profile_id=self.profile.id,
provider="webdav",
remote_path="/documents",
read_only=True,
is_active=True,
),
)
)
self.session.commit()
def tearDown(self) -> None:
self.session.close()
self.engine.dispose()
def test_state_does_not_overstate_live_health_or_expose_endpoint(self) -> None:
state = remote_storage_provider_states(
ExternalProviderStateContext(session=self.session, tenant_id="tenant-1")
)[0]
self.assertEqual("external_mirror", state.authority_mode)
self.assertEqual("unknown", state.health)
self.assertEqual("attention", state.recovery)
self.assertEqual(1, state.metrics["active_spaces"])
self.assertNotIn("files.example.test", str(state.to_dict()))
self.assertEqual(
(),
remote_storage_provider_states(
ExternalProviderStateContext(
session=self.session,
tenant_id="tenant-2",
)
),
)
def test_unimplemented_profile_is_fail_closed_and_manifest_registered(self) -> None:
self.profile.provider = "sharepoint"
self.session.flush()
state = remote_storage_provider_states(
ExternalProviderStateContext(session=self.session, tenant_id="tenant-1")
)[0]
self.assertEqual("error", state.health)
self.assertEqual("unsupported", state.recovery)
self.assertEqual(REMOTE_STORAGE_PROVIDER_ID, manifest.external_providers[0].id)
self.assertEqual(
REMOTE_STORAGE_PROVIDER_ID,
manifest.external_provider_state_providers[0].provider_id,
)
if __name__ == "__main__":
unittest.main()
+77
View File
@@ -1,11 +1,68 @@
from __future__ import annotations
import unittest
from collections import Counter
from inspect import signature
from govoplan_files.backend.router import router
from govoplan_files.backend.routes.assets import router as assets_router
from govoplan_files.backend.routes.connector_io import router as connector_io_router
from govoplan_files.backend.routes.connector_profiles import router as connector_profiles_router
from govoplan_files.backend.routes.connector_settings import router as connector_settings_router
from govoplan_files.backend.routes.folders import router as folders_router
from govoplan_files.backend.routes.integrity import router as integrity_router
from govoplan_files.backend.routes.listing import router as listing_router
from govoplan_files.backend.routes.shares import router as shares_router
from govoplan_files.backend.routes.spaces import router as spaces_router
from govoplan_files.backend.routes.transfers import router as transfers_router
from govoplan_files.backend.routes.uploads import router as uploads_router
class FilesRouterContractTests(unittest.TestCase):
@staticmethod
def _operation_keys(candidate_router) -> list[tuple[str, str]]:
return [
(method, route.path)
for route in candidate_router.routes
for method in sorted(route.methods or ())
]
def test_composed_router_contains_every_workflow_operation_once(self) -> None:
workflow_routers = (
spaces_router,
folders_router,
integrity_router,
listing_router,
uploads_router,
connector_settings_router,
connector_io_router,
connector_profiles_router,
assets_router,
shares_router,
transfers_router,
)
expected = [
operation
for workflow_router in workflow_routers
for operation in self._operation_keys(workflow_router)
]
actual = self._operation_keys(router)
self.assertEqual(expected, actual)
self.assertEqual(52, len(actual))
self.assertFalse(
[operation for operation, count in Counter(actual).items() if count > 1]
)
def test_archive_preview_and_confirmation_routes_are_exposed(self) -> None:
routes = {
(tuple(sorted(route.methods or ())), route.path)
for route in router.routes
}
self.assertIn((("POST",), "/files/archive-preview"), routes)
self.assertIn((("POST",), "/files/archive-confirm"), routes)
def test_connector_routes_keep_existing_api_paths(self) -> None:
routes = {(tuple(sorted(route.methods or ())), route.path) for route in router.routes}
@@ -31,6 +88,26 @@ class FilesRouterContractTests(unittest.TestCase):
self.assertIn((("POST",), "/files/bulk-rename"), routes)
self.assertIn((("POST",), "/files/transfer"), routes)
def test_share_lifecycle_routes_are_exposed(self) -> None:
routes = {(tuple(sorted(route.methods or ())), route.path) for route in router.routes}
self.assertIn((("GET",), "/files/{file_id}/shares"), routes)
self.assertIn((("POST",), "/files/{file_id}/shares"), routes)
self.assertIn((("DELETE",), "/files/{file_id}/shares/{share_id}"), routes)
self.assertIn((("GET",), "/files/{file_id}/share-target-options"), routes)
def test_file_listing_exposes_structured_property_filters(self) -> None:
route = next(
route
for route in listing_router.routes
if route.path == "/files" and "GET" in (route.methods or ())
)
parameters = signature(route.endpoint).parameters
self.assertIn("campaign_usage", parameters)
self.assertIn("audit_relevant", parameters)
self.assertIn("total", route.response_model.model_fields)
if __name__ == "__main__":
unittest.main()
+294
View File
@@ -0,0 +1,294 @@
from __future__ import annotations
import socket
import sys
import threading
import types
import unittest
from importlib.util import find_spec
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from unittest.mock import patch
from govoplan_core.security.outbound_http import (
OutboundHttpBlocked,
create_outbound_connection as core_create_outbound_connection,
)
from govoplan_files.backend.storage.sdk_peer_pinning import (
SdkPeerPinningError,
_botocore_transport_types,
create_pinned_s3_client,
install_pinned_smb_transport,
pinned_smb_connection_cache,
)
class _Socket:
def __init__(self) -> None:
self.connected_to: object | None = None
def settimeout(self, _value: object) -> None:
return None
def setsockopt(self, *_args: object) -> None:
return None
def connect(self, value: object) -> None:
self.connected_to = value
def close(self) -> None:
return None
class _S3Handler(BaseHTTPRequestHandler):
def do_GET(self) -> None: # noqa: N802 - stdlib handler contract
body = (
b'<?xml version="1.0" encoding="UTF-8"?>'
b'<ListAllMyBucketsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">'
b'<Owner><ID>govoplan</ID><DisplayName>GovOPlaN</DisplayName></Owner>'
b'<Buckets><Bucket><Name>evidence</Name><CreationDate>2026-08-04T00:00:00Z</CreationDate>'
b'</Bucket></Buckets></ListAllMyBucketsResult>'
)
self.send_response(200)
self.send_header("Content-Type", "application/xml")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
def log_message(self, _format: str, *_args: object) -> None:
return None
@unittest.skipUnless(find_spec("botocore") and find_spec("boto3"), "boto3 extra is not installed")
class S3PeerPinningTests(unittest.TestCase):
def test_client_is_born_with_the_pinned_http_session(self) -> None:
client = create_pinned_s3_client(
endpoint_url="http://127.0.0.1:9000",
region_name="eu-central-1",
aws_access_key_id="access",
aws_secret_access_key="secret",
)
try:
session_type, _, _ = _botocore_transport_types()
self.assertIsInstance(client._endpoint.http_session, session_type)
self.assertEqual({}, client._endpoint.http_session._proxy_config._proxies)
finally:
client.close()
def test_each_connection_attempt_resolves_and_pins_again(self) -> None:
_, http_connection, _ = _botocore_transport_types()
connection = http_connection(host="objects.example.test", port=80)
sockets = [_Socket(), _Socket()]
with patch(
"govoplan_files.backend.storage.sdk_peer_pinning.create_outbound_connection",
side_effect=sockets,
) as connector:
self.assertIs(sockets[0], connection._new_conn())
self.assertIs(sockets[1], connection._new_conn())
self.assertEqual(2, connector.call_count)
self.assertEqual("objects.example.test", connector.call_args.args[0])
def test_real_sdk_request_uses_the_pinned_socket_path(self) -> None:
server = ThreadingHTTPServer(("127.0.0.1", 0), _S3Handler)
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
endpoint = f"http://127.0.0.1:{server.server_port}"
try:
with patch.dict(
"os.environ",
{"APP_ENV": "production", "GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS": "true"},
), patch(
"govoplan_files.backend.storage.sdk_peer_pinning.create_outbound_connection",
wraps=core_create_outbound_connection,
) as connector:
client = create_pinned_s3_client(
endpoint_url=endpoint,
region_name="eu-central-1",
aws_access_key_id="access",
aws_secret_access_key="secret",
)
try:
response = client.list_buckets()
finally:
client.close()
finally:
server.shutdown()
server.server_close()
thread.join(timeout=2)
self.assertEqual("evidence", response["Buckets"][0]["Name"])
self.assertEqual("127.0.0.1", connector.call_args.args[0])
def test_every_sdk_selected_origin_uses_a_pinned_pool(self) -> None:
session_type, http_connection, https_connection = _botocore_transport_types()
session = session_type(proxies={})
try:
first = session._manager.connection_from_url("http://one.example.test/root")
redirected = session._manager.connection_from_url("https://two.example.test/root")
self.assertIs(first.ConnectionCls, http_connection)
self.assertIs(redirected.ConnectionCls, https_connection)
finally:
session.close()
def test_tls_connection_retains_the_configured_authority_for_sni(self) -> None:
_, _, https_connection = _botocore_transport_types()
connection = https_connection(host="objects.example.test", port=443)
with patch(
"govoplan_files.backend.storage.sdk_peer_pinning.create_outbound_connection",
return_value=_Socket(),
) as connector:
connection._new_conn()
self.assertEqual("objects.example.test", connection.host)
self.assertEqual("objects.example.test", connection._dns_host)
self.assertEqual("objects.example.test", connector.call_args.args[0])
def test_mixed_answer_and_peer_change_fail_closed(self) -> None:
_, http_connection, _ = _botocore_transport_types()
connection = http_connection(host="objects.example.test", port=80)
records = [
[(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("93.184.216.34", 80))],
[
(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("93.184.216.34", 80)),
(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("10.0.0.5", 80)),
],
]
with patch.dict(
"os.environ",
{"APP_ENV": "production", "GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS": "false"},
), patch(
"govoplan_core.security.outbound_http.socket.getaddrinfo",
side_effect=records,
), patch(
"govoplan_core.security.outbound_http.socket.socket",
return_value=_Socket(),
):
connection._new_conn()
with self.assertRaisesRegex(Exception, "non-public network"):
connection._new_conn()
class _FakeTcp:
def __init__(self, server: str, port: int, timeout: float | None = None) -> None:
self.server = server
self.port = port
self.timeout = timeout
self.connected = False
self._sock = None
self._sock_lock = threading.Lock()
class _FakeConnection:
def __init__(self, _guid: object, server: str, port: int, *, require_signing: bool) -> None:
self.server = server
self.port = port
self.require_signing = require_signing
self.transport = _FakeTcp(server, port)
self.session_table: dict[str, object] = {}
def connect(self, *, timeout: float) -> None:
self.transport.timeout = timeout
self.transport.connect()
class _FakeSession:
def __init__(
self,
connection: _FakeConnection,
*,
username: str | None,
password: str | None,
require_encryption: bool,
auth_protocol: str,
) -> None:
self.connection = connection
self.username = username
self.password = password
self.encrypt_data = require_encryption
self.auth_protocol = auth_protocol
self.encrypt = require_encryption
def connect(self) -> None:
self.connection.session_table[self.username or "anonymous"] = self
class SmbPeerPinningTests(unittest.TestCase):
def setUp(self) -> None:
pinned_smb_connection_cache.cache_clear()
def _modules(self) -> tuple[types.ModuleType, dict[str, types.ModuleType]]:
smbclient = types.ModuleType("smbclient")
pool = types.ModuleType("smbclient._pool")
def original_register_session(
server: str,
username: str | None = None,
password: str | None = None,
port: int = 445,
encrypt: bool | None = None,
connection_timeout: float = 60,
connection_cache: dict[str, object] | None = None,
auth_protocol: str = "negotiate",
require_signing: bool = True,
) -> None:
del server, username, password, port, encrypt, connection_timeout
del connection_cache, auth_protocol, require_signing
pool.register_session = original_register_session
pool.ClientConfig = lambda: types.SimpleNamespace(client_guid="guid")
connection = types.ModuleType("smbprotocol.connection")
connection.Connection = _FakeConnection
connection.Tcp = _FakeTcp
session = types.ModuleType("smbprotocol.session")
session.Session = _FakeSession
transport = types.ModuleType("smbprotocol.transport")
transport.Tcp = _FakeTcp
return smbclient, {
"smbclient._pool": pool,
"smbprotocol.connection": connection,
"smbprotocol.session": session,
"smbprotocol.transport": transport,
}
def test_initial_reconnect_and_referral_hosts_are_each_pinned(self) -> None:
smbclient, modules = self._modules()
sockets = [_Socket(), _Socket(), _Socket()]
with patch.dict(sys.modules, modules), patch(
"govoplan_files.backend.storage.sdk_peer_pinning.create_outbound_connection",
side_effect=sockets,
) as connector:
install_pinned_smb_transport(smbclient)
register = modules["smbclient._pool"].register_session
cache: dict[str, object] = {}
first = register("files.example.test", connection_cache=cache)
first.connection.transport.connected = False
register("files.example.test", connection_cache=cache)
register("dfs-target.example.test", connection_cache=cache)
self.assertEqual(
["files.example.test", "files.example.test", "dfs-target.example.test"],
[call.args[0] for call in connector.call_args_list],
)
def test_referral_to_disallowed_peer_fails_before_session_creation(self) -> None:
smbclient, modules = self._modules()
with patch.dict(sys.modules, modules), patch(
"govoplan_files.backend.storage.sdk_peer_pinning.create_outbound_connection",
side_effect=OutboundHttpBlocked("non-public network"),
):
install_pinned_smb_transport(smbclient)
register = modules["smbclient._pool"].register_session
with self.assertRaisesRegex(ValueError, "non-public network"):
register("private-referral.example.test", connection_cache={})
def test_transport_tampering_after_installation_fails_closed(self) -> None:
smbclient, modules = self._modules()
with patch.dict(sys.modules, modules):
install_pinned_smb_transport(smbclient)
modules["smbprotocol.connection"].Tcp = _FakeTcp
with self.assertRaisesRegex(SdkPeerPinningError, "changed after peer pinning"):
install_pinned_smb_transport(smbclient)
if __name__ == "__main__":
unittest.main()
+145
View File
@@ -0,0 +1,145 @@
from __future__ import annotations
from types import SimpleNamespace
import unittest
from sqlalchemy import create_engine
from sqlalchemy.orm import Session
from govoplan_access.backend.db.models import Account, Group, User
from govoplan_core.auth import ApiPrincipal
from govoplan_core.core.access import PrincipalRef
from govoplan_core.core.change_sequence import ChangeSequenceEntry
from govoplan_core.core.events import EventObjectRef, EventTenantRef, PlatformEvent
from govoplan_core.core.search import (
SearchAuthorizationRequest,
SearchBackfillRequest,
SearchResourceReference,
)
from govoplan_core.db.base import Base
from govoplan_files.backend.db.models import FileAsset, FileFolder, FileShare
from govoplan_files.backend.search_source import (
FilesSearchSource,
PROVIDER_ID,
)
class FilesSearchSourceTests(unittest.TestCase):
def setUp(self) -> None:
self.engine = create_engine("sqlite://")
Base.metadata.create_all(
self.engine,
tables=(
Account.__table__,
User.__table__,
Group.__table__,
FileAsset.__table__,
FileFolder.__table__,
FileShare.__table__,
ChangeSequenceEntry.__table__,
),
)
self.session = Session(self.engine)
self.session.add_all(
(
Account(
id="account-1",
email="one@example.test",
normalized_email="one@example.test",
),
User(
id="user-1",
tenant_id="tenant-1",
account_id="account-1",
email="one@example.test",
),
FileAsset(
id="file-1",
tenant_id="tenant-1",
owner_type="user",
owner_user_id="user-1",
display_path="records/permit.pdf",
filename="permit.pdf",
description="Monthly permit evidence",
),
FileAsset(
id="file-other",
tenant_id="tenant-2",
owner_type="user",
display_path="other.pdf",
filename="other.pdf",
),
)
)
self.session.commit()
self.source = FilesSearchSource()
def tearDown(self) -> None:
self.session.close()
self.engine.dispose()
def test_backfill_and_authorization_are_tenant_bounded(self) -> None:
page = self.source.backfill(
self.session,
request=SearchBackfillRequest(
tenant_id="tenant-1",
provider_id=PROVIDER_ID,
resource_type="file",
rebuild_id="rebuild-1",
),
)
self.assertEqual(("file-1",), tuple(doc.resource_id for doc in page.documents))
reference = SearchResourceReference(
tenant_id="tenant-1",
module_id="files",
resource_type="file",
resource_id="file-1",
)
request = SearchAuthorizationRequest(reference=reference, source_revision="1")
self.assertTrue(
self.source.authorize(
self.session,
_principal({"files:file:read"}),
requests=(request,),
)[reference.key]
)
self.assertFalse(
self.source.authorize(
self.session,
_principal(set()),
requests=(request,),
)[reference.key]
)
def test_committed_file_event_produces_authoritative_upsert(self) -> None:
event = PlatformEvent(
type="files.file.updated",
module_id="files",
tenant=EventTenantRef(id="tenant-1"),
resource=EventObjectRef(type="file", id="file-1"),
)
changes = self.source.index_changes_for_event(
self.session,
event=event,
delivery_key="delivery-1",
)
self.assertEqual(1, len(changes))
self.assertEqual("upsert", changes[0].kind)
self.assertEqual(event.event_id, changes[0].cursor)
def _principal(scopes: set[str]) -> ApiPrincipal:
return ApiPrincipal(
principal=PrincipalRef(
account_id="account-1",
membership_id="user-1",
tenant_id="tenant-1",
scopes=frozenset(scopes),
),
account=SimpleNamespace(id="account-1"),
user=SimpleNamespace(id="user-1"),
)
if __name__ == "__main__":
unittest.main()
+237
View File
@@ -0,0 +1,237 @@
from __future__ import annotations
import unittest
from datetime import timedelta
from unittest.mock import patch
from sqlalchemy import create_engine
from sqlalchemy.dialects import postgresql
from sqlalchemy.orm import sessionmaker
from govoplan_access.backend.db.models import Account, Group, User
from govoplan_core.core.change_sequence import ChangeSequenceEntry
from govoplan_core.db.base import Base
from govoplan_files.backend.db.models import FileAsset, FileShare
from govoplan_files.backend.storage.common import FileStorageError, utcnow
from govoplan_files.backend.storage.files import (
_asset_visibility_query_for_user,
get_asset_for_user,
list_file_shares,
revoke_file_share,
share_file,
)
TENANT_ID = "tenant-1"
OWNER_ID = "owner-1"
RECIPIENT_ID = "recipient-1"
GROUP_ID = "group-1"
class FileShareLifecycleTests(unittest.TestCase):
def setUp(self) -> None:
self.engine = create_engine("sqlite:///:memory:", future=True)
Base.metadata.create_all(
bind=self.engine,
tables=[
Account.__table__,
User.__table__,
Group.__table__,
ChangeSequenceEntry.__table__,
FileAsset.__table__,
FileShare.__table__,
],
)
self.session = sessionmaker(bind=self.engine, future=True)()
self.asset = FileAsset(
id="file-1",
tenant_id=TENANT_ID,
owner_type="user",
owner_user_id=OWNER_ID,
display_path="shared.pdf",
filename="shared.pdf",
)
self.session.add(self.asset)
self.session.commit()
def tearDown(self) -> None:
self.session.close()
self.engine.dispose()
@patch(
"govoplan_files.backend.storage.files.user_group_ids",
return_value=[],
)
def test_expired_share_stops_access_immediately(self, _groups) -> None:
self.session.add(
FileShare(
id="expired-share",
tenant_id=TENANT_ID,
file_asset_id=self.asset.id,
target_type="user",
target_id=RECIPIENT_ID,
permission="read",
expires_at=utcnow() - timedelta(seconds=1),
)
)
self.session.commit()
with self.assertRaisesRegex(FileStorageError, "No access"):
get_asset_for_user(
self.session,
tenant_id=TENANT_ID,
user_id=RECIPIENT_ID,
asset_id=self.asset.id,
)
self.assertEqual(
[],
list_file_shares(
self.session,
tenant_id=TENANT_ID,
asset_id=self.asset.id,
),
)
@patch(
"govoplan_files.backend.storage.files.user_group_ids",
return_value=[GROUP_ID],
)
def test_independent_active_grant_survives_other_expiry(self, _groups) -> None:
self.session.add_all(
[
FileShare(
id="expired-user-share",
tenant_id=TENANT_ID,
file_asset_id=self.asset.id,
target_type="user",
target_id=RECIPIENT_ID,
permission="read",
expires_at=utcnow() - timedelta(seconds=1),
),
FileShare(
id="active-group-share",
tenant_id=TENANT_ID,
file_asset_id=self.asset.id,
target_type="group",
target_id=GROUP_ID,
permission="read",
expires_at=utcnow() + timedelta(hours=1),
),
]
)
self.session.commit()
result = get_asset_for_user(
self.session,
tenant_id=TENANT_ID,
user_id=RECIPIENT_ID,
asset_id=self.asset.id,
)
self.assertEqual(self.asset.id, result.id)
self.assertEqual(
["active-group-share"],
[
share.id
for share in list_file_shares(
self.session,
tenant_id=TENANT_ID,
asset_id=self.asset.id,
)
],
)
@patch(
"govoplan_files.backend.storage.files.ensure_share_target_exists",
return_value=None,
)
def test_revoke_is_idempotent_and_future_expiry_is_persisted(
self, _target_exists
) -> None:
expiry = utcnow() + timedelta(days=1)
share = share_file(
self.session,
tenant_id=TENANT_ID,
asset=self.asset,
target_type="user",
target_id=RECIPIENT_ID,
permission="read",
user_id=OWNER_ID,
expires_at=expiry,
)
self.session.commit()
revoked, first_changed = revoke_file_share(
self.session,
tenant_id=TENANT_ID,
asset_id=self.asset.id,
share_id=share.id,
user_id=OWNER_ID,
)
self.session.commit()
repeated, second_changed = revoke_file_share(
self.session,
tenant_id=TENANT_ID,
asset_id=self.asset.id,
share_id=share.id,
user_id=OWNER_ID,
)
self.assertTrue(first_changed)
self.assertFalse(second_changed)
self.assertEqual(OWNER_ID, revoked.revoked_by_user_id)
self.assertEqual(revoked.revoked_at, repeated.revoked_at)
self.assertEqual([], list_file_shares(
self.session,
tenant_id=TENANT_ID,
asset_id=self.asset.id,
))
self.assertEqual(
[share.id],
[
item.id
for item in list_file_shares(
self.session,
tenant_id=TENANT_ID,
asset_id=self.asset.id,
include_inactive=True,
)
],
)
@patch(
"govoplan_files.backend.storage.files.ensure_share_target_exists",
return_value=None,
)
def test_past_expiry_is_rejected(self, _target_exists) -> None:
with self.assertRaisesRegex(FileStorageError, "future"):
share_file(
self.session,
tenant_id=TENANT_ID,
asset=self.asset,
target_type="user",
target_id=RECIPIENT_ID,
permission="read",
user_id=OWNER_ID,
expires_at=utcnow() - timedelta(seconds=1),
)
@patch(
"govoplan_files.backend.storage.files.user_group_ids",
return_value=[],
)
def test_visibility_query_is_postgresql_json_safe(self, _groups) -> None:
query = _asset_visibility_query_for_user(
self.session,
tenant_id=TENANT_ID,
user_id=RECIPIENT_ID,
)
compiled = str(query.statement.compile(dialect=postgresql.dialect()))
self.assertNotIn("SELECT DISTINCT", compiled.upper())
self.assertIn("EXISTS", compiled.upper())
if __name__ == "__main__":
unittest.main()
+51 -2
View File
@@ -1,19 +1,26 @@
from __future__ import annotations
import io
from types import ModuleType
import sys
import unittest
from unittest.mock import MagicMock, PropertyMock, patch
from govoplan_files.backend.storage.backends import S3StorageBackend, StorageBackendError
def _backend() -> S3StorageBackend:
def _backend(
*,
endpoint_url: str = "https://objects.example.test",
deployment_managed: bool = False,
) -> S3StorageBackend:
return S3StorageBackend(
bucket="files",
endpoint_url="https://objects.example.test",
endpoint_url=endpoint_url,
region_name="test",
access_key_id="access",
secret_access_key="secret",
deployment_managed=deployment_managed,
)
@@ -32,6 +39,48 @@ class S3StorageBackendTests(unittest.TestCase):
), self.assertRaisesRegex(StorageBackendError, "until that transport supports.*DNS/IP pinning"):
_backend().client
def test_installer_managed_garage_uses_exact_endpoint_and_path_style(self) -> None:
boto3 = ModuleType("boto3")
boto3.client = MagicMock(return_value=object())
botocore = ModuleType("botocore")
botocore.__path__ = []
botocore_config = ModuleType("botocore.config")
class Config:
def __init__(self, **values):
self.values = values
botocore_config.Config = Config
with patch.dict(
sys.modules,
{
"boto3": boto3,
"botocore": botocore,
"botocore.config": botocore_config,
},
):
_backend(
endpoint_url="http://garage:3900",
deployment_managed=True,
).client
_args, kwargs = boto3.client.call_args
self.assertEqual("http://garage:3900", kwargs["endpoint_url"])
self.assertEqual(
{"s3": {"addressing_style": "path"}},
kwargs["config"].values,
)
def test_installer_managed_garage_rejects_any_other_endpoint(self) -> None:
with self.assertRaisesRegex(
StorageBackendError,
"restricted to http://garage:3900",
):
_backend(
endpoint_url="http://other-s3:3900",
deployment_managed=True,
).client
def test_get_bytes_rejects_declared_oversize_object_without_reading(self) -> None:
body = MagicMock()
client = MagicMock()
+326
View File
@@ -0,0 +1,326 @@
from __future__ import annotations
import hashlib
from pathlib import Path
import tempfile
import unittest
from unittest.mock import patch
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from govoplan_access.backend.db.models import Account, Group, User
from govoplan_core.core.recovery import (
RecoveryCheckpoint,
RecoveryOperation,
RecoveryStatus,
)
from govoplan_core.core.runtime_coordination import (
DistributedLease,
RuntimeIdentity,
bind_process_runtime_identity,
)
from govoplan_core.db.base import Base
from govoplan_core.db.session import configure_database, reset_database
from govoplan_files.backend.db.models import (
FileBlob,
FileIntegrityFinding,
FileIntegrityScan,
)
from govoplan_files.backend.storage.backends import (
LocalFilesystemStorageBackend,
)
from govoplan_files.backend.storage.common import FileStorageError
from govoplan_files.backend.storage.files import _get_or_create_blob
from govoplan_files.backend.storage.integrity import cleanup_orphan_finding
from govoplan_files.backend.storage.recovery import begin_blob_write_recovery
TENANT_ID = "tenant-1"
USER_ID = "user-1"
class StorageRecoveryTests(unittest.TestCase):
def setUp(self) -> None:
self.temporary_directory = tempfile.TemporaryDirectory()
self.addCleanup(self.temporary_directory.cleanup)
root = Path(self.temporary_directory.name)
self.backend = LocalFilesystemStorageBackend(root / "objects")
database_path = root / "recovery.sqlite3"
self.engine = create_engine(f"sqlite:///{database_path}", future=True)
Base.metadata.create_all(
bind=self.engine,
tables=[
Account.__table__,
User.__table__,
Group.__table__,
DistributedLease.__table__,
RecoveryOperation.__table__,
RecoveryCheckpoint.__table__,
FileBlob.__table__,
FileIntegrityScan.__table__,
FileIntegrityFinding.__table__,
],
)
configure_database(
f"sqlite:///{database_path}",
engine=self.engine,
dispose_previous=True,
)
bind_process_runtime_identity(
RuntimeIdentity(
installation_id="files-recovery-test",
node_id="node-1",
incarnation="incarnation-1",
role="api",
software_version="test",
composition_hash="a" * 64,
)
)
self.Session = sessionmaker(
bind=self.engine,
expire_on_commit=False,
future=True,
)
self.enterContext(
patch(
"govoplan_files.backend.storage.files._storage_backend_name",
return_value=self.backend.name,
)
)
self.enterContext(
patch(
"govoplan_files.backend.storage.files._storage_bucket_name",
return_value="",
)
)
self.session = self.Session()
self.addCleanup(self._cleanup_runtime)
def _cleanup_runtime(self) -> None:
self.session.close()
bind_process_runtime_identity(None)
reset_database()
self.engine.dispose()
def test_committed_blob_write_is_independently_verified(self) -> None:
observed_running_operation: list[bool] = []
put_bytes = self.backend.put_bytes
class ObservingBackend:
name = self.backend.name
def __getattr__(backend_self, name):
return getattr(self.backend, name)
def put_bytes(backend_self, key, data, *, content_type=None):
with self.Session() as evidence_session:
operation = evidence_session.query(RecoveryOperation).one()
observed_running_operation.append(
operation.status == RecoveryStatus.RUNNING.value
and len(operation.request_sha256) == 64
)
put_bytes(key, data, content_type=content_type)
observing_backend = ObservingBackend()
with patch(
"govoplan_files.backend.storage.files.get_storage_backend",
return_value=observing_backend,
):
blob = _get_or_create_blob(
self.session,
tenant_id=TENANT_ID,
data=b"durable",
filename="private-name.txt",
content_type="text/plain",
actor_id=USER_ID,
)
self.session.commit()
operation = self._only_operation()
self.assertEqual(RecoveryStatus.SUCCEEDED.value, operation.status)
self.assertEqual([True], observed_running_operation)
self.assertTrue(self.backend.exists(blob.storage_key))
self.assertNotIn("private-name", blob.storage_key)
self.assertEqual(".blob", Path(blob.storage_key).suffix)
def test_rolled_back_blob_write_is_compensated(self) -> None:
with patch(
"govoplan_files.backend.storage.files.get_storage_backend",
return_value=self.backend,
):
blob = _get_or_create_blob(
self.session,
tenant_id=TENANT_ID,
data=b"rollback",
filename="rollback.txt",
content_type="text/plain",
actor_id=USER_ID,
)
storage_key = blob.storage_key
self.assertTrue(self.backend.exists(storage_key))
self.session.rollback()
operation = self._only_operation()
self.assertEqual(RecoveryStatus.RECOVERED.value, operation.status)
self.assertFalse(self.backend.exists(storage_key))
with self.Session() as evidence_session:
self.assertIsNone(evidence_session.get(FileBlob, blob.id))
def test_post_write_tamper_is_quarantined_and_recovery_required(self) -> None:
with patch(
"govoplan_files.backend.storage.files.get_storage_backend",
return_value=self.backend,
):
blob = _get_or_create_blob(
self.session,
tenant_id=TENANT_ID,
data=b"expected",
filename="evidence.bin",
content_type="application/octet-stream",
actor_id=USER_ID,
)
self.backend.put_bytes(blob.storage_key, b"tampered")
self.session.commit()
operation = self._only_operation()
self.assertEqual(
RecoveryStatus.RECOVERY_REQUIRED.value,
operation.status,
)
with self.Session() as evidence_session:
persisted = evidence_session.get(FileBlob, blob.id)
self.assertIsNotNone(persisted)
self.assertEqual("checksum_mismatch", persisted.integrity_status)
self.assertIsNotNone(persisted.quarantined_at)
def test_missing_optional_encryption_fails_before_object_effect(self) -> None:
with patch(
"govoplan_files.backend.storage.files.get_storage_backend",
return_value=self.backend,
), patch(
"govoplan_files.backend.storage.content_protection.encryption_content_cipher",
return_value=None,
), self.assertRaisesRegex(FileStorageError, "Encryption module"):
_get_or_create_blob(
self.session,
tenant_id=TENANT_ID,
data=b"protected",
filename="protected.bin",
content_type="application/octet-stream",
actor_id=USER_ID,
encryption_vault_id="vault-1",
)
self.session.rollback()
operation = self._only_operation()
self.assertEqual(RecoveryStatus.REJECTED.value, operation.status)
objects = self.backend.list_objects(
prefix=f"tenants/{TENANT_ID}/files/",
limit=10,
)
self.assertEqual((), objects.objects)
def test_orphan_cleanup_forward_completes_after_business_rollback(self) -> None:
key = f"tenants/{TENANT_ID}/files/orphan.bin"
self.backend.put_bytes(key, b"orphan")
scan = FileIntegrityScan(
id="scan-1",
tenant_id=TENANT_ID,
storage_backend=self.backend.name,
storage_prefix=f"tenants/{TENANT_ID}/files/",
status="completed",
)
finding = FileIntegrityFinding(
id="finding-1",
scan_id=scan.id,
tenant_id=TENANT_ID,
kind="orphan_object",
state="open",
storage_key=key,
observed_size_bytes=6,
observed_checksum_sha256=hashlib.sha256(b"orphan").hexdigest(),
)
self.session.add_all([scan, finding])
self.session.commit()
cleanup_orphan_finding(
self.session,
finding,
user_id=USER_ID,
dry_run=False,
backend=self.backend,
)
self.assertFalse(self.backend.exists(key))
self.session.rollback()
operation = self._only_operation()
self.assertEqual(RecoveryStatus.SUCCEEDED.value, operation.status)
with self.Session() as evidence_session:
persisted = evidence_session.get(FileIntegrityFinding, finding.id)
self.assertIsNotNone(persisted)
self.assertEqual("deleted", persisted.state)
def test_missing_runtime_identity_blocks_before_object_write(self) -> None:
bind_process_runtime_identity(None)
with patch(
"govoplan_files.backend.storage.files.get_storage_backend",
return_value=self.backend,
), self.assertRaisesRegex(FileStorageError, "recovery ledger"):
_get_or_create_blob(
self.session,
tenant_id=TENANT_ID,
data=b"blocked",
filename="blocked.bin",
content_type="application/octet-stream",
actor_id=USER_ID,
)
self.session.rollback()
objects = self.backend.list_objects(
prefix=f"tenants/{TENANT_ID}/files/",
limit=10,
)
self.assertEqual((), objects.objects)
def test_blob_fence_blocks_a_competing_runtime_before_effect(self) -> None:
checksum = hashlib.sha256(b"fenced").hexdigest()
begin_blob_write_recovery(
self.session,
backend=self.backend,
tenant_id=TENANT_ID,
blob_id="blob-fenced",
storage_key=f"tenants/{TENANT_ID}/files/fenced.blob",
semantic_checksum_sha256=checksum,
semantic_size_bytes=6,
protection_discriminator="plaintext",
created_new=True,
)
with self.Session() as competing_session, self.assertRaisesRegex(
FileStorageError,
"already owned",
):
begin_blob_write_recovery(
competing_session,
backend=self.backend,
tenant_id=TENANT_ID,
blob_id="blob-fenced",
storage_key=f"tenants/{TENANT_ID}/files/fenced.blob",
semantic_checksum_sha256=checksum,
semantic_size_bytes=6,
protection_discriminator="plaintext",
created_new=True,
)
self.session.rollback()
self.assertEqual(RecoveryStatus.REJECTED.value, self._only_operation().status)
def _only_operation(self) -> RecoveryOperation:
with self.Session() as session:
operations = session.query(RecoveryOperation).all()
self.assertEqual(1, len(operations))
session.expunge(operations[0])
return operations[0]
if __name__ == "__main__":
unittest.main()
+108
View File
@@ -0,0 +1,108 @@
from __future__ import annotations
import unittest
from sqlalchemy import create_engine, event
from sqlalchemy.orm import Session
from govoplan_access.backend.db.models import Account, Group, User
from govoplan_core.core.change_sequence import ChangeSequenceEntry
from govoplan_core.db.base import Base
from govoplan_files.backend.db.models import (
FileAsset,
FileConnectorCredential,
FileConnectorPolicy,
FileConnectorProfile,
FileConnectorSpace,
)
from govoplan_files.backend.manifest import _tenant_summary_batch
class FilesTenantSummaryBatchTests(unittest.TestCase):
def test_batch_summary_uses_one_grouped_query_per_owned_table(self) -> None:
engine = create_engine("sqlite+pysqlite:///:memory:")
Base.metadata.create_all(
engine,
tables=[
Account.__table__,
User.__table__,
Group.__table__,
FileAsset.__table__,
FileConnectorCredential.__table__,
FileConnectorPolicy.__table__,
FileConnectorProfile.__table__,
FileConnectorSpace.__table__,
ChangeSequenceEntry.__table__,
],
)
try:
with Session(engine) as session:
session.add_all(
[
FileAsset(
id="file-1",
tenant_id="tenant-1",
owner_type="tenant",
display_path="/one.txt",
filename="one.txt",
),
FileConnectorCredential(
id="credential-1",
tenant_id="tenant-1",
label="Credential",
),
FileConnectorPolicy(
id="policy-1",
tenant_id="tenant-1",
scope_type="tenant",
),
FileConnectorProfile(
id="profile-1",
tenant_id="tenant-1",
label="Profile",
provider="webdav",
),
FileConnectorSpace(
id="space-1",
tenant_id="tenant-1",
owner_type="tenant",
label="Space",
connector_profile_id="profile-1",
provider="webdav",
),
]
)
session.commit()
query_count = 0
def count_query(*_args: object) -> None:
nonlocal query_count
query_count += 1
event.listen(engine, "before_cursor_execute", count_query)
try:
counts = _tenant_summary_batch(
session,
["tenant-1", "tenant-empty"],
)
finally:
event.remove(engine, "before_cursor_execute", count_query)
self.assertEqual(5, query_count)
self.assertEqual(
{
"files": 1,
"connector_credentials": 1,
"connector_policies": 1,
"connector_profiles": 1,
"connector_spaces": 1,
},
counts["tenant-1"],
)
self.assertEqual(0, counts["tenant-empty"]["files"])
finally:
engine.dispose()
if __name__ == "__main__":
unittest.main()
+10 -8
View File
@@ -1,6 +1,6 @@
{
"name": "@govoplan/files-webui",
"version": "0.1.9",
"version": "0.1.16",
"private": true,
"type": "module",
"main": "src/index.ts",
@@ -14,17 +14,19 @@
"./styles/file-manager.css": "./src/styles/file-manager.css"
},
"scripts": {
"test:file-drop-target": "node scripts/test-file-drop-target-structure.mjs"
"test:file-drop-target": "node scripts/test-file-drop-target-structure.mjs",
"test:file-property-filters": "node scripts/test-file-property-filters-structure.mjs",
"test:interface-pattern-language": "node scripts/test-interface-pattern-language.mjs"
},
"peerDependencies": {
"@vitejs/plugin-react": "^4.3.4",
"vite": "^6.0.6",
"@vitejs/plugin-react": "^5.2.0",
"vite": "^7.3.6",
"typescript": "^5.7.2",
"react": "^19.0.0",
"react-dom": "^19.0.0",
"react-router-dom": "^7.1.1",
"react": ">=19.2.7 <20",
"react-dom": ">=19.2.7 <20",
"react-router": ">=8.3.0 <9",
"lucide-react": "^1.23.0",
"@govoplan/core-webui": "^0.1.9"
"@govoplan/core-webui": "^0.1.16"
},
"peerDependenciesMeta": {
"@govoplan/core-webui": {
@@ -0,0 +1,36 @@
import { readFileSync } from "node:fs";
const page = readFileSync(new URL("../src/features/files/FilesPage.tsx", import.meta.url), "utf8").replace(/\s+/g, " ");
const api = readFileSync(new URL("../src/api/files.ts", import.meta.url), "utf8").replace(/\s+/g, " ");
function assertIncludes(source, fragment, message) {
if (!source.includes(fragment.replace(/\s+/g, " "))) throw new Error(message);
}
assertIncludes(
api,
"campaign_usage?: FileCampaignUsageFilter; audit_relevant?: boolean;",
"the Files API must expose structured campaign and audit filters"
);
assertIncludes(
api,
"do { const response = await listFiles",
"property filtering must traverse every server result page"
);
assertIncludes(
page,
"if (propertyFiltersActive && propertyFilterResults) return propertyFilterResults;",
"server-filtered rows must replace, not narrow, the currently loaded explorer window"
);
assertIncludes(
page,
'<option value="unlinked">Not linked or used</option>',
"the high-priority unlinked campaign-file filter must remain available"
);
assertIncludes(
page,
"setPropertyFilterTotal(response.total);",
"the UI must retain the exact server-side filtered count"
);
console.log("Files property filters cover the complete server result window.");
@@ -0,0 +1,56 @@
import assert from "node:assert/strict";
import { readFileSync } from "node:fs";
import { fileURLToPath } from "node:url";
function read(relativePath) {
return readFileSync(fileURLToPath(new URL(relativePath, import.meta.url)), "utf8");
}
const connector = read("../src/features/files/FileConnectorSettingsPanel.tsx");
const integrity = read("../src/features/files/FileIntegrityPanel.tsx");
const filesApi = read("../src/api/files.ts");
const filesPage = read("../src/features/files/FilesPage.tsx");
const moduleSource = read("../src/module.ts");
const styles = read("../src/styles/file-manager.css");
const migration = read("../../docs/INTERFACE_PATTERN_MIGRATION.md");
assert.match(connector, /ActionBlockerHint,[\s\S]*AdvancedOptionsPanel,[\s\S]*DocumentationHelpLink/);
assert.match(connector, /ConnectionTree,[\s\S]*ConfirmDialog,[\s\S]*Dialog,[\s\S]*LoadingFrame/);
assert.match(connector, /topicId: "files\.governed-connectors-and-provenance"/);
assert.match(connector, /documentation=\{CONNECTOR_DOCUMENTATION\}/);
assert.match(connector, /disabledReason=\{profileSaveDisabledReason\}/);
assert.match(connector, /disabledReason=\{credentialSaveDisabledReason\}/);
assert.match(connector, /disabledReason: mutationBlocker/);
assert.match(connector, /<AdvancedOptionsPanel title="Connection metadata JSON"/);
assert.match(connector, /<AdvancedOptionsPanel title="Credential metadata JSON"/);
assert.doesNotMatch(connector, /window\.(?:alert|confirm)\(/);
assert.match(integrity, /File integrity/);
assert.match(filesApi, /expected_revision/);
assert.match(integrity, /cleanupFileIntegrityFinding\(settings, finding, true\)/);
assert.match(integrity, /<ConfirmDialog[\s\S]*Delete unreferenced storage object/);
assert.match(integrity, /files\.reference\.integrity-recovery-and-fail-closed-transports/);
assert.match(moduleSource, /files\.admin\.tenant-integrity/);
assert.doesNotMatch(integrity, /window\.(?:alert|confirm)\(/);
assert.match(filesPage, /DocumentationHelpLink/);
assert.match(filesPage, /topicId: "files\.workflow\.organize-managed-files"/);
assert.match(filesPage, /disabledReason=\{uploadBlocker\}/);
assert.match(filesPage, /disabledReason=\{deleteBlocker\}/);
assert.match(filesPage, /<ConfirmDialog[\s\S]*tone="danger"/);
assert.match(filesPage, /className="workspace-data-page module-entry-page file-manager-page file-manager-fullscreen files-page"/);
assert.doesNotMatch(filesPage, /window\.(?:alert|confirm)\(/);
assert.doesNotMatch(`${connector}\n${integrity}\n${filesPage}\n${moduleSource}`, /@govoplan\/(?:campaign|mail|docs)-webui|govoplan_(?:campaign|mail|docs)/);
assert.match(moduleSource, /"files\.connectors"/);
assert.match(moduleSource, /"files\.fileExplorer"/);
assert.match(styles, /@media \(max-width: 1050px\)[\s\S]*\.files-page \.file-manager-shell[\s\S]*grid-template-columns: 1fr/);
assert.match(styles, /@media \(max-width: 760px\)[\s\S]*\.file-connector-profile-row[\s\S]*grid-template-columns: 1fr/);
for (const archetype of ["Directory/explorer", "Adaptive create/edit", "Effective-policy editor", "Dashboard widget"]) {
assert.match(migration, new RegExp(archetype.replace("/", "\\/")));
}
assert.match(migration, /Shared `Dialog` owns focus entry, Escape handling and focus return/);
assert.match(migration, /Secret values are[\s\S]*never returned for rendering/);
console.log("Files surfaces satisfy the recorded interface pattern-language contract.");
+326 -6
View File
@@ -1,4 +1,15 @@
import { ApiError, apiFetch, apiUrl, authHeaders, csrfToken, type ApiSettings, type DeltaDeletedItem, type FilesManagedFileLinkTarget } from "@govoplan/core-webui";
import {
ApiError,
apiFetch,
apiReferenceOptionProvider,
apiUrl,
authHeaders,
csrfToken,
type ApiSettings,
type DeltaDeletedItem,
type FilesManagedFileLinkTarget,
type ReferenceOptionProvider
} from "@govoplan/core-webui";
export { fetchResourceAccessExplanation } from "@govoplan/core-webui";
export type {
AccessDecisionProvenanceItem,
@@ -24,11 +35,16 @@ export type FileSpace = {
export type FileShare = {
id: string;
file_asset_id: string;
target_type: string;
target_id: string;
permission: string;
created_by_user_id?: string | null;
created_at: string;
expires_at?: string | null;
revoked_at?: string | null;
revoked_by_user_id?: string | null;
active: boolean;
};
export type FileSourceProvenance = {
@@ -191,6 +207,57 @@ export type FileConnectorCredentialUpdatePayload = Partial<Omit<FileConnectorCre
clear_token?: boolean;
};
export type FileIntegrityScan = {
id: string;
tenant_id: string;
storage_backend: string;
storage_prefix: string;
status: string;
revision: number;
phase: string;
verify_checksums: boolean;
batch_size: number;
scanned_blob_count: number;
verified_blob_count: number;
quarantined_blob_count: number;
scanned_object_count: number;
orphan_object_count: number;
created_by_user_id?: string | null;
started_at?: string | null;
completed_at?: string | null;
last_error?: string | null;
created_at: string;
updated_at: string;
};
export type FileIntegrityFinding = {
id: string;
scan_id: string;
tenant_id: string;
kind: string;
state: string;
revision: number;
blob_id?: string | null;
storage_key: string;
expected_size_bytes?: number | null;
observed_size_bytes?: number | null;
expected_checksum_sha256?: string | null;
observed_checksum_sha256?: string | null;
resolved_at?: string | null;
resolved_by_user_id?: string | null;
created_at: string;
updated_at: string;
};
export type FileIntegrityActionResult = {
action: string;
changed: boolean;
dry_run: boolean;
finding: FileIntegrityFinding;
inspection_kind?: string | null;
inspection_valid?: boolean | null;
};
export type FileConnectorBrowseItem = {
kind: "library" | "folder" | "file";
name: string;
@@ -297,7 +364,8 @@ export type ManagedFile = {
shares?: FileShare[];
};
export type FileListResponse = {files: ManagedFile[];cursor?: string | null;next_cursor?: string | null;watermark?: string | null;};
export type FileCampaignUsageFilter = "linked" | "unlinked";
export type FileListResponse = {files: ManagedFile[];total: number;cursor?: string | null;next_cursor?: string | null;watermark?: string | null;};
export type FileDeltaDeletedItem = {id: string;resource_type?: string | null;revision?: string | null;deleted_at?: string | null;};
export type FileDeltaResponse = {
files: ManagedFile[];
@@ -311,6 +379,25 @@ export type ManagedFileSnapshotResponse = {files: ManagedFile[];folders: FileFol
export type FileSpacesResponse = {spaces: FileSpace[];};
export type FileUploadResponse = {files: ManagedFile[];};
export type FileUploadProgress = {loaded: number;total?: number;percentage: number | null;};
export type ArchivePreviewEntry = {
path: string;
kind: "file" | "directory";
size_bytes: number;
compressed_size_bytes?: number | null;
encrypted: boolean;
};
export type ArchivePreviewResponse = {
preview_token: string;
archive_format: string;
entries: ArchivePreviewEntry[];
file_count: number;
directory_count: number;
expanded_size_bytes: number;
compressed_size_bytes: number;
requires_password: boolean;
password_verified: boolean;
expires_at: string;
};
export type FileConnectorFilePayload = {
library_id: string;
path: string;
@@ -383,15 +470,43 @@ payload: {owner_type: "user" | "group";owner_id: string;path: string;recursive?:
return apiFetch<FolderDeleteResponse>(settings, "/api/v1/files/folders/delete", { method: "POST", body: JSON.stringify({ recursive: true, ...payload }) });
}
export function listFiles(settings: ApiSettings, params: {owner_type?: string;owner_id?: string;campaign_id?: string;path_prefix?: string;page_size?: number;cursor?: string | null;} = {}): Promise<FileListResponse> {
export function listFiles(settings: ApiSettings, params: {owner_type?: string;owner_id?: string;campaign_id?: string;path_prefix?: string;campaign_usage?: FileCampaignUsageFilter;audit_relevant?: boolean;page_size?: number;cursor?: string | null;} = {}): Promise<FileListResponse> {
const search = new URLSearchParams();
for (const [key, value] of Object.entries(params)) {
if (value) search.set(key, String(value));
if (value !== undefined && value !== null && value !== "") search.set(key, String(value));
}
const suffix = search.toString() ? `?${search.toString()}` : "";
return apiFetch<FileListResponse>(settings, `/api/v1/files${suffix}`);
}
export async function listFilesByProperties(
settings: ApiSettings,
params: {
owner_type: "user" | "group";
owner_id: string;
path_prefix?: string;
campaign_usage?: FileCampaignUsageFilter;
audit_relevant?: boolean;
page_size?: number;
})
: Promise<{files: ManagedFile[];total: number;}> {
const pageSize = params.page_size ?? DEFAULT_MANAGED_FILE_WINDOW_SIZE;
let cursor: string | null | undefined = null;
let total = 0;
let files: ManagedFile[] = [];
do {
const response = await listFiles(settings, {
...params,
page_size: pageSize,
cursor
});
files = files.concat(response.files);
total = response.total;
cursor = response.next_cursor;
} while (cursor);
return { files, total };
}
export function listFilesDelta(
settings: ApiSettings,
params: {owner_type?: string;owner_id?: string;campaign_id?: string;path_prefix?: string;since?: string;limit?: number;} = {})
@@ -511,14 +626,80 @@ options: {
return apiFetch<FileUploadResponse>(settings, "/api/v1/files/upload", { method: "POST", body: form });
}
export function previewArchiveUpload(
settings: ApiSettings,
file: File,
options: {
owner_type: "user" | "group";
owner_id: string;
path?: string;
campaign_id?: string;
password?: string;
})
: Promise<ArchivePreviewResponse> {
const form = new FormData();
form.append("file", file);
form.append("owner_type", options.owner_type);
form.append("owner_id", options.owner_id);
form.append("path", options.path ?? "");
if (options.campaign_id) form.append("campaign_id", options.campaign_id);
if (options.password) form.append("password", options.password);
return apiFetch<ArchivePreviewResponse>(settings, "/api/v1/files/archive-preview", { method: "POST", body: form });
}
export function confirmArchiveUpload(
settings: ApiSettings,
file: File,
options: {
preview_token: string;
selected_paths: string[];
owner_type: "user" | "group";
owner_id: string;
path?: string;
campaign_id?: string;
password?: string;
conflict_strategy?: ConflictStrategy;
conflict_resolutions?: ConflictResolution[];
source_provenance?: FileSourceProvenance;
source_revision?: string;
connector_policy_sources?: FileConnectorPolicySource[];
onProgress?: (progress: FileUploadProgress) => void;
})
: Promise<FileUploadResponse> {
const form = new FormData();
form.append("file", file);
form.append("preview_token", options.preview_token);
form.append("selected_paths_json", JSON.stringify(options.selected_paths));
form.append("owner_type", options.owner_type);
form.append("owner_id", options.owner_id);
form.append("path", options.path ?? "");
if (options.campaign_id) form.append("campaign_id", options.campaign_id);
if (options.password) form.append("password", options.password);
if (options.conflict_strategy) form.append("conflict_strategy", options.conflict_strategy);
if (options.conflict_resolutions?.length) form.append("conflict_resolutions_json", JSON.stringify(options.conflict_resolutions));
if (options.source_provenance) form.append("source_provenance_json", JSON.stringify(options.source_provenance));
if (options.source_revision) form.append("source_revision", options.source_revision);
if (options.connector_policy_sources?.length) form.append("connector_policy_json", JSON.stringify({ sources: options.connector_policy_sources }));
if (options.onProgress) {
return uploadFilesWithProgress(
settings,
form,
options.onProgress,
"/api/v1/files/archive-confirm"
);
}
return apiFetch<FileUploadResponse>(settings, "/api/v1/files/archive-confirm", { method: "POST", body: form });
}
function uploadFilesWithProgress(
settings: ApiSettings,
form: FormData,
onProgress: (progress: FileUploadProgress) => void)
onProgress: (progress: FileUploadProgress) => void,
endpoint = "/api/v1/files/upload")
: Promise<FileUploadResponse> {
return new Promise((resolve, reject) => {
const xhr = new XMLHttpRequest();
xhr.open("POST", apiUrl(settings, "/api/v1/files/upload"));
xhr.open("POST", apiUrl(settings, endpoint));
xhr.withCredentials = true;
for (const [key, value] of authHeaders(settings)) xhr.setRequestHeader(key, value);
const csrf = csrfToken();
@@ -564,6 +745,12 @@ export function bulkDeleteFiles(settings: ApiSettings, fileIds: string[]): Promi
}
export type FileBulkShareResponse = {shares: FileShare[];shared_count: number;};
export type FileSharePayload = {
target_type: "user" | "group" | "tenant" | "campaign";
target_id: string;
permission: "read" | "write" | "manage";
expires_at?: string | null;
};
export function virtualFolderResourceId(options: {tenantId: string;ownerType: "user" | "group";ownerId: string;path: string;}): string {
return `virtual-folder:v1:${options.tenantId}:${options.ownerType}:${options.ownerId}:${base64Url(options.path.replace(/\\/g, "/").replace(/^\/+|\/+$/g, ""))}`;
@@ -583,6 +770,54 @@ export function shareFilesWithTarget(settings: ApiSettings, fileIds: string[], t
});
}
export function listFileShares(
settings: ApiSettings,
fileId: string,
options: { includeInactive?: boolean } = {}
): Promise<{ shares: FileShare[] }> {
const suffix = options.includeInactive ? "?include_inactive=true" : "";
return apiFetch<{ shares: FileShare[] }>(
settings,
`/api/v1/files/${encodeURIComponent(fileId)}/shares${suffix}`
);
}
export function createFileShare(
settings: ApiSettings,
fileId: string,
payload: FileSharePayload
): Promise<FileShare> {
return apiFetch<FileShare>(
settings,
`/api/v1/files/${encodeURIComponent(fileId)}/shares`,
{ method: "POST", body: JSON.stringify(payload) }
);
}
export function revokeFileShare(
settings: ApiSettings,
fileId: string,
shareId: string
): Promise<FileShare> {
return apiFetch<FileShare>(
settings,
`/api/v1/files/${encodeURIComponent(fileId)}/shares/${encodeURIComponent(shareId)}`,
{ method: "DELETE" }
);
}
export function fileShareTargetProvider(
settings: ApiSettings,
fileId: string,
targetType: "user" | "group"
): ReferenceOptionProvider {
return apiReferenceOptionProvider(
settings,
`/api/v1/files/${encodeURIComponent(fileId)}/share-target-options`,
{ target_type: targetType }
);
}
export function evaluateFileConnectorPolicy(
settings: ApiSettings,
payload: {source_provenance: FileSourceProvenance;operation?: string;policy_sources?: FileConnectorPolicySource[];})
@@ -855,6 +1090,91 @@ payload: {
return apiFetch<TransferResponse>(settings, "/api/v1/files/transfer", { method: "POST", body: JSON.stringify(payload) });
}
export async function listFileIntegrityScans(
settings: ApiSettings,
limit = 100
): Promise<FileIntegrityScan[]> {
const response = await apiFetch<{ scans: FileIntegrityScan[] }>(
settings,
`/api/v1/files/integrity/scans?limit=${Math.max(1, Math.min(limit, 200))}`
);
return response.scans;
}
export function createFileIntegrityScan(
settings: ApiSettings,
payload: { verify_checksums: boolean; batch_size: number }
): Promise<FileIntegrityScan> {
return apiFetch<FileIntegrityScan>(settings, "/api/v1/files/integrity/scans", {
method: "POST",
body: JSON.stringify(payload)
});
}
export function runFileIntegrityScanBatch(
settings: ApiSettings,
scan: Pick<FileIntegrityScan, "id" | "revision">
): Promise<FileIntegrityScan> {
return apiFetch<FileIntegrityScan>(
settings,
`/api/v1/files/integrity/scans/${encodeURIComponent(scan.id)}/run`,
{
method: "POST",
body: JSON.stringify({ expected_revision: scan.revision })
}
);
}
export async function listFileIntegrityFindings(
settings: ApiSettings,
scanId: string,
state?: string
): Promise<FileIntegrityFinding[]> {
const search = new URLSearchParams({ limit: "1000" });
if (state) search.set("state", state);
const response = await apiFetch<{ findings: FileIntegrityFinding[] }>(
settings,
`/api/v1/files/integrity/scans/${encodeURIComponent(scanId)}/findings?${search}`
);
return response.findings;
}
export function recheckFileIntegrityFinding(
settings: ApiSettings,
finding: Pick<FileIntegrityFinding, "id" | "revision">,
dryRun = false
): Promise<FileIntegrityActionResult> {
return apiFetch<FileIntegrityActionResult>(
settings,
`/api/v1/files/integrity/findings/${encodeURIComponent(finding.id)}/recheck`,
{
method: "POST",
body: JSON.stringify({
dry_run: dryRun,
expected_revision: finding.revision
})
}
);
}
export function cleanupFileIntegrityFinding(
settings: ApiSettings,
finding: Pick<FileIntegrityFinding, "id" | "revision">,
dryRun: boolean
): Promise<FileIntegrityActionResult> {
return apiFetch<FileIntegrityActionResult>(
settings,
`/api/v1/files/integrity/findings/${encodeURIComponent(finding.id)}/cleanup`,
{
method: "POST",
body: JSON.stringify({
dry_run: dryRun,
expected_revision: finding.revision
})
}
);
}
export async function downloadFile(settings: ApiSettings, file: ManagedFile): Promise<void> {
const response = await fetch(apiUrl(settings, `/api/v1/files/${file.id}/download`), { headers: authHeaders(settings), credentials: "include" });
if (!response.ok) throw new Error(`${response.status} ${response.statusText}: ${await response.text()}`);
@@ -6,24 +6,29 @@ import {
ConnectionTree,
ConfirmDialog,
CredentialPanel,
DisabledActionTooltip,
DismissibleAlert,
Dialog,
DocumentationHelpLink,
FormField,
ActionBlockerHint,
AdvancedOptionsPanel,
LoadingFrame,
mergeDeltaRows,
ReferenceMultiSelect,
SegmentedControl,
staticReferenceOptionProvider,
StatusBadge,
TableActionGroup,
ToggleSwitch,
useDeltaWatermarks,
usePlatformLanguage,
useUnsavedDraftGuard,
wildcardReferenceOption,
type ApiSettings,
type ConnectionTreeColumn,
type FileConnectorScope,
type FileConnectorTargetOption,
type ReferenceOption,
i18nMessage } from
"@govoplan/core-webui";
import {
@@ -144,6 +149,16 @@ const ENDPOINT_PLACEHOLDERS: Record<Provider, string> = {
generic: "https://files.example.test"
};
const CONNECTOR_DOCUMENTATION = {
topicId: "files.governed-connectors-and-provenance",
documentationType: "admin"
} as const;
const CONNECTOR_RECOVERY_DOCUMENTATION = {
topicId: "files.reference.integrity-recovery-and-fail-closed-transports",
documentationType: "admin"
} as const;
export default function FileConnectorSettingsPanel({
settings,
scopeType,
@@ -186,6 +201,7 @@ export default function FileConnectorSettingsPanel({
const [credentialLoginResult, setCredentialLoginResult] = useState<{ok: boolean;message: string;} | null>(null);
const [message, setMessage] = useState("");
const [error, setError] = useState("");
const { translateText } = usePlatformLanguage();
const { getDeltaWatermark, setDeltaWatermark, resetDeltaWatermark } = useDeltaWatermarks();
const requiresTarget = scopeType === "user" || scopeType === "group";
@@ -198,6 +214,10 @@ export default function FileConnectorSettingsPanel({
const profileDirty = Boolean(draft) && connectorProfileDraftKey(draft) !== savedDraftKey;
const credentialDirty = Boolean(credentialDraft) && connectorCredentialDraftKey(credentialDraft) !== savedCredentialDraftKey;
const policyDirty = scopeReady && centralPolicyDraftKey(policyDraft) !== savedPolicyDraftKey;
const policyConflicts = useMemo(
() => policyReferenceConflicts(policyDraft),
[policyDraft]
);
useUnsavedDraftGuard({
dirty: profileDirty || credentialDirty || policyDirty,
@@ -225,6 +245,79 @@ export default function FileConnectorSettingsPanel({
() => credentials.filter((credential) => connectorBelongsToEditableScope(credential, scopeType, activeScopeId)),
[activeScopeId, credentials, scopeType]
);
const selectableCredentials = useMemo(
() => credentials.filter((credential) =>
connectorBelongsToEditableScope(credential, scopeType, activeScopeId) ||
credential.source_kind === "credential_envelope" &&
reusableCredentialAvailableAtScope(credential, scopeType, activeScopeId)
),
[activeScopeId, credentials, scopeType]
);
const connectorReferenceOptions = useMemo<ReferenceOption[]>(
() => profiles.map((profile) => ({
value: profile.id,
label: profile.label || profile.id,
description: [
profile.provider,
profile.source_path,
profile.enabled ? null : "Inactive"
].filter(Boolean).join(" · "),
kind: "file_connection",
availability: profile.enabled ? "available" : "inactive",
disabled: !profile.enabled,
sourceModule: "files",
provenance: {
scopeType: profile.scope_type,
scopeId: profile.scope_id,
sourcePath: profile.source_path
}
})),
[profiles]
);
const credentialReferenceOptions = useMemo<ReferenceOption[]>(
() => credentials.map((credential) => ({
value: credential.id,
label: credential.label || credential.id,
description: [
credential.provider || "shared",
credential.source_path,
credential.enabled ? null : "Inactive"
].filter(Boolean).join(" · "),
kind: "file_credential",
availability: credential.enabled ? "available" : "inactive",
disabled: !credential.enabled,
sourceModule: credential.source_kind === "credential_envelope" ? "core" : "files",
provenance: {
scopeType: credential.scope_type,
scopeId: credential.scope_id,
sourcePath: credential.source_path,
sourceKind: credential.source_kind
}
})),
[credentials]
);
const providerReferenceOptions = useMemo<ReferenceOption[]>(
() => PROVIDERS.map((provider) => ({
value: provider.id,
label: translateText(provider.label),
description: provider.id,
kind: "file_provider",
sourceModule: "files"
})),
[translateText]
);
const connectorReferenceProvider = useMemo(
() => staticReferenceOptionProvider(connectorReferenceOptions),
[connectorReferenceOptions]
);
const credentialReferenceProvider = useMemo(
() => staticReferenceOptionProvider(credentialReferenceOptions),
[credentialReferenceOptions]
);
const providerReferenceProvider = useMemo(
() => staticReferenceOptionProvider(providerReferenceOptions),
[providerReferenceOptions]
);
useEffect(() => {
resetDeltaWatermark(deltaKey);
@@ -424,7 +517,7 @@ export default function FileConnectorSettingsPanel({
endpoint_url: cleanOrNull(draft.endpointUrl),
base_path: cleanOrNull(draft.basePath),
enabled: draft.enabled,
credential_profile_id: cleanOrNull(credentialProfileIdForDraft(draft, scopedCredentials)),
credential_profile_id: cleanOrNull(credentialProfileIdForDraft(draft, selectableCredentials)),
credential_mode: draft.credentialMode,
capabilities,
policy: policyFromDraft(draft),
@@ -694,7 +787,7 @@ export default function FileConnectorSettingsPanel({
const panelTitle = title ?? i18nMessage("i18n:govoplan-files.value_file_connections", { value0: scopeLabel(scopeType) });
const selectedCredentialIds = new Set(scopedProfiles.map((profile) => profile.credential_profile_id).filter((value): value is string => Boolean(value)));
const firstCompatibleProfileByCredential = new Map<string, string>();
for (const credential of scopedCredentials) {
for (const credential of selectableCredentials) {
const selectedProfile = scopedProfiles.find((profile) => profile.credential_profile_id === credential.id);
const fallbackProfile = selectedProfile ?? scopedProfiles.find((profile) => !selectedCredentialIds.has(credential.id) && credentialMatchesProfile(credential, profile));
if (fallbackProfile) firstCompatibleProfileByCredential.set(credential.id, fallbackProfile.id);
@@ -713,12 +806,16 @@ export default function FileConnectorSettingsPanel({
const credentialSaveDisabled = !credentialDraft || saving || credentialMissingRequired;
const profileSaveTooltip = requiredFieldsTooltip(profileMissingFields);
const credentialSaveTooltip = requiredFieldsTooltip(credentialMissingFields);
const profileSaveDisabledReason = saving ? "Wait for the current connector change to finish." : profileSaveTooltip;
const credentialSaveDisabledReason = saving ? "Wait for the current credential change to finish." : credentialSaveTooltip;
const canDiscoverProfileEndpoint = Boolean(draft && (draft.provider === "webdav" || draft.provider === "nextcloud"));
const credentialTestUnsupported = Boolean(credentialTestProfile && credentialTestProfile.provider !== "webdav" && credentialTestProfile.provider !== "nextcloud");
const credentialTestDisabled = !credentialDraft || saving || testingCredentialLogin || !credentialTestProfile || credentialTestUnsupported;
const credentialTestTooltip = credentialTestHelpText(credentialTestProfile, credentialTestUnsupported);
const profileCredentialOptions = draft ? credentialOptionsForProfileDraft(scopedCredentials, draft) : [];
const profileCredentialSelectValue = draft ? credentialProfileIdForDraft(draft, scopedCredentials) : "";
const credentialTestTooltip = saving ? "Wait for the current connector change to finish." :
testingCredentialLogin ? "The credential test is already running." :
credentialTestHelpText(credentialTestProfile, credentialTestUnsupported);
const profileCredentialOptions = draft ? credentialOptionsForProfileDraft(selectableCredentials, draft) : [];
const profileCredentialSelectValue = draft ? credentialProfileIdForDraft(draft, selectableCredentials) : "";
const connectorColumns: ConnectionTreeColumn<ConnectorTreeRow>[] = [
{
id: "connection",
@@ -755,7 +852,7 @@ export default function FileConnectorSettingsPanel({
function connectorChildren(row: ConnectorTreeRow): ConnectorTreeRow[] {
if (row.kind !== "profile") return [];
return scopedCredentials.
return selectableCredentials.
filter((credential) => firstCompatibleProfileByCredential.get(credential.id) === row.profile.id).
map((credential) => ({ kind: "credential" as const, id: `credential:${row.profile.id}:${credential.id}`, credential, profile: row.profile }));
}
@@ -763,18 +860,23 @@ export default function FileConnectorSettingsPanel({
function renderConnectorActions(row: ConnectorTreeRow) {
if (row.kind === "credential") {
const readOnly = row.credential.source_kind !== "database";
const mutationBlocker = connectorMutationBlocker(canWrite, readOnly, saving);
return <TableActionGroup actions={[
{ id: "edit", label: i18nMessage("i18n:govoplan-files.edit_value.fad75899", { value0: row.credential.label }), icon: <Edit3 size={15} />, disabled: !canWrite || readOnly || saving, onClick: () => startCredentialEdit(row.credential) },
{ id: "disable", label: i18nMessage("i18n:govoplan-files.disable_value.09485f7f", { value0: row.credential.label }), icon: <Trash2 size={15} />, variant: "danger", applicable: row.credential.enabled, disabled: !canWrite || readOnly || saving, onClick: () => setPendingCredentialDeactivate(row.credential) }
{ id: "edit", label: i18nMessage("i18n:govoplan-files.edit_value.fad75899", { value0: row.credential.label }), icon: <Edit3 size={15} />, disabled: Boolean(mutationBlocker), disabledReason: mutationBlocker, onClick: () => startCredentialEdit(row.credential) },
{ id: "disable", label: i18nMessage("i18n:govoplan-files.disable_value.09485f7f", { value0: row.credential.label }), icon: <Trash2 size={15} />, variant: "danger", disabled: Boolean(mutationBlocker) || !row.credential.enabled, disabledReason: mutationBlocker || (!row.credential.enabled ? "This credential is already disabled." : ""), onClick: () => setPendingCredentialDeactivate(row.credential) }
]} />;
}
const readOnly = row.profile.source_kind !== "database";
const mutationBlocker = connectorMutationBlocker(canWrite, readOnly, saving);
const testBlocker = !row.profile.enabled ? "Enable this connection before testing it." :
saving ? "Wait for the current connector change to finish." :
testingProfileId === row.profile.id ? "This connection test is already running." : "";
return <TableActionGroup actions={[
{ id: "test", label: i18nMessage("i18n:govoplan-files.test_value.6a5d10a5", { value0: row.profile.label }), icon: <RefreshCw size={15} />, applicable: row.profile.enabled, disabled: saving || testingProfileId === row.profile.id, onClick: () => void testBrowse(row.profile) },
{ id: "add-credential", label: i18nMessage("i18n:govoplan-files.add_credential_for_value.0fa9c1fe", { value0: row.profile.label }), icon: <UserRoundKey size={15} />, variant: "primary", disabled: !canWrite || saving, onClick: () => startCredentialCreate(row.profile) },
{ id: "edit", label: i18nMessage("i18n:govoplan-files.edit_value.fad75899", { value0: row.profile.label }), icon: <Edit3 size={15} />, disabled: !canWrite || readOnly || saving, onClick: () => startEdit(row.profile) },
{ id: "disable", label: i18nMessage("i18n:govoplan-files.disable_value.09485f7f", { value0: row.profile.label }), icon: <Trash2 size={15} />, variant: "danger", applicable: row.profile.enabled, disabled: !canWrite || readOnly || saving, onClick: () => setPendingDeactivate(row.profile) }
{ id: "test", label: i18nMessage("i18n:govoplan-files.test_value.6a5d10a5", { value0: row.profile.label }), icon: <RefreshCw size={15} />, disabled: Boolean(testBlocker), disabledReason: testBlocker, onClick: () => void testBrowse(row.profile) },
{ id: "add-credential", label: i18nMessage("i18n:govoplan-files.add_credential_for_value.0fa9c1fe", { value0: row.profile.label }), icon: <UserRoundKey size={15} />, variant: "primary", disabled: Boolean(mutationBlocker), disabledReason: mutationBlocker, onClick: () => startCredentialCreate(row.profile) },
{ id: "edit", label: i18nMessage("i18n:govoplan-files.edit_value.fad75899", { value0: row.profile.label }), icon: <Edit3 size={15} />, disabled: Boolean(mutationBlocker), disabledReason: mutationBlocker, onClick: () => startEdit(row.profile) },
{ id: "disable", label: i18nMessage("i18n:govoplan-files.disable_value.09485f7f", { value0: row.profile.label }), icon: <Trash2 size={15} />, variant: "danger", disabled: Boolean(mutationBlocker) || !row.profile.enabled, disabledReason: mutationBlocker || (!row.profile.enabled ? "This connection is already disabled." : ""), onClick: () => setPendingDeactivate(row.profile) }
]} />;
}
@@ -793,7 +895,8 @@ export default function FileConnectorSettingsPanel({
requiredAction: "Ask an administrator to grant the required permission or make the change at a higher scope.",
actor: "System or tenant administrator",
target: "Administration > GLOBAL or TENANT > File Connectors"
}} />
}}
documentation={CONNECTOR_DOCUMENTATION} />
}
{targetSelectionRequired && !hasSelectableTarget &&
@@ -805,7 +908,8 @@ export default function FileConnectorSettingsPanel({
requiredAction: "Create or select a user or group first.",
actor: "Access administrator",
target: targetPluralLabel(scopeType, targetLabel)
}} />
}}
documentation={CONNECTOR_DOCUMENTATION} />
}
{targetSelectionRequired &&
@@ -828,10 +932,11 @@ export default function FileConnectorSettingsPanel({
actions={
canWrite ?
<div className="button-row compact-actions">
<Button onClick={() => void loadProfiles()} disabled={loading}>{loading ? "i18n:govoplan-files.loading.b04ba49f" : "i18n:govoplan-files.reload.cce71553"}</Button>
<Button variant="primary" onClick={startCreate} disabled={saving}><Plus size={16} aria-hidden="true" /> i18n:govoplan-files.new_connection.ac979fe4</Button>
<DocumentationHelpLink reference={CONNECTOR_DOCUMENTATION} />
<Button onClick={() => void loadProfiles()} disabled={loading || saving} disabledReason={loading ? "File connections are already loading." : saving ? "Wait for the current connector change to finish." : undefined}>{loading ? "i18n:govoplan-files.loading.b04ba49f" : "i18n:govoplan-files.reload.cce71553"}</Button>
<Button variant="primary" onClick={startCreate} disabled={saving} disabledReason={saving ? "Wait for the current connector change to finish." : undefined}><Plus size={16} aria-hidden="true" /> i18n:govoplan-files.new_connection.ac979fe4</Button>
</div> :
null
<DocumentationHelpLink reference={CONNECTOR_DOCUMENTATION} />
}>
<LoadingFrame loading={loading} label="i18n:govoplan-files.loading_file_connections.bd68f224">
@@ -848,31 +953,82 @@ export default function FileConnectorSettingsPanel({
<Card
title={i18nMessage("i18n:govoplan-files.value_connector_policy.0bf7f53b", { value0: scopeLabel(scopeType) })}
actions={canWrite ?
<Button variant="primary" onClick={() => void savePolicyDraft()} disabled={saving || loading}>
<div className="button-row compact-actions">
<DocumentationHelpLink reference={CONNECTOR_DOCUMENTATION} />
<Button variant="primary" onClick={() => void savePolicyDraft()} disabled={saving || loading} disabledReason={saving ? "Wait for the current connector change to finish." : loading ? "Wait until the effective connector policy has loaded." : undefined}>
<ShieldCheck size={16} aria-hidden="true" /> {saving ? "i18n:govoplan-files.saving.ae7e8875" : "i18n:govoplan-files.save_policy.77d67ce3"}
</Button> :
null}>
</Button>
</div> :
<DocumentationHelpLink reference={CONNECTOR_DOCUMENTATION} />}>
<LoadingFrame loading={loading} label="i18n:govoplan-files.loading_connector_policy.f12ab285">
<>
<div className="form-grid two">
<FormField label="i18n:govoplan-files.allowed_connection_ids.0df854c8">
<textarea value={policyDraft.allowedConnectors} disabled={saving || !canWrite} onChange={(event) => patchPolicyDraft({ allowedConnectors: event.target.value })} rows={3} placeholder={"tenant-webdav\nseafile-*"} />
<ReferenceMultiSelect
values={lines(policyDraft.allowedConnectors)}
onChange={(values) => patchPolicyDraft({ allowedConnectors: values.join("\n") })}
provider={connectorReferenceProvider}
createCustomOption={(value) => wildcardReferenceOption(value)}
aria-label="Allowed file connections"
placeholder="Add a connection or wildcard pattern"
disabled={saving || !canWrite}
/>
</FormField>
<FormField label="i18n:govoplan-files.denied_connection_ids.a95218b4">
<textarea value={policyDraft.deniedConnectors} disabled={saving || !canWrite} onChange={(event) => patchPolicyDraft({ deniedConnectors: event.target.value })} rows={3} placeholder={"legacy-*\nblocked-smb"} />
<ReferenceMultiSelect
values={lines(policyDraft.deniedConnectors)}
onChange={(values) => patchPolicyDraft({ deniedConnectors: values.join("\n") })}
provider={connectorReferenceProvider}
createCustomOption={(value) => wildcardReferenceOption(value)}
aria-label="Denied file connections"
placeholder="Add a connection or wildcard pattern"
disabled={saving || !canWrite}
/>
</FormField>
<FormField label="i18n:govoplan-files.allowed_credential_ids.efcaba2d">
<textarea value={policyDraft.allowedCredentials} disabled={saving || !canWrite} onChange={(event) => patchPolicyDraft({ allowedCredentials: event.target.value })} rows={3} placeholder={"tenant-webdav-credentials\nshared-*"} />
<ReferenceMultiSelect
values={lines(policyDraft.allowedCredentials)}
onChange={(values) => patchPolicyDraft({ allowedCredentials: values.join("\n") })}
provider={credentialReferenceProvider}
createCustomOption={(value) => wildcardReferenceOption(value)}
aria-label="Allowed file credentials"
placeholder="Add a credential or wildcard pattern"
disabled={saving || !canWrite}
/>
</FormField>
<FormField label="i18n:govoplan-files.denied_credential_ids.b6838bc2">
<textarea value={policyDraft.deniedCredentials} disabled={saving || !canWrite} onChange={(event) => patchPolicyDraft({ deniedCredentials: event.target.value })} rows={3} placeholder={"personal-*\nblocked-*"} />
<ReferenceMultiSelect
values={lines(policyDraft.deniedCredentials)}
onChange={(values) => patchPolicyDraft({ deniedCredentials: values.join("\n") })}
provider={credentialReferenceProvider}
createCustomOption={(value) => wildcardReferenceOption(value)}
aria-label="Denied file credentials"
placeholder="Add a credential or wildcard pattern"
disabled={saving || !canWrite}
/>
</FormField>
<FormField label="i18n:govoplan-files.allowed_providers.82a9c23e">
<textarea value={policyDraft.allowedProviders} disabled={saving || !canWrite} onChange={(event) => patchPolicyDraft({ allowedProviders: event.target.value })} rows={3} placeholder={"webdav\nnextcloud\nsmb"} />
<ReferenceMultiSelect
values={lines(policyDraft.allowedProviders)}
onChange={(values) => patchPolicyDraft({ allowedProviders: values.join("\n") })}
provider={providerReferenceProvider}
createCustomOption={(value) => wildcardReferenceOption(value)}
aria-label="Allowed file providers"
placeholder="Add a provider or wildcard pattern"
disabled={saving || !canWrite}
/>
</FormField>
<FormField label="i18n:govoplan-files.denied_providers.149b29f4">
<textarea value={policyDraft.deniedProviders} disabled={saving || !canWrite} onChange={(event) => patchPolicyDraft({ deniedProviders: event.target.value })} rows={3} placeholder={"generic\nnfs"} />
<ReferenceMultiSelect
values={lines(policyDraft.deniedProviders)}
onChange={(values) => patchPolicyDraft({ deniedProviders: values.join("\n") })}
provider={providerReferenceProvider}
createCustomOption={(value) => wildcardReferenceOption(value)}
aria-label="Denied file providers"
placeholder="Add a provider or wildcard pattern"
disabled={saving || !canWrite}
/>
</FormField>
<FormField label="i18n:govoplan-files.allowed_paths.c953b4be">
<textarea value={policyDraft.allowedPaths} disabled={saving || !canWrite} onChange={(event) => patchPolicyDraft({ allowedPaths: event.target.value })} rows={3} placeholder={"GovOPlaN\nreports/*"} />
@@ -887,6 +1043,20 @@ export default function FileConnectorSettingsPanel({
<textarea value={policyDraft.deniedUrls} disabled={saving || !canWrite} onChange={(event) => patchPolicyDraft({ deniedUrls: event.target.value })} rows={3} placeholder={"http://*\n*://legacy.example.test/*"} />
</FormField>
</div>
{policyConflicts.length ? (
<DismissibleAlert
tone="warning"
resetKey={policyConflicts.join("\u0000")}
>
Allow and deny rules contain the same values for{" "}
{policyConflicts.join("; ")}. Deny rules take precedence.
</DismissibleAlert>
) : (
<p className="muted small-note">
When an allow and deny rule both match, the deny rule takes
precedence.
</p>
)}
<div className="file-connector-settings-section file-connector-policy-locks">
<ToggleSwitch label="i18n:govoplan-files.lower_connection_limits.4f9838bc" checked={policyDraft.allowLowerConnectionLimits} disabled={saving || !canWrite} onChange={(allowLowerConnectionLimits) => patchPolicyDraft({ allowLowerConnectionLimits })} />
<ToggleSwitch label="i18n:govoplan-files.lower_credential_limits.ec2b72bc" checked={policyDraft.allowLowerCredentialLimits} disabled={saving || !canWrite} onChange={(allowLowerCredentialLimits) => patchPolicyDraft({ allowLowerCredentialLimits })} />
@@ -915,11 +1085,9 @@ export default function FileConnectorSettingsPanel({
footer={credentialDraft ?
<>
<Button onClick={closeCredentialDialog} disabled={saving}>i18n:govoplan-files.cancel.77dfd213</Button>
<DisabledActionTooltip reason={credentialSaveTooltip}>
<Button variant="primary" onClick={() => void saveCredentialDraft()} disabled={credentialSaveDisabled}>
<Button variant="primary" onClick={() => void saveCredentialDraft()} disabled={credentialSaveDisabled} disabledReason={credentialSaveDisabledReason}>
<Save size={16} aria-hidden="true" /> {saving ? "i18n:govoplan-files.saving.ae7e8875" : "i18n:govoplan-files.save_credential.2c02a6d2"}
</Button>
</DisabledActionTooltip>
</> :
null}>
@@ -960,7 +1128,7 @@ export default function FileConnectorSettingsPanel({
/>
</FormField>
</div>
<FormField label="i18n:govoplan-files.credential_mode.23fdd899">
<FormField label="i18n:govoplan-files.credential_mode.23fdd899" documentation={CONNECTOR_DOCUMENTATION}>
<select value={credentialDraft.credentialMode} disabled={saving} onChange={(event) => patchCredentialDraft({ credentialMode: event.target.value as CredentialMode })}>
<option value="none">i18n:govoplan-files.none.6eef6648</option>
<option value="anonymous">i18n:govoplan-files.anonymous.9bed5104</option>
@@ -1028,11 +1196,9 @@ export default function FileConnectorSettingsPanel({
</div>
}
<div className="button-row compact-actions">
<DisabledActionTooltip reason={credentialTestDisabled ? credentialTestTooltip : ""}>
<Button type="button" onClick={() => void testCredentialLogin()} disabled={credentialTestDisabled}>
<Button type="button" onClick={() => void testCredentialLogin()} disabled={credentialTestDisabled} disabledReason={credentialTestTooltip}>
<RefreshCw size={16} aria-hidden="true" /> {testingCredentialLogin ? "Testing login" : "Test login"}
</Button>
</DisabledActionTooltip>
</div>
{credentialLoginResult &&
<DismissibleAlert tone={credentialLoginResult.ok ? "success" : "warning"} resetKey={credentialLoginResult.message}>
@@ -1047,7 +1213,7 @@ export default function FileConnectorSettingsPanel({
<p>Limit where lower levels may use this credential.</p>
</header>
<div className="form-grid two">
<FormField label="i18n:govoplan-files.policy.bb9cf141">
<FormField label="i18n:govoplan-files.policy.bb9cf141" documentation={CONNECTOR_DOCUMENTATION}>
<select value={credentialDraft.policyMode} disabled={saving} onChange={(event) => patchCredentialDraft({ policyMode: event.target.value as PolicyMode })}>
<option value="allow-provider">i18n:govoplan-files.can_use_this_credential.f4a41971</option>
<option value="allowed-paths">i18n:govoplan-files.must_stay_in_allowed_paths.dc5b4eb4</option>
@@ -1064,7 +1230,7 @@ export default function FileConnectorSettingsPanel({
</FormField>
</div>
<AdvancedOptionsPanel title="Credential metadata JSON" summary="For diagnostics or provider-specific compatibility flags. Normal setup should not need this.">
<FormField label="i18n:govoplan-files.metadata_json.b0e4c283">
<FormField label="i18n:govoplan-files.metadata_json.b0e4c283" documentation={CONNECTOR_RECOVERY_DOCUMENTATION}>
<textarea value={credentialDraft.metadataJson} disabled={saving} onChange={(event) => patchCredentialDraft({ metadataJson: event.target.value })} rows={5} spellCheck={false} />
</FormField>
</AdvancedOptionsPanel>
@@ -1083,11 +1249,9 @@ export default function FileConnectorSettingsPanel({
footer={draft ?
<>
<Button onClick={closeProfileDialog} disabled={saving}>i18n:govoplan-files.cancel.77dfd213</Button>
<DisabledActionTooltip reason={profileSaveTooltip}>
<Button variant="primary" onClick={() => void saveDraft()} disabled={profileSaveDisabled}>
<Button variant="primary" onClick={() => void saveDraft()} disabled={profileSaveDisabled} disabledReason={profileSaveDisabledReason}>
<Save size={16} aria-hidden="true" /> {saving ? "i18n:govoplan-files.saving.ae7e8875" : "i18n:govoplan-files.save_connection.07796d38"}
</Button>
</DisabledActionTooltip>
</> :
null}>
@@ -1132,13 +1296,13 @@ export default function FileConnectorSettingsPanel({
<p>Only fields relevant for the selected provider and credential mode are shown.</p>
</header>
<div className="form-grid two">
<FormField label="i18n:govoplan-files.endpoint_url.65aaaa45">
<FormField label="i18n:govoplan-files.endpoint_url.65aaaa45" documentation={CONNECTOR_DOCUMENTATION}>
<input value={draft.endpointUrl} disabled={saving} onChange={(event) => patchDraft({ endpointUrl: event.target.value })} placeholder={ENDPOINT_PLACEHOLDERS[draft.provider]} />
</FormField>
<FormField label="i18n:govoplan-files.base_path.6a4867ca">
<input value={draft.basePath} disabled={saving} onChange={(event) => patchDraft({ basePath: event.target.value })} placeholder="GovOPlaN" />
</FormField>
<FormField label="i18n:govoplan-files.credential_mode.23fdd899">
<FormField label="i18n:govoplan-files.credential_mode.23fdd899" documentation={CONNECTOR_DOCUMENTATION}>
<select value={draft.credentialMode} disabled={saving} onChange={(event) => patchDraft({ credentialMode: event.target.value as CredentialMode })}>
<option value="none">i18n:govoplan-files.none.6eef6648</option>
<option value="anonymous">i18n:govoplan-files.anonymous.9bed5104</option>
@@ -1155,9 +1319,9 @@ export default function FileConnectorSettingsPanel({
</FormField>
</div>
<div className="button-row compact-actions">
<Button type="button" onClick={() => patchDraft({ endpointUrl: ENDPOINT_PLACEHOLDERS[draft.provider] })} disabled={saving}>Use provider example</Button>
<Button type="button" onClick={() => patchDraft({ endpointUrl: ENDPOINT_PLACEHOLDERS[draft.provider] })} disabled={saving} disabledReason={saving ? "Wait for the current connector change to finish." : undefined}>Use provider example</Button>
{canDiscoverProfileEndpoint &&
<Button type="button" onClick={() => void discoverProfileEndpoint()} disabled={saving || discoveringProfile || !draft.endpointUrl.trim()}>
<Button type="button" onClick={() => void discoverProfileEndpoint()} disabled={saving || discoveringProfile || !draft.endpointUrl.trim()} disabledReason={saving ? "Wait for the current connector change to finish." : discoveringProfile ? "Endpoint discovery is already running." : !draft.endpointUrl.trim() ? "Enter a Nextcloud or WebDAV URL before discovery." : undefined}>
<RefreshCw size={16} aria-hidden="true" /> {discoveringProfile ? "Checking endpoint" : "Discover endpoint"}
</Button>
}
@@ -1203,7 +1367,7 @@ export default function FileConnectorSettingsPanel({
<ToggleSwitch label="i18n:govoplan-files.sync.905f6309" checked={draft.canSync} disabled={saving} onChange={(canSync) => patchDraft({ canSync })} />
</div>
<div className="form-grid two">
<FormField label="i18n:govoplan-files.policy.bb9cf141">
<FormField label="i18n:govoplan-files.policy.bb9cf141" documentation={CONNECTOR_DOCUMENTATION}>
<select value={draft.policyMode} disabled={saving} onChange={(event) => patchDraft({ policyMode: event.target.value as PolicyMode })}>
<option value="allow-provider">i18n:govoplan-files.can_use_this_connection.5091a74c</option>
<option value="allowed-paths">i18n:govoplan-files.must_stay_in_allowed_paths.dc5b4eb4</option>
@@ -1220,7 +1384,7 @@ export default function FileConnectorSettingsPanel({
</FormField>
</div>
<AdvancedOptionsPanel title="Connection metadata JSON" summary="For diagnostics or provider-specific compatibility flags. Normal setup should not need this.">
<FormField label="i18n:govoplan-files.metadata_json.b0e4c283">
<FormField label="i18n:govoplan-files.metadata_json.b0e4c283" documentation={CONNECTOR_RECOVERY_DOCUMENTATION}>
<textarea value={draft.metadataJson} disabled={saving} onChange={(event) => patchDraft({ metadataJson: event.target.value })} rows={5} spellCheck={false} />
</FormField>
</AdvancedOptionsPanel>
@@ -1340,6 +1504,13 @@ function targetPluralLabel(scopeType: ConnectorScope, fallback: string): string
return fallback;
}
function connectorMutationBlocker(canWrite: boolean, readOnly: boolean, saving: boolean): string {
if (!canWrite) return "Connector administration permission is required for this action.";
if (readOnly) return "This entry is deployment-managed. Change its bootstrap configuration and restart the service.";
if (saving) return "Wait for the current connector change to finish.";
return "";
}
function emptyDraft(scopeType: ConnectorScope): ConnectorProfileDraft {
return {
id: `${scopeType}-webdav`,
@@ -1432,10 +1603,39 @@ function connectorCredentialDraftKey(draft: ConnectorCredentialDraft): string {
function credentialOptionsForProfileDraft(credentials: FileConnectorCredential[], draft: ConnectorProfileDraft): FileConnectorCredential[] {
return credentials.filter((credential) =>
credential.id === draft.credentialProfileId ||
credential.enabled && (!credential.provider || credential.provider === draft.provider)
credential.enabled &&
(!credential.provider || credential.provider === draft.provider) &&
reusableCredentialAllowsProfile(credential, draft.id)
);
}
function reusableCredentialAvailableAtScope(
credential: FileConnectorCredential,
scopeType: ConnectorScope,
scopeId: string | null
): boolean {
if (connectorBelongsToEditableScope(credential, scopeType, scopeId)) return true;
if (credential.source_kind !== "credential_envelope") return false;
const inherited = credential.metadata?.inherit_to_lower_scopes === true;
if (!inherited) return false;
if (credential.scope_type === "system") return scopeType !== "system";
if (credential.scope_type === "tenant") {
return scopeType === "group" || scopeType === "user";
}
return false;
}
function reusableCredentialAllowsProfile(
credential: FileConnectorCredential,
profileId: string
): boolean {
if (credential.source_kind !== "credential_envelope") return true;
const refs = Array.isArray(credential.metadata?.allowed_server_refs)
? credential.metadata.allowed_server_refs.filter((value): value is string => typeof value === "string")
: [];
return refs.length === 0 || refs.includes(`files:${profileId}`);
}
function credentialProfileIdForDraft(draft: ConnectorProfileDraft, credentials: FileConnectorCredential[]): string {
const options = credentialOptionsForProfileDraft(credentials, draft);
if (draft.credentialProfileId && options.some((credential) => credential.id === draft.credentialProfileId)) {
@@ -1656,6 +1856,26 @@ function lines(value: string): string[] {
return value.split(/\r?\n/).map((line) => line.trim()).filter(Boolean);
}
function policyReferenceConflicts(
draft: CentralConnectorPolicyDraft
): string[] {
return [
["connections", draft.allowedConnectors, draft.deniedConnectors],
["credentials", draft.allowedCredentials, draft.deniedCredentials],
["providers", draft.allowedProviders, draft.deniedProviders],
["paths", draft.allowedPaths, draft.deniedPaths],
["endpoint URLs", draft.allowedUrls, draft.deniedUrls]
].flatMap(([label, allowed, denied]) => {
const deniedValues = new Set(
lines(denied).map((value) => value.toLocaleLowerCase())
);
const overlaps = lines(allowed).filter((value) =>
deniedValues.has(value.toLocaleLowerCase())
);
return overlaps.length ? [`${label}: ${overlaps.join(", ")}`] : [];
});
}
function policyRuleList(policy: Record<string, unknown>, kind: "allow" | "deny", field: "connectors" | "providers" | "credentials" | "external_paths" | "external_urls"): string[] {
const rule = recordValue(policy[kind]) ??
recordValue(policy[kind === "allow" ? "allowlist" : "denylist"]) ??
@@ -0,0 +1,527 @@
import { useEffect, useMemo, useState } from "react";
import {
AdminPageLayout,
adminErrorMessage,
Button,
Card,
ConfirmDialog,
DataGrid,
Dialog,
DismissibleAlert,
DocumentationHelpLink,
FormField,
MetricCard,
StatusBadge,
TableActionGroup,
ToggleSwitch,
type ApiSettings,
type DataGridColumn
} from "@govoplan/core-webui";
import {
CheckCircle2,
Eye,
Play,
Plus,
RefreshCw,
RotateCw,
Trash2
} from "lucide-react";
import {
cleanupFileIntegrityFinding,
createFileIntegrityScan,
listFileIntegrityFindings,
listFileIntegrityScans,
recheckFileIntegrityFinding,
runFileIntegrityScanBatch,
type FileIntegrityActionResult,
type FileIntegrityFinding,
type FileIntegrityScan
} from "../../api/files";
type Props = {
settings: ApiSettings;
canWrite: boolean;
};
const DOCUMENTATION = {
topicId: "files.reference.integrity-recovery-and-fail-closed-transports",
documentationType: "admin" as const
};
export default function FileIntegrityPanel({ settings, canWrite }: Props) {
const [scans, setScans] = useState<FileIntegrityScan[]>([]);
const [selectedScanId, setSelectedScanId] = useState("");
const [findings, setFindings] = useState<FileIntegrityFinding[]>([]);
const [loading, setLoading] = useState(true);
const [busy, setBusy] = useState(false);
const [error, setError] = useState("");
const [success, setSuccess] = useState("");
const [createOpen, setCreateOpen] = useState(false);
const [verifyChecksums, setVerifyChecksums] = useState(true);
const [batchSize, setBatchSize] = useState(100);
const [cleanupPreview, setCleanupPreview] = useState<FileIntegrityActionResult | null>(null);
const selectedScan = scans.find((scan) => scan.id === selectedScanId) ?? null;
useEffect(() => {
void loadScans();
}, [settings.accessToken, settings.apiBaseUrl, settings.apiKey]);
useEffect(() => {
if (!selectedScanId) {
setFindings([]);
return;
}
void loadFindings(selectedScanId);
}, [selectedScanId, settings.accessToken, settings.apiBaseUrl, settings.apiKey]);
async function loadScans() {
setLoading(true);
setError("");
try {
const loaded = await listFileIntegrityScans(settings);
setScans(loaded);
setSelectedScanId((current) => (
loaded.some((scan) => scan.id === current) ? current : loaded[0]?.id ?? ""
));
} catch (err) {
setError(adminErrorMessage(err));
} finally {
setLoading(false);
}
}
async function loadFindings(scanId: string) {
setError("");
try {
setFindings(await listFileIntegrityFindings(settings, scanId));
} catch (err) {
setError(adminErrorMessage(err));
}
}
function replaceScan(next: FileIntegrityScan) {
setScans((current) => [
next,
...current.filter((scan) => scan.id !== next.id)
].sort((left, right) => right.created_at.localeCompare(left.created_at)));
}
function replaceFinding(next: FileIntegrityFinding) {
setFindings((current) => current.map((finding) => (
finding.id === next.id ? next : finding
)));
}
async function createScan() {
setBusy(true);
setError("");
setSuccess("");
try {
const scan = await createFileIntegrityScan(settings, {
verify_checksums: verifyChecksums,
batch_size: batchSize
});
replaceScan(scan);
setSelectedScanId(scan.id);
setCreateOpen(false);
setSuccess("Integrity scan created. Run its first bounded batch when ready.");
} catch (err) {
setError(adminErrorMessage(err));
} finally {
setBusy(false);
}
}
async function runBatch(scan: FileIntegrityScan) {
setBusy(true);
setError("");
setSuccess("");
try {
const next = await runFileIntegrityScanBatch(settings, scan);
replaceScan(next);
setSelectedScanId(next.id);
await loadFindings(next.id);
setSuccess(
next.status === "completed"
? "Integrity scan completed. Review and resolve every finding."
: `Completed one ${next.batch_size}-item batch; the scan can be resumed.`
);
} catch (err) {
setError(adminErrorMessage(err));
await loadScans();
} finally {
setBusy(false);
}
}
async function recheck(finding: FileIntegrityFinding) {
setBusy(true);
setError("");
setSuccess("");
try {
const result = await recheckFileIntegrityFinding(settings, finding);
replaceFinding(result.finding);
setSuccess(
result.inspection_valid
? "The stored object now matches its recorded integrity evidence."
: "The object still fails integrity verification and remains quarantined."
);
} catch (err) {
setError(adminErrorMessage(err));
if (selectedScanId) await loadFindings(selectedScanId);
} finally {
setBusy(false);
}
}
async function previewCleanup(finding: FileIntegrityFinding) {
setBusy(true);
setError("");
setSuccess("");
try {
const preview = await cleanupFileIntegrityFinding(settings, finding, true);
setCleanupPreview(preview);
if (preview.action !== "would_delete") {
replaceFinding(preview.finding);
setSuccess(cleanupActionMessage(preview.action));
}
} catch (err) {
setError(adminErrorMessage(err));
if (selectedScanId) await loadFindings(selectedScanId);
} finally {
setBusy(false);
}
}
async function confirmCleanup() {
if (!cleanupPreview) return;
setBusy(true);
setError("");
setSuccess("");
try {
const result = await cleanupFileIntegrityFinding(
settings,
cleanupPreview.finding,
false
);
replaceFinding(result.finding);
setCleanupPreview(null);
setSuccess(cleanupActionMessage(result.action));
} catch (err) {
setCleanupPreview(null);
setError(adminErrorMessage(err));
if (selectedScanId) await loadFindings(selectedScanId);
} finally {
setBusy(false);
}
}
const scanColumns = useMemo<DataGridColumn<FileIntegrityScan>[]>(() => [
{
id: "created",
header: "Created",
width: 180,
minWidth: 150,
sortable: true,
value: (scan) => scan.created_at,
render: (scan) => formatDateTime(scan.created_at)
},
{
id: "status",
header: "Status",
width: 130,
minWidth: 110,
sortable: true,
filterable: true,
filterType: "list",
value: (scan) => scan.status,
render: (scan) => <StatusBadge status={statusTone(scan.status)} label={scan.status} />
},
{
id: "phase",
header: "Phase",
width: 110,
minWidth: 90,
value: (scan) => scan.phase
},
{
id: "progress",
header: "Progress",
minWidth: 250,
resizable: true,
value: (scan) => `${scan.scanned_blob_count} blobs, ${scan.scanned_object_count} objects`,
render: (scan) => (
<span>
{scan.scanned_blob_count} blobs ({scan.verified_blob_count} verified, {scan.quarantined_blob_count} quarantined), {" "}
{scan.scanned_object_count} objects ({scan.orphan_object_count} orphaned)
</span>
)
},
{
id: "backend",
header: "Storage",
width: 120,
minWidth: 100,
value: (scan) => scan.storage_backend
},
{
id: "actions",
header: "Actions",
width: 105,
minWidth: 105,
sticky: "end",
align: "right",
render: (scan) => (
<TableActionGroup
minimumSlots={2}
actions={[
{
id: "inspect",
label: "Inspect findings",
icon: <Eye size={16} />,
onClick: () => setSelectedScanId(scan.id)
},
{
id: "run",
label: scan.status === "failed" ? "Resume next batch" : "Run next batch",
icon: <Play size={16} />,
onClick: () => void runBatch(scan),
disabled: busy || ["completed", "cancelled"].includes(scan.status),
disabledReason: ["completed", "cancelled"].includes(scan.status)
? "This scan is complete."
: undefined
}
]}
/>
)
}
], [busy]);
const findingColumns = useMemo<DataGridColumn<FileIntegrityFinding>[]>(() => [
{
id: "kind",
header: "Finding",
width: 180,
minWidth: 145,
sortable: true,
filterable: true,
value: (finding) => finding.kind.replaceAll("_", " ")
},
{
id: "state",
header: "State",
width: 115,
minWidth: 95,
sortable: true,
filterable: true,
value: (finding) => finding.state,
render: (finding) => <StatusBadge status={statusTone(finding.state)} label={finding.state} />
},
{
id: "object",
header: "Affected object",
minWidth: 260,
resizable: true,
filterable: true,
value: (finding) => finding.storage_key,
render: (finding) => (
<span title={finding.storage_key}>
{finding.blob_id ? `Blob ${finding.blob_id}` : finding.storage_key}
</span>
)
},
{
id: "evidence",
header: "Expected / observed",
minWidth: 230,
resizable: true,
value: (finding) => evidenceLabel(finding)
},
{
id: "updated",
header: "Updated",
width: 175,
minWidth: 145,
sortable: true,
value: (finding) => finding.updated_at,
render: (finding) => formatDateTime(finding.updated_at)
},
{
id: "actions",
header: "Actions",
width: 70,
minWidth: 70,
sticky: "end",
align: "right",
render: (finding) => (
<TableActionGroup
minimumSlots={1}
actions={finding.kind === "orphan_object"
? [{
id: "cleanup",
label: "Preview safe cleanup",
icon: <Trash2 size={16} />,
variant: "danger",
onClick: () => void previewCleanup(finding),
disabled: busy || finding.state === "deleted",
disabledReason: finding.state === "deleted" ? "The object is already absent." : undefined
}]
: [{
id: "recheck",
label: "Recheck stored object",
icon: <RotateCw size={16} />,
onClick: () => void recheck(finding),
disabled: busy
}]}
/>
)
}
], [busy]);
return (
<>
<AdminPageLayout
title="File integrity"
description="Run bounded storage reconciliation and resolve quarantined or unreferenced objects without acting on stale operator state."
loading={loading}
error={error}
success={success}
actions={(
<>
<Button
title="Reload scans and findings"
aria-label="Reload scans and findings"
onClick={() => void loadScans()}
disabled={loading || busy}
>
<RefreshCw size={16} />
</Button>
<Button variant="primary" onClick={() => setCreateOpen(true)} disabled={!canWrite || busy}>
<Plus size={16} /> New scan
</Button>
<DocumentationHelpLink reference={DOCUMENTATION} label="Open Files integrity documentation" />
</>
)}
>
<DismissibleAlert tone="info" dismissible={false} compact>
Cleanup is available only for objects with no managed database reference and always starts with a dry-run preview. Files does not yet implement legal-hold or hard-purge policy; such objects must remain outside cleanup until those controls exist.
</DismissibleAlert>
<Card title="Integrity scans">
<div className="admin-table-surface">
<DataGrid
id="files-integrity-scans-v1"
rows={scans}
columns={scanColumns}
getRowKey={(scan) => scan.id}
initialFit="container"
rowClassName={(scan) => scan.id === selectedScanId ? "is-selected" : undefined}
emptyText="No integrity scans have been created."
/>
</div>
</Card>
{selectedScan && (
<>
<div className="metric-grid">
<MetricCard label="Verified blobs" value={selectedScan.verified_blob_count} tone="good" />
<MetricCard label="Quarantined blobs" value={selectedScan.quarantined_blob_count} tone={selectedScan.quarantined_blob_count ? "danger" : "neutral"} />
<MetricCard label="Orphan objects" value={selectedScan.orphan_object_count} tone={selectedScan.orphan_object_count ? "warning" : "neutral"} />
<MetricCard label="Open findings" value={findings.filter((finding) => finding.state === "open").length} tone={findings.some((finding) => finding.state === "open") ? "warning" : "good"} />
</div>
{selectedScan.last_error && (
<DismissibleAlert tone="warning" dismissible={false} compact>
The last batch failed with {selectedScan.last_error}. Verify storage availability, then resume from the recorded cursor.
</DismissibleAlert>
)}
<Card title="Findings">
<div className="admin-table-surface">
<DataGrid
id={`files-integrity-findings-${selectedScan.id}`}
rows={findings}
columns={findingColumns}
getRowKey={(finding) => finding.id}
initialFit="container"
emptyText={selectedScan.status === "completed" ? "No findings were recorded." : "No findings in completed batches yet."}
/>
</div>
</Card>
</>
)}
</AdminPageLayout>
<Dialog
open={createOpen}
title="Create integrity scan"
onClose={() => setCreateOpen(false)}
closeDisabled={busy}
footer={(
<>
<Button onClick={() => setCreateOpen(false)} disabled={busy}>Cancel</Button>
<Button variant="primary" onClick={() => void createScan()} disabled={busy}>
<CheckCircle2 size={16} /> {busy ? "Creating..." : "Create scan"}
</Button>
</>
)}
>
<div className="settings-list">
<ToggleSwitch
label="Verify SHA-256 checksums"
checked={verifyChecksums}
onChange={setVerifyChecksums}
disabled={busy}
help="Checksum verification reads every selected object; disabling it verifies existence and size only."
/>
<FormField label="Items per batch" documentation={DOCUMENTATION}>
<input
type="number"
min={1}
max={1000}
value={batchSize}
onChange={(event) => setBatchSize(Math.max(1, Math.min(1000, Number(event.target.value) || 1)))}
disabled={busy}
/>
</FormField>
</div>
</Dialog>
<ConfirmDialog
open={cleanupPreview?.action === "would_delete"}
title="Delete unreferenced storage object?"
message={cleanupPreview ? `The dry run confirmed that ${cleanupPreview.finding.storage_key} has no managed Files reference. Deletion cannot be undone from GovOPlaN.` : ""}
confirmLabel="Delete object"
tone="danger"
busy={busy}
onConfirm={() => void confirmCleanup()}
onCancel={() => setCleanupPreview(null)}
/>
</>
);
}
function statusTone(value: string): string {
if (["completed", "resolved", "verified"].includes(value)) return "success";
if (["failed", "deleted", "missing", "checksum_mismatch", "size_mismatch"].includes(value)) return "danger";
if (["running", "pending", "open"].includes(value)) return "warning";
return "neutral";
}
function evidenceLabel(finding: FileIntegrityFinding): string {
const expected = finding.expected_size_bytes == null ? "-" : `${finding.expected_size_bytes} B`;
const observed = finding.observed_size_bytes == null ? "-" : `${finding.observed_size_bytes} B`;
return `${expected} / ${observed}`;
}
function cleanupActionMessage(action: string): string {
if (action === "retained_referenced") return "Cleanup was blocked because the object has a managed database reference.";
if (action === "already_deleted" || action === "already_absent") return "The object was already absent; the finding is reconciled.";
if (action === "deleted") return "The unreferenced storage object was deleted and recovery evidence was recorded.";
return `Integrity cleanup result: ${action.replaceAll("_", " ")}.`;
}
function formatDateTime(value: string | null | undefined): string {
if (!value) return "-";
const parsed = new Date(value);
return Number.isNaN(parsed.getTime()) ? value : parsed.toLocaleString();
}
@@ -0,0 +1,98 @@
import { useCallback } from "react";
import { Folder, Network } from "lucide-react";
import { Link } from "react-router";
import {
DashboardWidgetList,
DismissibleAlert,
LoadingFrame,
StatusBadge,
useDashboardWidgetData,
type ApiSettings,
type DashboardWidgetConfiguration
} from "@govoplan/core-webui";
import {
listFileSpaces,
type FileSpace
} from "../../api/files";
export default function FileSpacesWidget({
settings,
refreshKey,
configuration
}: {
settings: ApiSettings;
refreshKey: number;
configuration: DashboardWidgetConfiguration;
}) {
const maxItems = numberSetting(configuration.maxItems, 6, 1, 16);
const includeConnectors = configuration.includeConnectors !== false;
const load = useCallback(async () => {
const response = await listFileSpaces(settings);
return response.spaces
.filter(
(space) =>
includeConnectors || (space.space_type ?? "managed") === "managed"
)
.sort((left, right) => left.label.localeCompare(right.label))
.slice(0, maxItems);
}, [includeConnectors, maxItems, settings]);
const { data: spaces, loading, error } = useDashboardWidgetData(
load,
refreshKey
);
return (
<LoadingFrame loading={loading} label="Loading file spaces">
{error && (
<DismissibleAlert tone="warning" resetKey={error}>
{error}
</DismissibleAlert>
)}
<DashboardWidgetList
emptyText="No file spaces are available."
items={(spaces ?? []).map((space) => ({
id: space.id,
title: space.label,
detail: spaceDescription(space),
meta: space.owner_type === "group" ? "Group" : "Personal",
leading:
space.space_type === "connector" ? (
<Network size={17} aria-hidden="true" />
) : (
<Folder size={17} aria-hidden="true" />
),
trailing: space.read_only ? (
<StatusBadge status="locked" label="Read only" />
) : undefined,
to: "/files"
}))}
/>
<div className="dashboard-contribution-footer">
<Link className="btn btn-secondary" to="/files">
Open files
</Link>
</div>
</LoadingFrame>
);
}
function spaceDescription(space: FileSpace): string {
if (space.space_type !== "connector") {
return space.description || "Managed storage";
}
return [space.provider, space.library_id, space.remote_path]
.filter(Boolean)
.join(" · ") || "Connected storage";
}
function numberSetting(
value: unknown,
fallback: number,
minimum: number,
maximum: number
): number {
const numeric = typeof value === "number" ? value : Number(value);
return Number.isFinite(numeric)
? Math.max(minimum, Math.min(maximum, Math.floor(numeric)))
: fallback;
}
+525 -31
View File
@@ -1,13 +1,15 @@
import { useEffect, useMemo, useRef, useState, type DragEvent as ReactDragEvent, type KeyboardEvent as ReactKeyboardEvent, type MouseEvent as ReactMouseEvent } from "react";
import { ArrowUp, ChevronRight, Copy, Download, File, Folder, Home, KeyRound, Link2, MoveRight, Plus, RefreshCw, Search, Trash2, UploadCloud } from "lucide-react";
import { ArrowUp, ChevronRight, Copy, Download, File, Folder, Home, KeyRound, Link2, ListFilter, MoveRight, Plus, RefreshCw, Search, Share2, Trash2, UploadCloud } from "lucide-react";
import {
Button,
ConfirmDialog,
DocumentationHelpLink,
DismissibleAlert,
FieldLabel,
FileDropZone,
FormField,
LoadingIndicator,
PasswordField,
ResourceAccessExplanation,
ToggleSwitch,
hasScope,
@@ -20,24 +22,30 @@ import {
browseFileConnectorProfile,
createFileConnectorSpace,
createFolder,
confirmArchiveUpload,
deleteFolder,
downloadFile,
downloadFilesAsZip,
fetchResourceAccessExplanation,
listFilesDelta,
listFilesByProperties,
listFileConnectorProfiles,
listFileSpaces,
listManagedFileSnapshot,
previewArchiveUpload,
resolveFilePatterns,
syncFileConnectorFile,
transferFiles,
uploadFiles,
virtualFolderResourceId,
type ArchivePreviewEntry,
type ArchivePreviewResponse,
type ConflictResolution,
type ConflictStrategy,
type FileConnectorBrowseItem,
type FileConnectorProfile,
type FileDeltaResponse,
type FileCampaignUsageFilter,
type FileFolder,
type FileSpace,
type ManagedFile,
@@ -46,6 +54,7 @@ import {
"../../api/files";
import { EMPTY_FILES, EMPTY_FOLDERS, EMPTY_SPACES, INTERNAL_DRAG_TYPE } from "./constants";
import { FileConflictDialog, FileContextMenu, FileDialog, FolderTree, RenamePreviewList, TransferFolderSelector } from "./components/FileManagerComponents";
import FileShareDialog from "./components/FileShareDialog";
import type {
ContextMenuState,
DialogKind,
@@ -88,6 +97,7 @@ import { useFileDialogs } from "./hooks/useFileDialogs";
import { useFileDragDropState } from "./hooks/useFileDragDropState";
type UploadPhase = "idle" | "uploading" | "unpacking" | "finalizing";
type AuditRelevantFilter = "" | "true" | "false";
type FileAccessExplanationTarget = {
resourceType: "file" | "folder";
resourceId: string;
@@ -97,12 +107,18 @@ type FileAccessExplanationTarget = {
const DEFAULT_FILE_LIST_ROW_HEIGHT = 58;
const FILE_LIST_OVERSCAN_ROWS = 8;
const ARCHIVE_FILENAME_PATTERN = /\.(?:zip|tar|tar\.gz|tgz|tar\.bz2|tbz2|tar\.xz|txz)$/i;
const FILES_WORKFLOW_DOCUMENTATION = {
topicId: "files.workflow.organize-managed-files",
documentationType: "user"
} as const;
export default function FilesPage({ settings, auth }: {settings: ApiSettings;auth: AuthInfo;}) {
const canDownload = hasScope(auth, "files:download");
const canUpload = hasScope(auth, "files:upload");
const canOrganize = hasScope(auth, "files:organize");
const canDelete = hasScope(auth, "files:delete");
const canShare = hasScope(auth, "files:file:share");
const [spaces, setSpaces] = useState<FileSpace[]>(EMPTY_SPACES);
const [activeSpaceId, setActiveSpaceId] = useState("");
const activeSpace = spaces.find((space) => space.id === activeSpaceId) ?? spaces[0] ?? null;
@@ -118,10 +134,19 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut
const [searchCaseSensitive, setSearchCaseSensitive] = useState(false);
const [searchActive, setSearchActive] = useState(false);
const [searchResults, setSearchResults] = useState<ManagedFile[] | null>(null);
const [campaignUsageFilter, setCampaignUsageFilter] = useState<"" | FileCampaignUsageFilter>("");
const [auditRelevantFilter, setAuditRelevantFilter] = useState<AuditRelevantFilter>("");
const [propertyFiltersActive, setPropertyFiltersActive] = useState(false);
const [propertyFilterResults, setPropertyFilterResults] = useState<ManagedFile[] | null>(null);
const [propertyFilterTotal, setPropertyFilterTotal] = useState<number | null>(null);
const [sortColumn, setSortColumn] = useState<SortColumn>("name");
const [sortDirection, setSortDirection] = useState<SortDirection>("asc");
const [unmatchedCount, setUnmatchedCount] = useState<number | null>(null);
const [unpackZip, setUnpackZip] = useState(false);
const [archiveFile, setArchiveFile] = useState<File | null>(null);
const [archivePreview, setArchivePreview] = useState<ArchivePreviewResponse | null>(null);
const [archivePassword, setArchivePassword] = useState("");
const [selectedArchivePaths, setSelectedArchivePaths] = useState<Set<string>>(new Set());
const [busy, setBusy] = useState(false);
const [uploadActive, setUploadActive] = useState(false);
const [uploadPhase, setUploadPhase] = useState<UploadPhase>("idle");
@@ -146,6 +171,7 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut
const [accessExplanationTarget, setAccessExplanationTarget] = useState<FileAccessExplanationTarget | null>(null);
const [resourceAccessExplanation, setResourceAccessExplanation] = useState<ResourceAccessExplanationResponse | null>(null);
const [resourceAccessLoading, setResourceAccessLoading] = useState(false);
const [shareDialogFile, setShareDialogFile] = useState<ManagedFile | null>(null);
const { setDragActive, internalDrag, setInternalDrag, dropTargetKey, setDropTargetKey, clearDropState: clearDragDropState } = useFileDragDropState();
const {
dialog,
@@ -176,11 +202,19 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut
const [renameSuffix, setRenameSuffix] = useState("");
const [renameRecursive, setRenameRecursive] = useState(false);
const [renamePreview, setRenamePreview] = useState<RenameResponse | null>(null);
const visibleFiles = searchActive && searchResults ? searchResults : files;
const visibleFiles = useMemo(() => {
if (searchActive && searchResults && propertyFiltersActive && propertyFilterResults) {
const ids = new Set(propertyFilterResults.map((file) => file.id));
return searchResults.filter((file) => ids.has(file.id));
}
if (propertyFiltersActive && propertyFilterResults) return propertyFilterResults;
if (searchActive && searchResults) return searchResults;
return files;
}, [files, propertyFilterResults, propertyFiltersActive, searchActive, searchResults]);
const explorerEntries = useMemo(() => {
const entries = buildExplorerEntries(visibleFiles, folders, currentFolder, searchActive);
const entries = buildExplorerEntries(visibleFiles, folders, currentFolder, searchActive || propertyFiltersActive);
return sortExplorerEntries(entries, sortColumn, sortDirection);
}, [visibleFiles, folders, currentFolder, searchActive, sortColumn, sortDirection]);
}, [visibleFiles, folders, currentFolder, searchActive, propertyFiltersActive, sortColumn, sortDirection]);
const fileListViewportRef = useRef<HTMLDivElement | null>(null);
const fileListMeasureRowRef = useRef<HTMLDivElement | null>(null);
const managedSpaceLoadsInFlightRef = useRef<Set<string>>(new Set());
@@ -229,6 +263,14 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut
onOpenFolder: openFolder
});
const selectedFiles = useMemo(() => files.filter((file) => selectedFileIds.has(file.id)), [files, selectedFileIds]);
const shareManageableFile = useMemo(() => {
if (!canShare || selectedFiles.length !== 1 || selectedFolderPaths.size > 0) return null;
const file = selectedFiles[0];
const ownsFile = file.owner_type === "user"
? file.owner_id === auth.user.id
: auth.groups.some((group) => group.id === file.owner_id);
return ownsFile || hasScope(auth, "files:file:admin") ? file : null;
}, [auth, canShare, selectedFiles, selectedFolderPaths]);
const selectedDownloadFileIds = useMemo(() => fileIdsForSelection(files, selectedFileIds, selectedFolderPaths), [files, selectedFileIds, selectedFolderPaths]);
const accessExplainableTarget = useMemo(
() => accessTargetForSelection(selectedFileIds, selectedFolderPaths, activeSpaceId),
@@ -285,7 +327,7 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut
if (!viewport) return;
viewport.scrollTop = 0;
setFileListScrollTop(0);
}, [activeSpaceId, currentFolder, searchActive, searchResults, sortColumn, sortDirection]);
}, [activeSpaceId, currentFolder, propertyFiltersActive, propertyFilterResults, searchActive, searchResults, sortColumn, sortDirection]);
async function loadSpaces() {
setBusy(true);
@@ -373,6 +415,11 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut
setSearchActive(false);
setSearchPattern("");
setSearchResults(null);
setCampaignUsageFilter("");
setAuditRelevantFilter("");
setPropertyFiltersActive(false);
setPropertyFilterResults(null);
setPropertyFilterTotal(null);
}
} catch (err) {
setError(err instanceof Error ? err.message : String(err));
@@ -410,6 +457,11 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut
setSearchPattern("");
setSearchActive(false);
setSearchResults(null);
setCampaignUsageFilter("");
setAuditRelevantFilter("");
setPropertyFiltersActive(false);
setPropertyFilterResults(null);
setPropertyFilterTotal(null);
setSelectedFileIds(new Set());
setSelectedFolderPaths(new Set());
setConnectorSpaceSelectedItem(null);
@@ -431,6 +483,11 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut
setSearchActive(false);
setSearchPattern("");
setSearchResults(null);
setCampaignUsageFilter("");
setAuditRelevantFilter("");
setPropertyFiltersActive(false);
setPropertyFilterResults(null);
setPropertyFilterTotal(null);
}
setUnmatchedCount(null);
setRenamePreview(null);
@@ -459,9 +516,19 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut
setNewFolderName("");
setNewFolderError("");
}
if (kind === "upload") {
resetArchiveUploadState();
}
openRawDialog(kind, target);
}
function resetArchiveUploadState() {
setArchiveFile(null);
setArchivePreview(null);
setArchivePassword("");
setSelectedArchivePaths(new Set());
}
function closeDialog() {
closeRawDialog();
setNewFolderError("");
@@ -471,6 +538,7 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut
setConnectorError("");
setConnectorSelectedItem(null);
setConnectorSpaceLabel("");
resetArchiveUploadState();
}
function updateActiveDialogFolder(spaceId: string, folderPath: string) {
@@ -625,6 +693,156 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut
async function loadArchivePreview(
file: File,
target: FileActionTarget,
options: { preserveSelection?: boolean } = {}
) {
const targetSpace = findSpace(target.spaceId);
if (!targetSpace || isConnectorSpace(targetSpace)) {
setError(uploadRejectedReason(target));
return;
}
setArchiveFile(file);
setBusy(true);
setUploadActive(true);
setUploadPhase("uploading");
setUploadProgress(null);
setError("");
setMessage("");
try {
const response = await previewArchiveUpload(settings, file, {
owner_type: targetSpace.owner_type,
owner_id: targetSpace.owner_id,
path: target.folderPath,
password: archivePassword || undefined
});
const availableFiles = new Set(
response.entries
.filter((entry) => entry.kind === "file")
.map((entry) => entry.path)
);
setArchivePreview(response);
setSelectedArchivePaths((current) => {
if (!options.preserveSelection) return availableFiles;
const retained = new Set(
Array.from(current).filter((path) => availableFiles.has(path))
);
return retained.size > 0 ? retained : availableFiles;
});
} catch (err) {
setArchivePreview(null);
setSelectedArchivePaths(new Set());
setError(err instanceof Error ? err.message : String(err));
} finally {
setBusy(false);
setUploadActive(false);
setUploadPhase("idle");
setUploadProgress(null);
}
}
function archiveFilesForEntry(entry: ArchivePreviewEntry): string[] {
if (!archivePreview) return [];
if (entry.kind === "file") return [entry.path];
const prefix = `${entry.path}/`;
return archivePreview.entries
.filter((candidate) => candidate.kind === "file" && candidate.path.startsWith(prefix))
.map((candidate) => candidate.path);
}
function archiveEntryIsSelected(entry: ArchivePreviewEntry): boolean {
const paths = archiveFilesForEntry(entry);
return paths.length > 0 && paths.every((path) => selectedArchivePaths.has(path));
}
function toggleArchiveEntry(entry: ArchivePreviewEntry, selected: boolean) {
const paths = archiveFilesForEntry(entry);
setSelectedArchivePaths((current) => {
const next = new Set(current);
paths.forEach((path) => selected ? next.add(path) : next.delete(path));
return next;
});
}
function toggleAllArchiveFiles(selected: boolean) {
setSelectedArchivePaths(
selected && archivePreview
? new Set(
archivePreview.entries
.filter((entry) => entry.kind === "file")
.map((entry) => entry.path)
)
: new Set()
);
}
async function confirmCurrentArchive() {
const target = currentActionTarget();
const targetSpace = target ? findSpace(target.spaceId) : null;
if (
busy
|| !archiveFile
|| !archivePreview
|| !target
|| !targetSpace
|| isConnectorSpace(targetSpace)
) {
return;
}
if (selectedArchivePaths.size === 0) {
setError("Select at least one archive file to import.");
return;
}
if (archivePreview.requires_password && !archivePassword) {
setError("Enter the archive password before importing.");
return;
}
setBusy(true);
setUploadActive(true);
setUploadPhase("uploading");
setUploadProgress(0);
setError("");
setMessage("Uploading the archive for confirmed extraction.");
try {
const response = await confirmArchiveUpload(settings, archiveFile, {
preview_token: archivePreview.preview_token,
selected_paths: Array.from(selectedArchivePaths),
owner_type: targetSpace.owner_type,
owner_id: targetSpace.owner_id,
path: target.folderPath,
password: archivePassword || undefined,
conflict_strategy: "reject",
onProgress: ({ percentage }) => {
setUploadProgress(percentage);
if (percentage !== null && percentage >= 100) {
setUploadPhase("unpacking");
setMessage("Extracting the selected archive files.");
}
}
});
setUploadPhase("finalizing");
setUploadProgress(100);
const uploadedCount = response.files.length;
const destination = target.folderPath || "i18n:govoplan-files.root.e96857c5";
closeDialog();
setMessage(i18nMessage("i18n:govoplan-files.uploaded_value_file_s_into_value.d0fa052b", {
value0: uploadedCount,
value1: destination
}));
resetTransientState();
await loadSpaceContents(targetSpace, { silent: targetSpace.id !== activeSpaceId });
} catch (err) {
setError(err instanceof Error ? err.message : String(err));
setMessage("");
} finally {
setBusy(false);
setUploadActive(false);
setUploadPhase("idle");
setUploadProgress(null);
}
}
async function handleFilesUpload(fileList: FileList | File[], options: {conflictStrategy?: ConflictStrategy;conflictResolutions?: ConflictResolution[];bypassConflictDialog?: boolean;target?: FileActionTarget;} = {}) {
const target = options.target ?? currentActionTarget();
const targetSpace = target ? findSpace(target.spaceId) : null;
@@ -634,6 +852,14 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut
}
const selected = Array.from(fileList);
if (selected.length === 0) return;
if (unpackZip) {
if (selected.length !== 1 || !ARCHIVE_FILENAME_PATTERN.test(selected[0].name)) {
setError("Choose one ZIP, TAR, TAR.GZ, TAR.BZ2, or TAR.XZ archive to preview and unpack.");
return;
}
await loadArchivePreview(selected[0], target);
return;
}
if (!options.bypassConflictDialog && !unpackZip) {
const conflicts = uploadConflicts(selected, target);
if (conflicts.length > 0) {
@@ -649,7 +875,6 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut
return;
}
}
const uploadsZipArchive = unpackZip && selected.some((file) => file.name.toLowerCase().endsWith(".zip"));
setBusy(true);
setUploadActive(true);
setUploadPhase("uploading");
@@ -661,14 +886,13 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut
owner_type: targetSpace.owner_type,
owner_id: targetSpace.owner_id,
path: target.folderPath,
unpack_zip: unpackZip,
conflict_strategy: options.conflictStrategy ?? "reject",
conflict_resolutions: options.conflictResolutions,
onProgress: ({ percentage }) => {
setUploadProgress(percentage);
if (percentage !== null && percentage >= 100) {
setUploadPhase(uploadsZipArchive ? "unpacking" : "finalizing");
setMessage(uploadsZipArchive ? "i18n:govoplan-files.unpacking_zip_upload.35019691" : "i18n:govoplan-files.finalizing_upload.bcce936d");
setUploadPhase("finalizing");
setMessage("i18n:govoplan-files.finalizing_upload.bcce936d");
}
}
});
@@ -1001,6 +1225,50 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut
setSelectedFolderPaths(new Set());
}
async function runPropertyFilters(
campaignUsage: "" | FileCampaignUsageFilter,
auditRelevant: AuditRelevantFilter
) {
if (!activeSpace || activeSpaceIsConnector) return;
if (!campaignUsage && !auditRelevant) {
clearPropertyFilters();
return;
}
setBusy(true);
setError("");
setMessage("");
try {
const response = await listFilesByProperties(settings, {
owner_type: activeSpace.owner_type,
owner_id: activeSpace.owner_id,
path_prefix: currentFolder,
campaign_usage: campaignUsage || undefined,
audit_relevant: auditRelevant ? auditRelevant === "true" : undefined
});
setCampaignUsageFilter(campaignUsage);
setAuditRelevantFilter(auditRelevant);
setPropertyFilterResults(response.files);
setPropertyFilterTotal(response.total);
setPropertyFiltersActive(true);
setSelectedFileIds(new Set());
setSelectedFolderPaths(new Set());
} catch (err) {
setError(err instanceof Error ? err.message : String(err));
} finally {
setBusy(false);
}
}
function clearPropertyFilters() {
setCampaignUsageFilter("");
setAuditRelevantFilter("");
setPropertyFiltersActive(false);
setPropertyFilterResults(null);
setPropertyFilterTotal(null);
setSelectedFileIds(new Set());
setSelectedFolderPaths(new Set());
}
function deleteSelected(
fileIds: Set<string> = selectedFileIds,
folderPaths: Set<string> = selectedFolderPaths,
@@ -1808,30 +2076,45 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut
message === "i18n:govoplan-files.unpacking_zip_upload.35019691" ||
message === "i18n:govoplan-files.finalizing_upload.bcce936d" ? "info" : "success";
const managedSpaceBlocker = !activeSpace ? "Select a file space first." :
activeSpaceIsConnector ? "This action writes managed storage and is unavailable in a read-only connector space." : "";
const workingBlocker = busy ? "Wait for the current file operation to finish." : "";
const uploadBlocker = workingBlocker || managedSpaceBlocker || (!canUpload ? "File upload permission is required." : "");
const organizeBlocker = workingBlocker || managedSpaceBlocker || (!canOrganize ? "File organization permission is required." : "");
const selectionBlocker = !hasSelection ? "Select at least one file or folder first." : "";
const downloadBlocker = workingBlocker || managedSpaceBlocker || (!canDownload ? "File download permission is required." : "") || (selectedDownloadFileIds.length === 0 ? "Select at least one downloadable file first." : "");
const shareBlocker = workingBlocker || managedSpaceBlocker || (!canShare ? "File sharing permission is required." : "") || (!shareManageableFile ? "Select one manageable file first." : "");
const accessExplanationBlocker = workingBlocker || (activeSpaceIsConnector ? "Access explanations apply to managed files and folders." : "") || (!canExplainResourceAccess ? "Permission to inspect resource access is required." : "") || (!accessExplainableTarget ? "Select one file or folder first." : "");
const deleteBlocker = workingBlocker || managedSpaceBlocker || (!canDelete ? "File deletion permission is required." : "") || selectionBlocker;
const syncBlocker = workingBlocker || (!activeSpace ? "Select a file space first." : "") || (!canUpload ? "File upload permission is required to import synchronized content." : "") || (activeSpaceIsConnector && connectorSpaceSelectedItem?.kind !== "file" ? "Select one remote file to synchronize." : "");
const toolbar =
<div className="file-manager-toolbar" aria-label="i18n:govoplan-files.file_actions.9e1b94c5">
<Button variant="primary" onClick={() => openDialog("upload", toolbarTarget())} disabled={busy || !activeSpace || activeSpaceIsConnector || !canUpload}><UploadCloud size={16} aria-hidden="true" /> i18n:govoplan-files.upload.8bdf057f</Button>
<Button variant="primary" onClick={() => openDialog("upload", toolbarTarget())} disabled={Boolean(uploadBlocker)} disabledReason={uploadBlocker}><UploadCloud size={16} aria-hidden="true" /> i18n:govoplan-files.upload.8bdf057f</Button>
<Button
onClick={() => activeSpaceIsConnector ? void syncConnectorSpaceSelection() : void openConnectorSyncDialog(toolbarTarget())}
disabled={busy || !activeSpace || !canUpload || activeSpaceIsConnector && connectorSpaceSelectedItem?.kind !== "file"}>
disabled={Boolean(syncBlocker)}
disabledReason={syncBlocker}>
<RefreshCw size={16} aria-hidden="true" /> i18n:govoplan-files.sync.905f6309
</Button>
<Button onClick={() => void openConnectorSpaceDialog()} disabled={busy || !canOrganize}>
<Button onClick={() => void openConnectorSpaceDialog()} disabled={busy || !canOrganize} disabledReason={workingBlocker || (!canOrganize ? "File organization permission is required to add a connector space." : "")}>
<Link2 size={16} aria-hidden="true" /> i18n:govoplan-files.add_space.e4d674d4
</Button>
<Button onClick={() => void downloadSelection()} disabled={busy || activeSpaceIsConnector || !canDownload || selectedDownloadFileIds.length === 0}><Download size={16} aria-hidden="true" /> {downloadLabel}</Button>
<Button onClick={() => openDialog("create-folder", toolbarTarget())} disabled={busy || !activeSpace || activeSpaceIsConnector || !canOrganize}><Plus size={16} aria-hidden="true" /> i18n:govoplan-files.create_folder.97bafaba</Button>
<Button onClick={() => openTransferDialog("move")} disabled={busy || activeSpaceIsConnector || !canOrganize || !hasSelection}><MoveRight size={16} aria-hidden="true" /> i18n:govoplan-files.move.76cdb950</Button>
<Button onClick={() => openTransferDialog("copy")} disabled={busy || activeSpaceIsConnector || !canOrganize || !hasSelection}><Copy size={16} aria-hidden="true" /> i18n:govoplan-files.copy.af74f7c5</Button>
{hasSelection && <Button onClick={openRenameDialog} disabled={busy || activeSpaceIsConnector || !canOrganize}>{selectedEntryCount === 1 ? "i18n:govoplan-files.rename.d3f4cb89" : "i18n:govoplan-files.bulk_rename.7dcaa624"}</Button>}
<Button onClick={() => accessExplainableTarget && void openAccessExplanation(accessExplainableTarget)} disabled={busy || activeSpaceIsConnector || !canExplainResourceAccess || !accessExplainableTarget}><KeyRound size={16} aria-hidden="true" /> i18n:govoplan-files.explain_access.4d5fac37</Button>
<Button variant="danger" onClick={() => void deleteSelected()} disabled={busy || activeSpaceIsConnector || !canDelete || !hasSelection}><Trash2 size={16} aria-hidden="true" /> i18n:govoplan-files.delete.f6fdbe48</Button>
<Button onClick={() => void downloadSelection()} disabled={Boolean(downloadBlocker)} disabledReason={downloadBlocker}><Download size={16} aria-hidden="true" /> {downloadLabel}</Button>
<Button onClick={() => shareManageableFile && setShareDialogFile(shareManageableFile)} disabled={Boolean(shareBlocker)} disabledReason={shareBlocker}><Share2 size={16} aria-hidden="true" /> Manage shares</Button>
<Button onClick={() => openDialog("create-folder", toolbarTarget())} disabled={Boolean(organizeBlocker)} disabledReason={organizeBlocker}><Plus size={16} aria-hidden="true" /> i18n:govoplan-files.create_folder.97bafaba</Button>
<Button onClick={() => openTransferDialog("move")} disabled={Boolean(organizeBlocker || selectionBlocker)} disabledReason={organizeBlocker || selectionBlocker}><MoveRight size={16} aria-hidden="true" /> i18n:govoplan-files.move.76cdb950</Button>
<Button onClick={() => openTransferDialog("copy")} disabled={Boolean(organizeBlocker || selectionBlocker)} disabledReason={organizeBlocker || selectionBlocker}><Copy size={16} aria-hidden="true" /> i18n:govoplan-files.copy.af74f7c5</Button>
{hasSelection && <Button onClick={openRenameDialog} disabled={Boolean(organizeBlocker)} disabledReason={organizeBlocker}>{selectedEntryCount === 1 ? "i18n:govoplan-files.rename.d3f4cb89" : "i18n:govoplan-files.bulk_rename.7dcaa624"}</Button>}
<Button onClick={() => accessExplainableTarget && void openAccessExplanation(accessExplainableTarget)} disabled={Boolean(accessExplanationBlocker)} disabledReason={accessExplanationBlocker}><KeyRound size={16} aria-hidden="true" /> i18n:govoplan-files.explain_access.4d5fac37</Button>
<Button variant="danger" onClick={() => void deleteSelected()} disabled={Boolean(deleteBlocker)} disabledReason={deleteBlocker}><Trash2 size={16} aria-hidden="true" /> i18n:govoplan-files.delete.f6fdbe48</Button>
{activeSpaceIsConnector &&
<Button onClick={() => activeSpace && void loadConnectorSpaceContents(activeSpace)} disabled={busy || connectorSpaceLoading || !activeSpace}>
<Button onClick={() => activeSpace && void loadConnectorSpaceContents(activeSpace)} disabled={busy || connectorSpaceLoading || !activeSpace} disabledReason={workingBlocker || (connectorSpaceLoading ? "This connector space is already refreshing." : !activeSpace ? "Select a connector space first." : "")}>
<RefreshCw size={16} aria-hidden="true" /> i18n:govoplan-files.refresh.56e3badc
</Button>
}
<DocumentationHelpLink reference={FILES_WORKFLOW_DOCUMENTATION} />
</div>;
@@ -2067,6 +2350,15 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut
disabled={!activeSpace || activeSpaceIsConnector}
onSearch={(patternValue, caseSensitiveValue) => void runPatternSearch(patternValue, caseSensitiveValue)}
onClear={() => void clearSearch()} />
<FilePropertyFilterRow
campaignUsage={campaignUsageFilter}
auditRelevant={auditRelevantFilter}
busy={busy}
active={propertyFiltersActive}
disabled={!activeSpace || activeSpaceIsConnector}
total={propertyFilterTotal}
onApply={(campaignUsage, auditRelevant) => void runPropertyFilters(campaignUsage, auditRelevant)}
onClear={clearPropertyFilters} />
<div className="file-list-meta">
<span>{activeSpaceIsConnector ? connectorSpaceSelectedItem?.kind === "file" ? connectorSpaceSelectedItem.name : "i18n:govoplan-files.remote_connector_space.d8956863" : selectedSummary}</span>
@@ -2223,19 +2515,32 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut
</FileDialog>
}
{shareDialogFile &&
<FileShareDialog
settings={settings}
file={shareDialogFile}
tenantId={auth.active_tenant?.id || auth.tenant.id}
onClose={() => setShareDialogFile(null)}
onChanged={() => {
if (activeSpace) void loadSpaceContents(activeSpace, { silent: true });
}} />
}
{dialog === "upload" &&
<FileDialog title={i18nMessage("i18n:govoplan-files.upload_to_value_value.b83a34b4", { value0: activeDialogSpace?.label || "i18n:govoplan-files.files.6ce6c512", value1: activeDialogTarget?.folderPath || "i18n:govoplan-files.root.e96857c5" })} onClose={closeDialog}>
<FileDropZone
disabled={busy || !activeDialogTarget || !activeDialogSpace || isConnectorSpace(activeDialogSpace) || !canUpload}
busy={uploadActive}
progress={visibleUploadProgress}
busyLabel={uploadBusyLabel}
progressLabel={uploadProgressLabel}
note={`Files are uploaded into ${activeDialogTarget?.folderPath || "i18n:govoplan-files.root.e96857c5"}.`}
onRejectedDrop={(reason) => setError(reason === "disabled" ? uploadRejectedReason(activeDialogTarget) : "The browser did not provide readable file data for this drop. Use the file picker, or drag local files from a file manager that exposes file contents to the browser.")}
onFiles={(files) => handleFilesUpload(files, { target: activeDialogTarget || undefined })} />
{!archivePreview &&
<>
<ToggleSwitch
label="Preview and unpack archive"
checked={unpackZip}
onChange={setUnpackZip}
disabled={busy} />
<ToggleSwitch label="i18n:govoplan-files.unpack_zip_uploads.256fead4" checked={unpackZip} onChange={setUnpackZip} disabled={busy} />
{unpackZip &&
<p className="form-help archive-upload-help">
Supports ZIP, TAR, TAR.GZ, TAR.BZ2, and TAR.XZ. Nothing is written until you review and confirm the archive contents.
</p>
}
<div className="field-block">
<FieldLabel className="form-label" help="i18n:govoplan-files.choose_the_destination_folder_before_selecting_o.a4922d75">i18n:govoplan-files.destination_folder.f8ccb63b</FieldLabel>
<TransferFolderSelector
@@ -2246,9 +2551,124 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut
onSelect={(folderPath) => activeDialogTarget && updateActiveDialogFolder(activeDialogTarget.spaceId, folderPath)} />
</div>
<FileDropZone
disabled={busy || !activeDialogTarget || !activeDialogSpace || isConnectorSpace(activeDialogSpace) || !canUpload}
busy={uploadActive}
progress={visibleUploadProgress}
busyLabel={uploadBusyLabel}
progressLabel={uploadProgressLabel}
note={`Files are uploaded into ${activeDialogTarget?.folderPath || "i18n:govoplan-files.root.e96857c5"}.`}
onRejectedDrop={(reason) => setError(reason === "disabled" ? uploadRejectedReason(activeDialogTarget) : "The browser did not provide readable file data for this drop. Use the file picker, or drag local files from a file manager that exposes file contents to the browser.")}
onFiles={(files) => handleFilesUpload(files, { target: activeDialogTarget || undefined })} />
<div className="button-row compact-actions align-end">
<Button onClick={closeDialog} disabled={busy}>i18n:govoplan-files.cancel.77dfd213</Button>
</div>
</>
}
{archivePreview && archiveFile &&
<div className="archive-preview">
<div className="archive-preview-summary">
<div>
<strong>{archiveFile.name}</strong>
<span>{archivePreview.archive_format.toUpperCase()} · {formatBytes(archivePreview.compressed_size_bytes)} compressed</span>
</div>
<div>
<strong>{archivePreview.file_count} files</strong>
<span>{formatBytes(archivePreview.expanded_size_bytes)} expanded · {archivePreview.directory_count} folders</span>
</div>
<div>
<strong>Destination</strong>
<span>{activeDialogTarget?.folderPath || "i18n:govoplan-files.root.e96857c5"}</span>
</div>
</div>
{archivePreview.requires_password &&
<FormField
label="Archive password"
help="The password stays in this dialog and is sent only while inspecting or importing this archive.">
<PasswordField
value={archivePassword}
onValueChange={setArchivePassword}
disabled={busy}
autoComplete="off" />
</FormField>
}
<div className="archive-selection-toolbar">
<label>
<input
type="checkbox"
checked={selectedArchivePaths.size === archivePreview.file_count && archivePreview.file_count > 0}
onChange={(event) => toggleAllArchiveFiles(event.target.checked)}
disabled={busy || archivePreview.file_count === 0} />
<span>Select all files</span>
</label>
<span>{selectedArchivePaths.size} of {archivePreview.file_count} selected</span>
</div>
<div className="archive-entry-list" role="list" aria-label="Archive contents">
{archivePreview.entries.map((entry) => {
const selectableFiles = archiveFilesForEntry(entry);
const depth = Math.max(0, entry.path.split("/").length - 1);
return (
<label
key={`${entry.kind}:${entry.path}`}
className={`archive-entry-row ${entry.kind === "directory" ? "is-directory" : ""}`}
style={{ paddingLeft: `${12 + depth * 18}px` }}>
<input
type="checkbox"
checked={archiveEntryIsSelected(entry)}
onChange={(event) => toggleArchiveEntry(entry, event.target.checked)}
disabled={busy || selectableFiles.length === 0} />
{entry.kind === "directory" ?
<Folder size={16} aria-hidden="true" /> :
<File size={16} aria-hidden="true" />
}
<span className="archive-entry-path">{entry.path.split("/").at(-1)}</span>
<span className="archive-entry-size">{entry.kind === "file" ? formatBytes(entry.size_bytes) : `${selectableFiles.length} files`}</span>
</label>);
})}
</div>
<p className="form-help">
Preview expires {formatDate(archivePreview.expires_at)}. The original archive is uploaded again only when you confirm.
</p>
<div className="button-row compact-actions archive-preview-actions">
<Button
onClick={() => {
resetArchiveUploadState();
setError("");
}}
disabled={busy}>
Choose another file
</Button>
{archivePreview.requires_password &&
<Button
onClick={() => activeDialogTarget && void loadArchivePreview(archiveFile, activeDialogTarget, { preserveSelection: true })}
disabled={busy || !archivePassword}>
<RefreshCw size={15} aria-hidden="true" /> Verify password
</Button>
}
<span className="archive-preview-action-spacer" />
<Button onClick={closeDialog} disabled={busy}>i18n:govoplan-files.cancel.77dfd213</Button>
<Button
variant="primary"
onClick={() => void confirmCurrentArchive()}
disabled={
busy
|| selectedArchivePaths.size === 0
|| archivePreview.requires_password && !archivePassword
}>
Import selected
</Button>
</div>
</div>
}
</FileDialog>
}
@@ -2509,6 +2929,72 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut
}
function FilePropertyFilterRow({
campaignUsage,
auditRelevant,
busy,
active,
disabled,
total,
onApply,
onClear
}: {
campaignUsage: "" | FileCampaignUsageFilter;
auditRelevant: AuditRelevantFilter;
busy: boolean;
active: boolean;
disabled: boolean;
total: number | null;
onApply: (campaignUsage: "" | FileCampaignUsageFilter, auditRelevant: AuditRelevantFilter) => void;
onClear: () => void;
}) {
const [campaignUsageDraft, setCampaignUsageDraft] = useState(campaignUsage);
const [auditRelevantDraft, setAuditRelevantDraft] = useState(auditRelevant);
useEffect(() => {
setCampaignUsageDraft(campaignUsage);
}, [campaignUsage]);
useEffect(() => {
setAuditRelevantDraft(auditRelevant);
}, [auditRelevant]);
return (
<div className="file-property-filter-row">
<ListFilter size={17} aria-hidden="true" />
<label className="file-property-filter-field">
<span>Campaign use</span>
<select
value={campaignUsageDraft}
disabled={busy || disabled}
onChange={(event) => setCampaignUsageDraft(event.target.value as "" | FileCampaignUsageFilter)}>
<option value="">Any</option>
<option value="linked">Linked or used</option>
<option value="unlinked">Not linked or used</option>
</select>
</label>
<label className="file-property-filter-field">
<span>Audit evidence</span>
<select
value={auditRelevantDraft}
disabled={busy || disabled}
onChange={(event) => setAuditRelevantDraft(event.target.value as AuditRelevantFilter)}>
<option value="">Any</option>
<option value="true">Audit relevant</option>
<option value="false">Not audit relevant</option>
</select>
</label>
<Button
onClick={() => onApply(campaignUsageDraft, auditRelevantDraft)}
disabled={busy || disabled || !campaignUsageDraft && !auditRelevantDraft}>
Apply filters
</Button>
{active && <Button onClick={onClear} disabled={busy}>Clear filters</Button>}
{active && total !== null && <span className="file-property-filter-count">{total} matching file{total === 1 ? "" : "s"}</span>}
</div>
);
}
function FileSearchRow({
value,
caseSensitive,
@@ -2559,7 +3045,15 @@ function FileSearchRow({
}
function activeCampaignShares(file: ManagedFile) {
return (file.shares ?? []).filter((share) => share.target_type === "campaign" && !share.revoked_at);
return (file.shares ?? []).filter((share) =>
share.target_type === "campaign" && fileShareIsActive(share)
);
}
function fileShareIsActive(share: NonNullable<ManagedFile["shares"]>[number]): boolean {
if (typeof share.active === "boolean") return share.active;
return !share.revoked_at
&& (!share.expires_at || new Date(share.expires_at).getTime() > Date.now());
}
function activeCampaignShareCount(file: ManagedFile): number {
@@ -0,0 +1,300 @@
import { useEffect, useMemo, useState } from "react";
import { Trash2 } from "lucide-react";
import {
Button,
ConfirmDialog,
DataGrid,
DismissibleAlert,
FormField,
ReferenceSelect,
StatusBadge,
TableActionGroup,
type ApiSettings,
type DataGridColumn
} from "@govoplan/core-webui";
import {
createFileShare,
fileShareTargetProvider,
listFileShares,
revokeFileShare,
type FileShare,
type ManagedFile
} from "../../../api/files";
import { FileDialog } from "./FileManagerComponents";
import { formatDate } from "../utils/fileManagerUtils";
type ShareTargetType = "user" | "group" | "tenant";
type SharePermission = "read" | "write" | "manage";
export default function FileShareDialog({
settings,
file,
tenantId,
onClose,
onChanged
}: {
settings: ApiSettings;
file: ManagedFile;
tenantId: string;
onClose: () => void;
onChanged: () => void;
}) {
const [shares, setShares] = useState<FileShare[]>([]);
const [targetType, setTargetType] = useState<ShareTargetType>("user");
const [targetId, setTargetId] = useState("");
const [permission, setPermission] = useState<SharePermission>("read");
const [expiresAt, setExpiresAt] = useState("");
const [busy, setBusy] = useState(false);
const [error, setError] = useState("");
const [revokeTarget, setRevokeTarget] = useState<FileShare | null>(null);
const targetProvider = useMemo(
() => targetType === "tenant"
? null
: fileShareTargetProvider(settings, file.id, targetType),
[file.id, settings, targetType]
);
useEffect(() => {
void refresh();
}, [file.id]);
async function refresh() {
setBusy(true);
setError("");
try {
const response = await listFileShares(settings, file.id, {
includeInactive: true
});
setShares(response.shares);
} catch (err) {
setError(err instanceof Error ? err.message : String(err));
} finally {
setBusy(false);
}
}
async function grant() {
const effectiveTargetId = targetType === "tenant" ? tenantId : targetId;
if (!effectiveTargetId) return;
setBusy(true);
setError("");
try {
await createFileShare(settings, file.id, {
target_type: targetType,
target_id: effectiveTargetId,
permission,
expires_at: expiresAt ? new Date(expiresAt).toISOString() : null
});
setTargetId("");
setExpiresAt("");
await refresh();
onChanged();
} catch (err) {
setError(err instanceof Error ? err.message : String(err));
setBusy(false);
}
}
async function revoke() {
if (!revokeTarget) return;
setBusy(true);
setError("");
try {
await revokeFileShare(settings, file.id, revokeTarget.id);
setRevokeTarget(null);
await refresh();
onChanged();
} catch (err) {
setError(err instanceof Error ? err.message : String(err));
setBusy(false);
}
}
const columns = useMemo<DataGridColumn<FileShare>[]>(() => [
{
id: "target",
header: "Target",
minWidth: 220,
resizable: true,
sortable: true,
filterable: true,
value: (share) => `${share.target_type}: ${share.target_id}`
},
{
id: "permission",
header: "Permission",
width: 115,
sortable: true,
filterable: true,
value: (share) => share.permission
},
{
id: "created",
header: "Created",
minWidth: 160,
resizable: true,
sortable: true,
value: (share) => formatDate(share.created_at)
},
{
id: "creator",
header: "Created by",
minWidth: 150,
resizable: true,
filterable: true,
value: (share) => share.created_by_user_id || "System"
},
{
id: "expiry",
header: "Expires",
minWidth: 160,
resizable: true,
sortable: true,
value: (share) => share.expires_at ? formatDate(share.expires_at) : "No expiry"
},
{
id: "status",
header: "Status",
width: 115,
sortable: true,
value: (share) => shareStatus(share),
render: (share) => (
<StatusBadge
status={share.active ? "active" : "inactive"}
label={shareStatus(share)}
/>
)
},
{
id: "actions",
header: "Actions",
width: 72,
sticky: "end",
align: "right",
render: (share) => (
<TableActionGroup
minimumSlots={1}
actions={[
{
id: "revoke",
label: "Revoke share",
icon: <Trash2 size={16} aria-hidden="true" />,
variant: "danger",
disabled: !share.active || busy,
disabledReason: !share.active ? "This share is already inactive." : undefined,
onClick: () => setRevokeTarget(share)
}
]}
/>
)
}
], [busy]);
return (
<>
<FileDialog title={`Manage shares - ${file.filename}`} onClose={() => !busy && onClose()}>
<div className="file-share-dialog-content">
{error ? (
<DismissibleAlert tone="danger" resetKey={error}>
{error}
</DismissibleAlert>
) : null}
<div className="file-share-form">
<FormField label="Target type">
<select
value={targetType}
disabled={busy}
onChange={(event) => {
setTargetType(event.target.value as ShareTargetType);
setTargetId("");
}}
>
<option value="user">User</option>
<option value="group">Group</option>
<option value="tenant">Entire tenant</option>
</select>
</FormField>
<FormField label="Target">
{targetType === "tenant" ? (
<input value={tenantId} disabled readOnly />
) : targetProvider ? (
<ReferenceSelect
value={targetId}
onChange={setTargetId}
provider={targetProvider}
disabled={busy}
placeholder={`Select a ${targetType}`}
aria-label={`Share target ${targetType}`}
required
/>
) : null}
</FormField>
<FormField label="Permission">
<select
value={permission}
disabled={busy}
onChange={(event) => setPermission(event.target.value as SharePermission)}
>
<option value="read">Read</option>
<option value="write">Write</option>
<option value="manage">Manage</option>
</select>
</FormField>
<FormField label="Expires (optional)">
<input
type="datetime-local"
value={expiresAt}
min={minimumLocalDateTime()}
disabled={busy}
onChange={(event) => setExpiresAt(event.target.value)}
/>
</FormField>
<Button
variant="primary"
disabled={busy || (targetType !== "tenant" && !targetId)}
onClick={() => void grant()}
>
Grant access
</Button>
</div>
<DataGrid
id={`file-${file.id}-shares`}
rows={shares}
columns={columns}
getRowKey={(share) => share.id}
emptyText={busy ? "Loading shares..." : "This file has no direct shares."}
initialFit="container"
/>
<div className="button-row compact-actions align-end">
<Button onClick={onClose} disabled={busy}>Close</Button>
</div>
</div>
</FileDialog>
<ConfirmDialog
open={Boolean(revokeTarget)}
title="Revoke file share"
message="Revoke this direct grant? Other independent grants remain in effect."
confirmLabel="Revoke"
cancelLabel="Cancel"
tone="danger"
busy={busy}
onConfirm={() => void revoke()}
onCancel={() => setRevokeTarget(null)}
/>
</>
);
}
function shareStatus(share: FileShare): string {
if (share.revoked_at) return "Revoked";
if (share.expires_at && new Date(share.expires_at).getTime() <= Date.now()) {
return "Expired";
}
return "Active";
}
function minimumLocalDateTime(): string {
const now = new Date(Date.now() + 60_000);
const local = new Date(now.getTime() - now.getTimezoneOffset() * 60_000);
return local.toISOString().slice(0, 16);
}
@@ -1024,7 +1024,14 @@ renderPatternPreview?: FilesManagedFileChooserProps["renderPatternPreview"])
function isLinkedToTarget(file: ManagedFile, target?: FilesManagedFileLinkTarget): boolean {
if (!target) return false;
return Boolean(file.shares?.some((share) => share.target_type === target.type && share.target_id === target.id && !share.revoked_at));
return Boolean(file.shares?.some((share) =>
share.target_type === target.type
&& share.target_id === target.id
&& (typeof share.active === "boolean"
? share.active
: !share.revoked_at
&& (!share.expires_at || new Date(share.expires_at).getTime() > Date.now()))
));
}
function readRememberedState(key: string): RememberedChooserState {
+1
View File
@@ -1,6 +1,7 @@
export { default } from "./module";
export * from "./module";
export { default as FilesPage } from "./features/files/FilesPage";
export { default as FileIntegrityPanel } from "./features/files/FileIntegrityPanel";
export * from "./api/files";
export type { PlatformWebModule, PlatformNavItem, PlatformRouteContribution, PlatformRouteContext } from "@govoplan/core-webui";
export { FolderTree } from "./features/files/components/FileManagerComponents";
+85 -2
View File
@@ -1,23 +1,79 @@
import { createElement, lazy } from "react";
import { hasScope, type AdminSectionsUiCapability, type FilesConnectorsUiCapability, type FilesFileExplorerUiCapability, type PlatformWebModule } from "@govoplan/core-webui";
import {
hasScope,
type AdminSectionsUiCapability,
type DashboardWidgetsUiCapability,
type FilesConnectorsUiCapability,
type FilesFileExplorerUiCapability,
type PlatformWebModule
} from "@govoplan/core-webui";
import { FolderTree } from "./features/files/components/FileManagerComponents";
import FileConnectorSettingsPanel from "./features/files/FileConnectorSettingsPanel";
import ManagedFileChooser from "./features/files/components/ManagedFileChooser";
import FileSpacesWidget from "./features/files/FileSpacesWidget";
import { listFileSpaces, listFiles, listFilesDelta, listFolders, resolveFilePatterns, shareFilesWithTarget } from "./api/files";
import { generatedTranslations } from "./i18n/generatedTranslations";
import "./styles/file-manager.css";
const FilesPage = lazy(() => import("./features/files/FilesPage"));
const FileIntegrityPanel = lazy(() => import("./features/files/FileIntegrityPanel"));
const fileRead = ["files:file:read"];
const translations = {
en: generatedTranslations.en,
de: generatedTranslations.de
};
const fileDashboardWidgets: DashboardWidgetsUiCapability = {
widgets: [
{
id: "files.spaces",
surfaceId: "files.widget.spaces",
title: "File spaces",
description: "Managed and connected file spaces available to you.",
moduleId: "files",
category: "Content",
order: 35,
defaultVisible: false,
defaultSize: "medium",
supportedSizes: ["medium", "wide"],
anyOf: fileRead,
refreshIntervalMs: 60_000,
defaultConfiguration: {
maxItems: 6,
includeConnectors: true
},
configurationFields: [
{
id: "maxItems",
label: "Maximum spaces",
kind: "number",
min: 1,
max: 16,
step: 1,
required: true
},
{
id: "includeConnectors",
label: "Include connected storage",
kind: "boolean"
}
],
render: ({ settings, refreshKey, configuration }) =>
createElement(FileSpacesWidget, {
settings,
refreshKey,
configuration
})
}
]
};
const fileConnectorAdminSections: AdminSectionsUiCapability = {
sections: [
{
id: "system-file-connectors",
moduleId: "files",
kind: "settings",
surfaceId: "files.admin.system-connectors",
label: "i18n:govoplan-files.file_connections.1e362326",
group: "SYSTEM",
order: 75,
@@ -30,6 +86,9 @@ const fileConnectorAdminSections: AdminSectionsUiCapability = {
},
{
id: "tenant-file-connectors",
moduleId: "files",
kind: "settings",
surfaceId: "files.admin.tenant-connectors",
label: "i18n:govoplan-files.file_connections.1e362326",
group: "TENANT",
order: 65,
@@ -39,6 +98,20 @@ const fileConnectorAdminSections: AdminSectionsUiCapability = {
scopeType: "tenant",
canWrite: hasScope(auth, "admin:settings:write") || hasScope(auth, "files:file:admin")
})
},
{
id: "tenant-file-integrity",
moduleId: "files",
kind: "operations",
surfaceId: "files.admin.tenant-integrity",
label: "File integrity",
group: "TENANT",
order: 66,
allOf: ["files:file:admin"],
render: ({ settings, auth }) => createElement(FileIntegrityPanel, {
settings,
canWrite: hasScope(auth, "files:file:admin")
})
}]
};
@@ -49,6 +122,15 @@ export const filesModule: PlatformWebModule = {
version: "1.0.0",
dependencies: ["access"],
translations,
viewSurfaces: [
{ id: "files.admin.system-connectors", moduleId: "files", kind: "section", label: "System file connections", order: 75 },
{ id: "files.admin.tenant-connectors", moduleId: "files", kind: "section", label: "Tenant file connections", order: 65 },
{ id: "files.admin.tenant-integrity", moduleId: "files", kind: "section", label: "File integrity", order: 66 },
{ id: "files.admin.group-connectors", moduleId: "files", kind: "section", label: "Group file connections", order: 65 },
{ id: "files.admin.user-connectors", moduleId: "files", kind: "section", label: "User file connections", order: 65 },
{ id: "files.settings.connectors", moduleId: "files", kind: "section", label: "Personal file connections", order: 20 },
{ id: "files.widget.spaces", moduleId: "files", kind: "section", label: "File spaces widget", order: 35 }
],
navItems: [{ to: "/files", label: "i18n:govoplan-files.files.6ce6c512", iconName: "folder", anyOf: fileRead, order: 40 }],
routes: [
{ path: "/files", anyOf: fileRead, order: 40, render: ({ settings, auth }) => createElement(FilesPage, { settings, auth }) }],
@@ -67,7 +149,8 @@ export const filesModule: PlatformWebModule = {
"files.connectors": {
FileConnectorScopeManager: FileConnectorSettingsPanel
} satisfies FilesConnectorsUiCapability,
"admin.sections": fileConnectorAdminSections
"admin.sections": fileConnectorAdminSections,
"dashboard.widgets": fileDashboardWidgets
}
};
+193
View File
@@ -11,6 +11,31 @@
padding: 4px 0 10px 14px;
}
.file-property-filter-row {
display: flex;
align-items: end;
gap: 12px;
min-width: 0;
padding: 0 0 10px 14px;
}
.file-property-filter-field {
display: grid;
gap: 4px;
min-width: 180px;
}
.file-property-filter-field > span,
.file-property-filter-count {
color: var(--muted);
font-size: var(--font-size-sm);
}
.file-property-filter-count {
align-self: center;
margin-left: auto;
}
.file-list-drop-target.is-active,
.file-list-drop-target.is-drop-target {
background: var(--line);
@@ -263,6 +288,165 @@
width: min(820px, 100%);
}
.file-dialog:has(.archive-preview) {
width: min(860px, 100%);
}
.archive-upload-help {
margin: -8px 0 0;
}
.archive-preview {
display: grid;
min-width: 0;
gap: 16px;
}
.archive-preview-summary {
display: grid;
grid-template-columns: minmax(0, 1.3fr) minmax(0, 1fr) minmax(0, 1fr);
gap: 1px;
overflow: hidden;
border: var(--border-line);
border-radius: 6px;
background: var(--line);
}
.archive-preview-summary > div {
display: grid;
min-width: 0;
gap: 4px;
padding: 11px 12px;
background: var(--panel-soft);
}
.archive-preview-summary strong,
.archive-preview-summary span {
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
.archive-preview-summary span {
color: var(--muted);
font-size: var(--font-size-sm);
}
.archive-selection-toolbar {
display: flex;
align-items: center;
justify-content: space-between;
gap: 12px;
}
.archive-selection-toolbar label,
.archive-entry-row {
display: flex;
align-items: center;
gap: 9px;
}
.archive-selection-toolbar label {
font-weight: 700;
}
.archive-selection-toolbar > span,
.archive-entry-size {
color: var(--muted);
font-size: var(--font-size-sm);
}
.archive-entry-list {
display: grid;
align-content: start;
max-height: min(380px, 42vh);
overflow: auto;
scrollbar-gutter: stable;
border: var(--border-line);
border-radius: 6px;
background: var(--surface);
}
.archive-entry-row {
min-width: 0;
min-height: 38px;
padding-top: 7px;
padding-right: 12px;
padding-bottom: 7px;
border-bottom: var(--border-line);
cursor: pointer;
}
.archive-entry-row:last-child {
border-bottom: 0;
}
.archive-entry-row:hover {
background: var(--hover-tint);
}
.archive-entry-row.is-directory {
background: var(--panel-soft);
font-weight: 700;
}
.archive-entry-row.is-directory:hover {
background: var(--primary-soft);
}
.archive-entry-path {
min-width: 0;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
.archive-entry-size {
flex: 0 0 auto;
margin-left: auto;
}
.archive-preview-actions {
align-items: center;
}
.archive-preview-action-spacer {
flex: 1 1 auto;
}
.file-dialog:has(.file-share-dialog-content) {
width: min(1080px, 100%);
}
.file-share-dialog-content {
display: grid;
min-width: 0;
gap: 16px;
}
.file-share-form {
display: grid;
grid-template-columns: minmax(130px, .7fr) minmax(220px, 1.4fr) minmax(120px, .7fr) minmax(190px, 1fr) auto;
gap: 12px;
align-items: end;
}
@media (max-width: 900px) {
.file-share-form {
grid-template-columns: minmax(0, 1fr) minmax(0, 1fr);
}
}
@media (max-width: 620px) {
.archive-preview-summary {
grid-template-columns: minmax(0, 1fr);
}
.file-share-form {
grid-template-columns: minmax(0, 1fr);
}
}
.connector-sync-grid {
display: grid;
grid-template-columns: minmax(0, .9fr) minmax(0, 1.1fr);
@@ -565,6 +749,15 @@
.file-search-row {
grid-template-columns: 1fr;
}
.file-property-filter-row {
align-items: stretch;
flex-direction: column;
}
.file-property-filter-count {
margin-left: 0;
}
}
.file-conflict-summary {