Compare commits
28
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a8c3192c3a | ||
|
|
ce4eaefbfd | ||
|
|
6c3cf1c55e | ||
|
|
95aef18955 | ||
|
|
a9c7a7a40c | ||
|
|
e1361427b8 | ||
|
|
ad55d47645 | ||
|
|
21122e058e | ||
|
|
25140fb0b6 | ||
|
|
e9561a87e1 | ||
|
|
9d7d3f96ec | ||
|
|
44ba0ec9c1 | ||
|
|
1e511e4c15 | ||
|
|
cb37079465 | ||
|
|
04e2bae190 | ||
|
|
aa9f90c71e | ||
|
|
60c3d14f08 | ||
|
|
3a3360650f | ||
|
|
c9f8e9262e | ||
|
|
edee29af51 | ||
|
|
0293e49def | ||
|
|
aaa364b8d4 | ||
|
|
4bb17c0f4c | ||
|
|
92e649477f | ||
|
|
0c68e904cf | ||
|
|
8ec31b16ee | ||
|
|
7e6be4b017 | ||
|
|
04882f1628 |
@@ -0,0 +1,270 @@
|
|||||||
|
name: Module Package Release
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
tags:
|
||||||
|
- "v*"
|
||||||
|
workflow_dispatch:
|
||||||
|
inputs:
|
||||||
|
release_tag:
|
||||||
|
description: Existing protected version tag to publish
|
||||||
|
required: true
|
||||||
|
type: string
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
publish-packages:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
env:
|
||||||
|
GITEA_REPOSITORY: ${{ gitea.repository }}
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
|
||||||
|
with:
|
||||||
|
fetch-depth: 0
|
||||||
|
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065
|
||||||
|
with:
|
||||||
|
python-version: "3.12"
|
||||||
|
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020
|
||||||
|
with:
|
||||||
|
node-version: "22"
|
||||||
|
- name: Select and validate protected release tag
|
||||||
|
shell: bash
|
||||||
|
env:
|
||||||
|
REQUESTED_TAG: ${{ inputs.release_tag }}
|
||||||
|
TRIGGER_TAG: ${{ gitea.ref_name }}
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
tag="${REQUESTED_TAG:-$TRIGGER_TAG}"
|
||||||
|
case "$tag" in
|
||||||
|
v[0-9]*.[0-9]*.[0-9]*) ;;
|
||||||
|
*) echo "Release tag must start with a SemVer-shaped vX.Y.Z value" >&2; exit 1 ;;
|
||||||
|
esac
|
||||||
|
git fetch --force origin "refs/tags/$tag:refs/tags/$tag" refs/heads/main:refs/remotes/origin/main
|
||||||
|
tag_commit="$(git rev-list -n 1 "$tag")"
|
||||||
|
git merge-base --is-ancestor "$tag_commit" refs/remotes/origin/main || {
|
||||||
|
echo "Release tag is not contained in main" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
git checkout --detach "$tag"
|
||||||
|
printf 'RELEASE_TAG=%s\n' "$tag" >> "$GITEA_ENV"
|
||||||
|
printf 'SOURCE_DATE_EPOCH=%s\n' "$(git show -s --format=%ct HEAD)" >> "$GITEA_ENV"
|
||||||
|
- name: Validate package versions
|
||||||
|
run: |
|
||||||
|
python - <<'PY'
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import tomllib
|
||||||
|
|
||||||
|
tag = os.environ["RELEASE_TAG"]
|
||||||
|
expected = tag.removeprefix("v")
|
||||||
|
project = tomllib.loads(Path("pyproject.toml").read_text(encoding="utf-8"))["project"]
|
||||||
|
if project.get("version") != expected:
|
||||||
|
raise SystemExit(f"pyproject version {project.get('version')!r} does not match {tag}")
|
||||||
|
if re.fullmatch(r"govoplan-[a-z0-9-]+", str(project.get("name", ""))) is None:
|
||||||
|
raise SystemExit("Python distribution name must use the govoplan-* namespace")
|
||||||
|
webui = Path("webui/package.json")
|
||||||
|
if webui.is_file():
|
||||||
|
package = json.loads(webui.read_text(encoding="utf-8"))
|
||||||
|
if package.get("version") != expected:
|
||||||
|
raise SystemExit(f"WebUI version {package.get('version')!r} does not match {tag}")
|
||||||
|
if re.fullmatch(r"@govoplan/[a-z0-9-]+-webui", str(package.get("name", ""))) is None:
|
||||||
|
raise SystemExit("WebUI package name must use the @govoplan/*-webui namespace")
|
||||||
|
release = Path("webui/package.release.json")
|
||||||
|
if release.is_file():
|
||||||
|
release_package = json.loads(release.read_text(encoding="utf-8"))
|
||||||
|
if (
|
||||||
|
release_package.get("name") != package.get("name")
|
||||||
|
or release_package.get("version") != expected
|
||||||
|
):
|
||||||
|
raise SystemExit("WebUI release package identity does not match package.json and the release tag")
|
||||||
|
PY
|
||||||
|
- name: Build immutable package artifacts
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
python -m pip install --disable-pip-version-check build==1.5.0 twine==7.0.0
|
||||||
|
rm -rf dist .package-webui
|
||||||
|
python -m build --wheel --outdir dist
|
||||||
|
python -m twine check dist/*.whl
|
||||||
|
if [[ -f webui/package.json ]]; then
|
||||||
|
mkdir .package-webui
|
||||||
|
cp -a webui/. .package-webui/
|
||||||
|
rm -rf .package-webui/node_modules .package-webui/dist
|
||||||
|
if [[ -f .package-webui/package.release.json ]]; then
|
||||||
|
cp .package-webui/package.release.json .package-webui/package.json
|
||||||
|
fi
|
||||||
|
node <<'NODE'
|
||||||
|
const fs = require("node:fs");
|
||||||
|
const path = ".package-webui/package.json";
|
||||||
|
const packageJson = JSON.parse(fs.readFileSync(path, "utf8"));
|
||||||
|
const groups = ["dependencies", "optionalDependencies", "peerDependencies"];
|
||||||
|
for (const group of groups) {
|
||||||
|
for (const [name, specifier] of Object.entries(packageJson[group] || {})) {
|
||||||
|
if (!name.startsWith("@govoplan/")) continue;
|
||||||
|
if (typeof specifier !== "string") {
|
||||||
|
throw new Error(`${group}.${name} must use a string version`);
|
||||||
|
}
|
||||||
|
const packageSlug = name.slice("@govoplan/".length);
|
||||||
|
if (!packageSlug.endsWith("-webui")) {
|
||||||
|
throw new Error(`${group}.${name} is outside the WebUI package namespace`);
|
||||||
|
}
|
||||||
|
const repository = `govoplan-${packageSlug.slice(0, -"-webui".length)}`;
|
||||||
|
const escapedRepository = repository.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||||
|
const gitTag = specifier.match(
|
||||||
|
new RegExp(
|
||||||
|
`^git\\+(?:ssh://git@|https://)git\\.add-ideas\\.de/(?:GovOPlaN|add-ideas)/${escapedRepository}\\.git#v([0-9]+\\.[0-9]+\\.[0-9]+)$`,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
if (gitTag) {
|
||||||
|
packageJson[group][name] = gitTag[1];
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (specifier.startsWith("file:") || specifier.startsWith("git+")) {
|
||||||
|
throw new Error(
|
||||||
|
`${group}.${name} must resolve to an exact registry version for publication`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
delete packageJson.private;
|
||||||
|
fs.writeFileSync(path, `${JSON.stringify(packageJson, null, 2)}\n`);
|
||||||
|
NODE
|
||||||
|
npm pkg delete private --prefix .package-webui
|
||||||
|
(cd .package-webui && npm pack --ignore-scripts --pack-destination ../dist)
|
||||||
|
fi
|
||||||
|
python - <<'PY'
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
artifacts = []
|
||||||
|
for path in sorted(Path("dist").iterdir()):
|
||||||
|
if path.suffix not in {".whl", ".tgz"}:
|
||||||
|
continue
|
||||||
|
digest = hashlib.sha256(path.read_bytes()).hexdigest()
|
||||||
|
artifacts.append({"filename": path.name, "sha256": digest, "size": path.stat().st_size})
|
||||||
|
payload = {
|
||||||
|
"schema_version": "1",
|
||||||
|
"repository": os.environ["GITEA_REPOSITORY"],
|
||||||
|
"tag": os.environ["RELEASE_TAG"],
|
||||||
|
"commit": subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip(),
|
||||||
|
"artifacts": artifacts,
|
||||||
|
}
|
||||||
|
Path("dist/package-artifacts.json").write_text(
|
||||||
|
json.dumps(payload, indent=2, sort_keys=True) + "\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
PY
|
||||||
|
- name: Retain package hash evidence
|
||||||
|
uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32
|
||||||
|
with:
|
||||||
|
name: module-packages-${{ gitea.ref_name }}
|
||||||
|
path: dist/package-artifacts.json
|
||||||
|
- name: Check immutable registry state
|
||||||
|
shell: bash
|
||||||
|
env:
|
||||||
|
PACKAGE_TOKEN: ${{ secrets.GOVOPLAN_PACKAGE_TOKEN }}
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
test -n "$PACKAGE_TOKEN"
|
||||||
|
python - <<'PY'
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import tomllib
|
||||||
|
from urllib.error import HTTPError
|
||||||
|
from urllib.parse import quote
|
||||||
|
from urllib.request import Request, urlopen
|
||||||
|
|
||||||
|
api_root = "https://git.add-ideas.de/api/v1/packages/GovOPlaN"
|
||||||
|
token = os.environ["PACKAGE_TOKEN"]
|
||||||
|
|
||||||
|
def should_publish(kind, name, version, path):
|
||||||
|
package_url = "/".join(
|
||||||
|
(api_root, kind, quote(name, safe=""), quote(version, safe=""), "files")
|
||||||
|
)
|
||||||
|
request = Request(
|
||||||
|
package_url,
|
||||||
|
headers={"Accept": "application/json", "Authorization": f"token {token}"},
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
with urlopen(request, timeout=30) as response:
|
||||||
|
files = json.load(response)
|
||||||
|
except HTTPError as exc:
|
||||||
|
if exc.code == 404:
|
||||||
|
print(f"{kind} package {name}=={version} is not published yet")
|
||||||
|
return True
|
||||||
|
raise
|
||||||
|
if not isinstance(files, list) or len(files) != 1:
|
||||||
|
raise SystemExit(
|
||||||
|
f"immutable {kind} package {name}=={version} has an unexpected file set"
|
||||||
|
)
|
||||||
|
expected_sha256 = hashlib.sha256(path.read_bytes()).hexdigest()
|
||||||
|
if files[0].get("sha256") != expected_sha256:
|
||||||
|
raise SystemExit(
|
||||||
|
f"immutable {kind} package {name}=={version} already exists with a different SHA-256"
|
||||||
|
)
|
||||||
|
print(f"verified existing {kind} package {name}=={version} ({expected_sha256})")
|
||||||
|
return False
|
||||||
|
|
||||||
|
project = tomllib.loads(Path("pyproject.toml").read_text(encoding="utf-8"))["project"]
|
||||||
|
wheels = tuple(Path("dist").glob("*.whl"))
|
||||||
|
if len(wheels) != 1:
|
||||||
|
raise SystemExit("release build must contain exactly one wheel")
|
||||||
|
publish_pypi = should_publish(
|
||||||
|
"pypi", str(project["name"]), str(project["version"]), wheels[0]
|
||||||
|
)
|
||||||
|
|
||||||
|
tarballs = tuple(Path("dist").glob("*.tgz"))
|
||||||
|
if len(tarballs) > 1:
|
||||||
|
raise SystemExit("release build must contain at most one npm package")
|
||||||
|
publish_npm = False
|
||||||
|
if tarballs:
|
||||||
|
webui = json.loads(
|
||||||
|
Path(".package-webui/package.json").read_text(encoding="utf-8")
|
||||||
|
)
|
||||||
|
publish_npm = should_publish(
|
||||||
|
"npm", str(webui["name"]), str(webui["version"]), tarballs[0]
|
||||||
|
)
|
||||||
|
|
||||||
|
with Path(os.environ["GITEA_ENV"]).open("a", encoding="utf-8") as env_file:
|
||||||
|
env_file.write(f"PUBLISH_PYPI={int(publish_pypi)}\n")
|
||||||
|
env_file.write(f"PUBLISH_NPM={int(publish_npm)}\n")
|
||||||
|
PY
|
||||||
|
- name: Publish wheel and WebUI package
|
||||||
|
shell: bash
|
||||||
|
env:
|
||||||
|
PACKAGE_USERNAME: ${{ secrets.GOVOPLAN_PACKAGE_USERNAME }}
|
||||||
|
PACKAGE_TOKEN: ${{ secrets.GOVOPLAN_PACKAGE_TOKEN }}
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
test -n "$PACKAGE_USERNAME"
|
||||||
|
test -n "$PACKAGE_TOKEN"
|
||||||
|
if [[ "$PUBLISH_PYPI" == 1 ]]; then
|
||||||
|
TWINE_USERNAME="$PACKAGE_USERNAME" TWINE_PASSWORD="$PACKAGE_TOKEN" \
|
||||||
|
python -m twine upload --non-interactive \
|
||||||
|
--repository-url https://git.add-ideas.de/api/packages/GovOPlaN/pypi \
|
||||||
|
dist/*.whl
|
||||||
|
else
|
||||||
|
echo "Exact wheel is already present; skipping immutable retry."
|
||||||
|
fi
|
||||||
|
shopt -s nullglob
|
||||||
|
webui_packages=(dist/*.tgz)
|
||||||
|
if (( ${#webui_packages[@]} )) && [[ "$PUBLISH_NPM" == 1 ]]; then
|
||||||
|
npmrc="$(mktemp)"
|
||||||
|
trap 'rm -f "$npmrc"' EXIT
|
||||||
|
chmod 600 "$npmrc"
|
||||||
|
printf '%s\n' \
|
||||||
|
'@govoplan:registry=https://git.add-ideas.de/api/packages/GovOPlaN/npm/' \
|
||||||
|
"//git.add-ideas.de/api/packages/GovOPlaN/npm/:_authToken=$PACKAGE_TOKEN" \
|
||||||
|
> "$npmrc"
|
||||||
|
NPM_CONFIG_USERCONFIG="$npmrc" npm publish "./${webui_packages[0]}" \
|
||||||
|
--ignore-scripts --access public \
|
||||||
|
--registry https://git.add-ideas.de/api/packages/GovOPlaN/npm/
|
||||||
|
elif (( ${#webui_packages[@]} )); then
|
||||||
|
echo "Exact WebUI package is already present; skipping immutable retry."
|
||||||
|
fi
|
||||||
@@ -55,6 +55,15 @@ for example for managed attachment selection and campaign file sharing. Files
|
|||||||
does not import campaign internals; campaign share/existence checks use the core
|
does not import campaign internals; campaign share/existence checks use the core
|
||||||
`campaigns.access` capability registered by the campaign module.
|
`campaigns.access` capability registered by the campaign module.
|
||||||
|
|
||||||
|
Files also publishes the optional `privacy.dsar.files` capability. The Core
|
||||||
|
data-subject workflow can use it to collect bounded, tenant-scoped file,
|
||||||
|
version, share, folder, evidence, and non-secret configuration metadata for a
|
||||||
|
direct membership subject. The provider may revoke a subject-targeted share or
|
||||||
|
detach mutable actor references idempotently, but it never exports credential
|
||||||
|
material or raw bytes and never bypasses Files retention, legal hold, evidence,
|
||||||
|
purge approval, audit, or recovery controls. See the handbook's data-subject
|
||||||
|
request coverage section for the review and erasure boundary.
|
||||||
|
|
||||||
Platform RBAC and governance rules are documented in `govoplan-core/docs/`.
|
Platform RBAC and governance rules are documented in `govoplan-core/docs/`.
|
||||||
|
|
||||||
Managed files can carry source provenance for connector and import workflows.
|
Managed files can carry source provenance for connector and import workflows.
|
||||||
@@ -105,7 +114,8 @@ run profile policy with the `browse` operation, and support Seafile,
|
|||||||
WebDAV/Nextcloud, and SMB when the optional `smb` extra is installed. Seafile
|
WebDAV/Nextcloud, and SMB when the optional `smb` extra is installed. Seafile
|
||||||
profiles browse libraries and directories via Seafile's read-only API; profiles
|
profiles browse libraries and directories via Seafile's read-only API; profiles
|
||||||
can still opt into WebDAV browsing by setting `metadata.webdav_endpoint_url` or
|
can still opt into WebDAV browsing by setting `metadata.webdav_endpoint_url` or
|
||||||
`metadata.browse_protocol` to `webdav`.
|
`metadata.browse_protocol` to `webdav`. S3-compatible profiles support bucket
|
||||||
|
and prefix browsing when the optional `s3` extra is installed.
|
||||||
`POST /api/v1/files/connectors/profiles/{profile_id}/import` imports a Seafile
|
`POST /api/v1/files/connectors/profiles/{profile_id}/import` imports a Seafile
|
||||||
or WebDAV/Nextcloud/SMB file into managed storage through the same governance,
|
or WebDAV/Nextcloud/SMB file into managed storage through the same governance,
|
||||||
conflict handling, source provenance, and connector audit path as direct
|
conflict handling, source provenance, and connector audit path as direct
|
||||||
@@ -113,17 +123,29 @@ uploads. The Seafile provider uses account-token auth and the native file
|
|||||||
download-link API; Nextcloud and generic WebDAV profiles use authenticated `GET`
|
download-link API; Nextcloud and generic WebDAV profiles use authenticated `GET`
|
||||||
requests against the configured WebDAV endpoint. SMB profiles use
|
requests against the configured WebDAV endpoint. SMB profiles use
|
||||||
`smb://server[:port]/share[/path]` endpoints and deployment-owned or encrypted
|
`smb://server[:port]/share[/path]` endpoints and deployment-owned or encrypted
|
||||||
stored credentials through `smbprotocol`. Because that SDK cannot accept a
|
stored credentials through `smbprotocol`. Files installs a pinned transport for
|
||||||
preconnected socket and may follow DFS referrals to additional hosts, live SMB
|
every initial session, reconnect, alias, and DFS referral target. The deployment
|
||||||
access fails closed in both public-only and private-network deployments. It will
|
private-network policy is re-evaluated immediately before each socket opens.
|
||||||
remain disabled until every initial connection and referral target can be
|
|
||||||
policy-validated and pinned.
|
|
||||||
|
|
||||||
S3 connector browse/import remains fail-closed because a user-governed connector
|
S3 connector browse/import binds botocore HTTP and HTTPS pools to the same pinned
|
||||||
endpoint must use the GovOPlaN pinned HTTP trust model. Durable platform storage
|
socket policy. Retries, redirects, endpoint discovery, bucket aliases, and new
|
||||||
has a separate deployment boundary: installer-owned Garage is accepted only at
|
connections are therefore revalidated while TLS keeps the configured hostname
|
||||||
its exact generated endpoint, while an operator-selected external backend
|
for SNI and certificate checks. Connector clients do not use outbound proxies or
|
||||||
requires a clean HTTPS origin and
|
ambient AWS credential discovery; configure credentials on the governed profile,
|
||||||
|
or explicitly use an anonymous profile for public objects. Keep connector spaces
|
||||||
|
read-only unless a remote write is explicitly needed.
|
||||||
|
An S3 profile with the `write` capability can back a two-way connector space.
|
||||||
|
`POST /api/v1/files/connector-spaces/{space_id}/write-back` then conditionally
|
||||||
|
creates or replaces one remote object from a managed file, records durable
|
||||||
|
recovery intent before the provider effect, and verifies request/content markers
|
||||||
|
afterward. Automatic remote delete, rename, move, and ACL propagation remain
|
||||||
|
disabled; other connector providers remain read-only.
|
||||||
|
An incompatible SDK upgrade fails closed before a usable client/session is
|
||||||
|
returned.
|
||||||
|
|
||||||
|
Durable platform storage has a separate deployment boundary: installer-owned
|
||||||
|
Garage is accepted only at its exact generated endpoint, while an
|
||||||
|
operator-selected external backend requires a clean HTTPS origin and
|
||||||
`FILE_STORAGE_S3_ENDPOINT_TRUSTED=true`. That flag is deployment configuration,
|
`FILE_STORAGE_S3_ENDPOINT_TRUSTED=true`. That flag is deployment configuration,
|
||||||
cannot be supplied through a Files connector profile, and does not replace
|
cannot be supplied through a Files connector profile, and does not replace
|
||||||
operator responsibility for DNS, certificates, egress, bucket policy,
|
operator responsibility for DNS, certificates, egress, bucket policy,
|
||||||
@@ -160,11 +182,22 @@ the original archive and stores only the selected members. Password-protected
|
|||||||
ZIP passwords remain request-only and are never included in the preview token.
|
ZIP passwords remain request-only and are never included in the preview token.
|
||||||
|
|
||||||
Managed blob writes and applied orphan cleanup use Core's durable recovery
|
Managed blob writes and applied orphan cleanup use Core's durable recovery
|
||||||
ledger. Intent, request digests, recovery mode, and a distributed lease are
|
ledger. On PostgreSQL, intent, request digests, recovery mode, and a distributed
|
||||||
committed before physical storage effects; the Files session commit verifies
|
lease are committed before physical storage effects; the Files session commit
|
||||||
database and streamed object evidence, while rollback compensates only a newly
|
verifies database and streamed object evidence, while rollback compensates only
|
||||||
reserved unreferenced key. New object keys are opaque and do not retain the
|
a newly reserved unreferenced key. Development SQLite records blob intent in
|
||||||
uploaded filename. Uncertain or mismatched effects remain visible through Ops.
|
the caller transaction to avoid its second-writer deadlock, then verifies on
|
||||||
|
commit or reconstructs compensation evidence after handled rollback. Because a
|
||||||
|
hard loss before that commit can leave an unrecorded object, SQLite requires a
|
||||||
|
complete integrity scan after a crash and is not a production recovery profile.
|
||||||
|
New object keys are opaque and do not retain the uploaded filename. Uncertain
|
||||||
|
or mismatched effects remain visible through Ops.
|
||||||
|
|
||||||
|
Operators with `files:file:admin` can run bounded, resumable integrity scans in
|
||||||
|
Administration. Scan batches and finding actions carry monotonic revisions;
|
||||||
|
stale resume, recheck, or cleanup requests fail before touching object storage.
|
||||||
|
Orphan cleanup always requires a dry-run preview followed by separate
|
||||||
|
confirmation and records recovery-ledger evidence.
|
||||||
|
|
||||||
Bulk rename and transfer APIs are owner-scoped: callers must provide the active
|
Bulk rename and transfer APIs are owner-scoped: callers must provide the active
|
||||||
user or group file space with `owner_type` and `owner_id`. The storage layer
|
user or group file space with `owner_type` and `owner_id`. The storage layer
|
||||||
|
|||||||
@@ -19,6 +19,7 @@ normal product workflows and can share the same governance model:
|
|||||||
- Nextcloud through WebDAV
|
- Nextcloud through WebDAV
|
||||||
- generic WebDAV
|
- generic WebDAV
|
||||||
- SMB through the optional `smb` extra
|
- SMB through the optional `smb` extra
|
||||||
|
- S3-compatible stores through the optional `s3` extra
|
||||||
|
|
||||||
These providers are surfaced through connector descriptors at
|
These providers are surfaced through connector descriptors at
|
||||||
`GET /api/v1/files/connectors/providers`. Provider descriptors declare whether
|
`GET /api/v1/files/connectors/providers`. Provider descriptors declare whether
|
||||||
@@ -60,11 +61,14 @@ Every provider must:
|
|||||||
- use a transport that pins every connection to a policy-validated DNS/IP answer
|
- use a transport that pins every connection to a policy-validated DNS/IP answer
|
||||||
and revalidates redirects; SDK transports without that guarantee fail closed
|
and revalidates redirects; SDK transports without that guarantee fail closed
|
||||||
|
|
||||||
Live SMB access is disabled in all modes until `smbprotocol` initial connections
|
SMB initial connections, reconnects, aliases, and DFS referral targets are
|
||||||
and DFS referral targets can be pinned and policy-validated. An explicit IP is
|
created through a Files-owned pinned `smbprotocol` transport. S3 HTTP/HTTPS pools
|
||||||
not sufficient because the server may still issue a referral to another peer.
|
use the equivalent botocore adapter for every connection selected by retries,
|
||||||
Live S3 access is likewise disabled until `boto3`/botocore can be bound to the
|
redirects, endpoint discovery, and virtual-host addressing. Both adapters apply
|
||||||
pinned transport, including SDK-managed redirects and endpoint discovery.
|
the deployment-wide private-network policy at socket creation. S3 keeps the
|
||||||
|
configured hostname for TLS SNI and certificate verification, but does not use
|
||||||
|
outbound proxies or ambient AWS credential discovery. An incompatible optional
|
||||||
|
SDK release fails closed before a usable session or client is returned.
|
||||||
|
|
||||||
## Non-Goals For Files
|
## Non-Goals For Files
|
||||||
|
|
||||||
|
|||||||
+263
-63
@@ -1,6 +1,6 @@
|
|||||||
# GovOPlaN Files Handbook
|
# GovOPlaN Files Handbook
|
||||||
|
|
||||||
This handbook describes the Files module as implemented in version `0.1.9`.
|
This handbook describes the Files module as implemented in version `0.1.18`.
|
||||||
It is the operational source of truth for users, process owners, administrators,
|
It is the operational source of truth for users, process owners, administrators,
|
||||||
operators, auditors, and module integrators. Statements about future behavior
|
operators, auditors, and module integrators. Statements about future behavior
|
||||||
are marked **planned**; an unmarked statement describes the current code.
|
are marked **planned**; an unmarked statement describes the current code.
|
||||||
@@ -8,7 +8,8 @@ are marked **planned**; an unmarked statement describes the current code.
|
|||||||
Files is a governed snapshot store. It owns managed file content, versions,
|
Files is a governed snapshot store. It owns managed file content, versions,
|
||||||
logical folders, shares, source provenance, and the evidence that another
|
logical folders, shares, source provenance, and the evidence that another
|
||||||
GovOPlaN module used a particular file version. It can browse selected external
|
GovOPlaN module used a particular file version. It can browse selected external
|
||||||
stores read-only and import a frozen copy. It is not a general remote filesystem,
|
stores and import a frozen copy. Explicit, conditional S3 write-back is available
|
||||||
|
only through an administrator-enabled two-way connector space. Files is not a general remote filesystem,
|
||||||
a document collaboration engine, or a records-management system.
|
a document collaboration engine, or a records-management system.
|
||||||
|
|
||||||
## Choose a reading path
|
## Choose a reading path
|
||||||
@@ -50,16 +51,39 @@ The main domain objects are:
|
|||||||
|
|
||||||
| Object | Meaning | Lifecycle today |
|
| Object | Meaning | Lifecycle today |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| File asset | The user-facing file identity, owner, logical path, description, metadata, and current version | Created, organized, shared, and soft-deleted |
|
| File asset | The user-facing file identity, owner, logical path, description, metadata, current version, retention deadline, legal hold, and lifecycle revision | Created, organized, shared, soft-deleted, restored, or governed-purged |
|
||||||
| File version | A numbered snapshot of one asset and its blob | Appended by connector sync when bytes change; retained |
|
| File version | A numbered snapshot of one asset and its blob | Appended by connector sync when bytes change; retained until its asset is eligible for purge |
|
||||||
| File blob | Stored plaintext semantics plus stored-byte integrity, backend key, optional Encryption envelope, reference count, and retention timestamp | Reused only within a tenant and matching protection boundary; no automated garbage collection |
|
| File blob | Stored plaintext semantics plus stored-byte integrity, backend key, optional Encryption envelope, reference count, and retention timestamp | Reused only within a tenant and matching protection boundary; collected only after a fenced reference recheck |
|
||||||
| Folder | An explicit logical path in a user or group space | Created, moved/renamed through organize operations, and soft-deleted |
|
| Folder | An explicit logical path in a user or group space | Created, moved/renamed, soft-deleted, and restored when its original path is free |
|
||||||
| Share | A grant from an asset to a user, group, tenant, or campaign with `read`, `write`, or `manage` permission | Created or updated; no public revocation endpoint yet |
|
| Share | A grant from an asset to a user, group, tenant, or campaign with `read`, `write`, or `manage` permission | Created or updated; no public revocation endpoint yet |
|
||||||
| Connector profile | A governed external endpoint, scope, optional credential link, policy, and descriptive capabilities | Created, updated, disabled, or credential-scrubbed on deletion |
|
| Connector profile | A governed external endpoint, scope, optional credential link, policy, and descriptive capabilities | Created, updated, disabled, or credential-scrubbed on deletion |
|
||||||
| Connector credential | Reusable authentication material, optionally limited to a provider and scope | Encrypted when database-managed; immediately scrubbed on deletion |
|
| Connector credential | Reusable authentication material, optionally limited to a provider and scope | Encrypted when database-managed; immediately scrubbed on deletion |
|
||||||
| Connector policy | Allow and deny rules inherited from system through tenant to one leaf scope | Evaluated before configuration and connector use |
|
| Connector policy | Allow and deny rules inherited from system through tenant to one leaf scope | Evaluated before configuration and connector use |
|
||||||
| Connector space | A read-only, manually synchronized remote folder/library linked to a user or group space | Created, updated, disabled, and soft-deleted |
|
| Connector space | A manually synchronized remote folder/library linked to a user or group space; read-only by default, with explicit S3 two-way mode available | Created, updated, disabled, soft-deleted, and restored |
|
||||||
| Campaign attachment use | Evidence connecting a campaign job or entry to an exact asset, version, blob, checksum, and stage | Retained for campaign execution evidence |
|
| Campaign attachment use | Evidence connecting a campaign job or entry to an exact asset, version, blob, checksum, and stage | Retained for campaign execution evidence |
|
||||||
|
| Form evidence upload grant | A one-use, hash-only bearer grant tied to an exact Form instance/revision, purpose, custodian, size, and media-type policy | Issued for at most 15 minutes, consumed by one managed upload, then retained as evidence provenance |
|
||||||
|
|
||||||
|
## Deployment configuration packages
|
||||||
|
|
||||||
|
Files registers the `files.configuration` capability for `managed_storage`
|
||||||
|
fragments. Managed content storage is deployment-owned: the installer selects
|
||||||
|
local persistent storage, managed Garage, or an external S3-compatible service
|
||||||
|
and mounts a validated non-secret infrastructure capability receipt. Files
|
||||||
|
does not copy that endpoint or its credentials into module-owned tables.
|
||||||
|
|
||||||
|
Preflight compares the receipt's `files.storage` capability with the effective
|
||||||
|
runtime backend. It validates backend kind, sanitized S3 endpoint, bucket,
|
||||||
|
Garage management marker or external trust marker, an absolute persistent path
|
||||||
|
for local storage, and the presence of Files-owned `env:` secret references.
|
||||||
|
Secret values are never read into a plan, diagnostic, export, or fragment.
|
||||||
|
|
||||||
|
When runtime and receipt agree, the plan reports `skip`: the desired binding is
|
||||||
|
already effective, and repeated apply is a no-op. A mismatch blocks import and
|
||||||
|
explains which deployment setting must be reconciled. The provider deliberately
|
||||||
|
does not mutate process environment, migrate stored objects, probe remote
|
||||||
|
storage, or reinterpret an infrastructure replacement as safe. Use the Files
|
||||||
|
integrity and Ops checks after deployment and complete migration/recovery review
|
||||||
|
before changing an active backend.
|
||||||
|
|
||||||
## User tasks
|
## User tasks
|
||||||
|
|
||||||
@@ -82,9 +106,12 @@ Files administrator; governed API operations can administer other tenant-owned
|
|||||||
Files resources when their owner is specified. Linked connector spaces appear
|
Files resources when their owner is specified. Linked connector spaces appear
|
||||||
beside managed spaces when they are active and visible to the user.
|
beside managed spaces when they are active and visible to the user.
|
||||||
|
|
||||||
A connector space is intentionally read-only. It is a view of an approved
|
A connector space is read-only by default. It is a view of an approved remote
|
||||||
remote location and a starting point for importing or synchronizing selected
|
location and a starting point for importing or synchronizing selected files
|
||||||
files into managed storage; it is not a mounted write-through filesystem.
|
into managed storage. An administrator may opt an S3 space into two-way mode
|
||||||
|
only after enabling the profile's `write` capability. Even then, Files exposes
|
||||||
|
explicit, conditional file write-back—not a mounted filesystem. Automatic
|
||||||
|
remote delete, rename, move, and ACL propagation stay disabled.
|
||||||
|
|
||||||
### Upload files
|
### Upload files
|
||||||
|
|
||||||
@@ -147,6 +174,73 @@ Recursive folder deletion is the default. It soft-deletes the selected folder,
|
|||||||
its child folders, and files below it. A non-recursive delete fails when the
|
its child folders, and files below it. A non-recursive delete fails when the
|
||||||
folder is not empty.
|
folder is not empty.
|
||||||
|
|
||||||
|
### Restore, retain, hold, and purge
|
||||||
|
|
||||||
|
Soft deletion is reversible for callers with `files:file:restore`. Restoring a
|
||||||
|
file preserves its asset identity, complete version history, blob references,
|
||||||
|
and source provenance. Recursive folder restore reactivates the folder tree and
|
||||||
|
its deleted files. Connector-space restore reactivates only the local link.
|
||||||
|
Each restore fails if an active resource already occupies the original path or
|
||||||
|
label.
|
||||||
|
|
||||||
|
Retention and legal hold are independent lifecycle controls. A caller with
|
||||||
|
`files:file:retention` supplies the current lifecycle revision, a reason, an
|
||||||
|
optional retained-until time, and legal-hold state. Stale revisions fail rather
|
||||||
|
than overwriting a concurrent decision.
|
||||||
|
|
||||||
|
Hard purge is deliberately separate from ordinary delete:
|
||||||
|
|
||||||
|
1. A caller with `files:file:purge` previews 1–100 soft-deleted assets.
|
||||||
|
2. The preview reports retention, legal-hold, active-share, Campaign-evidence,
|
||||||
|
and Form-evidence blockers and returns a SHA-256 over current lifecycle and
|
||||||
|
blob-reference state.
|
||||||
|
3. Execution requires that exact hash, a stable idempotency key, an approval
|
||||||
|
reference, and the literal `PURGE` confirmation.
|
||||||
|
4. The recovery-ledger operation and tenant purge fence are durable before the
|
||||||
|
irreversible database transaction. A stale preview or blocker rejects the
|
||||||
|
operation without erasure.
|
||||||
|
5. Purge removes eligible asset, version, and inactive-share rows and
|
||||||
|
recalculates blob reference counts. It does not delete bytes inline.
|
||||||
|
6. A separately authorized bounded blob-GC call takes the same per-blob lease
|
||||||
|
used by uploads, rechecks all `FileVersion` references, deletes the exact
|
||||||
|
object, verifies absence, and only then deletes `FileBlob` metadata.
|
||||||
|
|
||||||
|
Automatic time-based purge scheduling is not implemented. Operators initiate
|
||||||
|
preview, execute, and garbage collection under their local retention process.
|
||||||
|
|
||||||
|
### Data-subject request coverage
|
||||||
|
|
||||||
|
Files registers `privacy.dsar.files` when the module is active. The provider
|
||||||
|
requires a corroborated tenant membership identifier (or a namespaced Files
|
||||||
|
user reference), searches only that tenant, and fails explicitly if its bounded
|
||||||
|
result limit would be exceeded. It reports managed assets, exact versions,
|
||||||
|
folders, user-targeted shares, Form and Campaign evidence, connector
|
||||||
|
configuration actor references, and integrity-operation evidence.
|
||||||
|
|
||||||
|
The export includes only governed metadata. It never embeds file bytes, blob
|
||||||
|
storage keys, passwords, tokens, environment-variable names, secret-provider
|
||||||
|
references, or encrypted connector values. A reviewer follows the authorized
|
||||||
|
version download route when the file itself must be inspected.
|
||||||
|
|
||||||
|
The erasure plan deliberately separates four outcomes:
|
||||||
|
|
||||||
|
- an active share aimed at the subject can be revoked idempotently;
|
||||||
|
- mutable creator/updater references can be detached after tenant, subject, and
|
||||||
|
current-value revalidation;
|
||||||
|
- legal hold, active retention, submitted Form evidence, Campaign delivery
|
||||||
|
evidence, connector configuration history, and integrity evidence are
|
||||||
|
retained with a reason; and
|
||||||
|
- unstructured file content, ownership, filenames, and paths require manual
|
||||||
|
review.
|
||||||
|
|
||||||
|
DSAR execution never invokes physical byte deletion. If the privacy decision
|
||||||
|
authorizes erasure, the operator must use the separate Files soft-delete, purge
|
||||||
|
preview, approval, execution, and blob-GC sequence. This preserves its distinct
|
||||||
|
authority, evidence blockers, audit trail, distributed fencing, and recovery
|
||||||
|
ledger semantics. Email, account, or identity selectors alone are insufficient
|
||||||
|
because Files does not import the Access directory; the Access search supplies
|
||||||
|
the corroborated membership reference for Files coverage.
|
||||||
|
|
||||||
### Find and download files
|
### Find and download files
|
||||||
|
|
||||||
Files can list by owner and path, use cursor pagination, and consume incremental
|
Files can list by owner and path, use cursor pagination, and consume incremental
|
||||||
@@ -187,7 +281,14 @@ inside the chosen owner space:
|
|||||||
- identical checksum and size updates provenance and returns `unchanged`;
|
- identical checksum and size updates provenance and returns `unchanged`;
|
||||||
- changed bytes append a version and return `updated`.
|
- changed bytes append a version and return `updated`.
|
||||||
|
|
||||||
Browse, import, and sync never write, rename, or delete the remote source.
|
Browse, import, and inbound sync never mutate the remote source. An S3 space in
|
||||||
|
explicit two-way mode can write a selected managed file to one remote object
|
||||||
|
path through `POST /api/v1/files/connector-spaces/{space_id}/write-back`.
|
||||||
|
Creating a path uses `If-None-Match`; overwriting requires the currently
|
||||||
|
observed ETag or version and uses a conditional request. The provider object is
|
||||||
|
then re-read and must contain both the expected content digest and the recovery
|
||||||
|
operation marker. Uncertain or mismatching outcomes remain fenced and visible
|
||||||
|
in Ops. Remote deletion, rename, move, and permission propagation are disabled.
|
||||||
|
|
||||||
Connector administration separates endpoint profiles, reusable credentials,
|
Connector administration separates endpoint profiles, reusable credentials,
|
||||||
and inherited policy. Ordinary setup uses typed fields and provider discovery;
|
and inherited policy. Ordinary setup uses typed fields and provider discovery;
|
||||||
@@ -297,6 +398,10 @@ those relationships first.
|
|||||||
| `files:file:organize` | Create folders, rename, move/copy, and manage linked connector spaces |
|
| `files:file:organize` | Create folders, rename, move/copy, and manage linked connector spaces |
|
||||||
| `files:file:share` | Create or update file shares |
|
| `files:file:share` | Create or update file shares |
|
||||||
| `files:file:delete` | Soft-delete accessible writable files and folders |
|
| `files:file:delete` | Soft-delete accessible writable files and folders |
|
||||||
|
| `files:file:restore` | Restore owned or administered soft-deleted files, folders, and connector-space links |
|
||||||
|
| `files:file:retention` | Set retention deadlines and legal holds with optimistic revision checks |
|
||||||
|
| `files:file:purge` | Preview and execute irreversible metadata purge and collect unreferenced blobs |
|
||||||
|
| `files:connector:write` | Write an accessible managed file to an explicitly writable connector space |
|
||||||
| `files:file:admin` | Administer all Files spaces and connector settings in the active tenant |
|
| `files:file:admin` | Administer all Files spaces and connector settings in the active tenant |
|
||||||
|
|
||||||
The `file_manager` role template grants all normal file operations except
|
The `file_manager` role template grants all normal file operations except
|
||||||
@@ -316,12 +421,14 @@ Keep these definitions separate:
|
|||||||
local policy, and descriptive operation capabilities;
|
local policy, and descriptive operation capabilities;
|
||||||
- a **policy** restricts what a scope may configure or use;
|
- a **policy** restricts what a scope may configure or use;
|
||||||
- a **connector space** links one approved profile/library/path to one user or
|
- a **connector space** links one approved profile/library/path to one user or
|
||||||
group and always uses manual, read-only synchronization today.
|
group, uses manual synchronization, and is read-only unless explicitly
|
||||||
|
configured for the supported two-way S3 write boundary.
|
||||||
|
|
||||||
Profile capability values such as `browse`, `import`, and `sync` are stored and
|
Profile capability values such as `browse`, `import`, `sync`, and `write` are
|
||||||
returned, but they are descriptive today. Provider implementation and policy
|
stored and returned. `write` is additionally enforced for S3 write-back, but
|
||||||
checks enforce actual availability; do not use the capability list as the sole
|
provider implementation, connector-space mode, operation permission, resource
|
||||||
security control.
|
access, and inherited policy all remain mandatory; never use the capability
|
||||||
|
list as the sole security control.
|
||||||
|
|
||||||
Profiles, credentials, and policies support `system`, `tenant`, `user`,
|
Profiles, credentials, and policies support `system`, `tenant`, `user`,
|
||||||
`group`, and `campaign` scopes. A normal user sees system and active-tenant
|
`group`, and `campaign` scopes. A normal user sees system and active-tenant
|
||||||
@@ -405,15 +512,15 @@ in a development/test runtime.
|
|||||||
| Seafile | Read-only native API browse/download-link import and manual sync implemented using the pinned HTTP transport; WebDAV opt-in supported |
|
| Seafile | Read-only native API browse/download-link import and manual sync implemented using the pinned HTTP transport; WebDAV opt-in supported |
|
||||||
| Nextcloud | Read-only WebDAV browse/import/manual sync implemented using the pinned HTTP transport |
|
| Nextcloud | Read-only WebDAV browse/import/manual sync implemented using the pinned HTTP transport |
|
||||||
| Generic WebDAV | Read-only browse/import/manual sync implemented using the pinned HTTP transport |
|
| Generic WebDAV | Read-only browse/import/manual sync implemented using the pinned HTTP transport |
|
||||||
| SMB | Browse/import code and descriptor implemented, but every live connection fails closed until initial connections and DFS referrals can be policy-validated and pinned |
|
| SMB | Read-only browse/import/manual sync implemented through a pinned smbprotocol transport for initial peers, reconnects, aliases, and DFS referral targets |
|
||||||
| S3 connector | Browse/import code and descriptor implemented, but every live SDK connection fails closed until botocore connections, redirects, and endpoint discovery can be validated and pinned |
|
| S3 connector | Bucket/prefix browse, import, manual inbound sync, and explicit conditional write-back implemented through pinned botocore pools covering retries, redirects, endpoint discovery, and provider aliases; automatic remote delete/rename/move/ACL propagation disabled |
|
||||||
| SharePoint and OneDrive | Provider keys/descriptors reserved; live Microsoft Graph browse/import is planned |
|
| SharePoint and OneDrive | Provider keys/descriptors reserved; live Microsoft Graph browse/import is planned |
|
||||||
| NFS and local connector | Described as optional future providers; the local managed-storage backend is a different feature |
|
| NFS and local connector | Described as optional future providers; the local managed-storage backend is a different feature |
|
||||||
|
|
||||||
Provider descriptors are available from
|
Provider descriptors are available from
|
||||||
`GET /api/v1/files/connectors/providers`. Use their `implemented`, `installed`,
|
`GET /api/v1/files/connectors/providers`. Use their `implemented`, `installed`,
|
||||||
and support fields for display, but expect a fail-closed transport error where
|
and support fields for display. An incompatible optional SDK release fails closed
|
||||||
the table above says live access is disabled.
|
before it can return a usable client or session.
|
||||||
|
|
||||||
## Operator runbook
|
## Operator runbook
|
||||||
|
|
||||||
@@ -478,17 +585,29 @@ The built-in HTTP transport:
|
|||||||
- bounds structured responses to 16 MiB and file transfers to 512 MiB by
|
- bounds structured responses to 16 MiB and file transfers to 512 MiB by
|
||||||
default.
|
default.
|
||||||
|
|
||||||
|
The S3 SDK adapter applies the same socket rule to every botocore pool selected
|
||||||
|
for a retry, redirect, discovered endpoint, or virtual-host bucket alias. The
|
||||||
|
original authority remains in the request and TLS SNI/certificate check. S3
|
||||||
|
connector clients use no outbound proxy and never discover ambient AWS
|
||||||
|
credentials: configure both access and secret keys on the governed profile, or
|
||||||
|
use an anonymous profile for a public source.
|
||||||
|
|
||||||
|
The SMB adapter owns a separate connection cache and replaces smbprotocol's TCP
|
||||||
|
factory process-wide with the stricter pinned socket. Initial peers, reconnects,
|
||||||
|
server aliases, domain-controller connections, and DFS referral targets therefore
|
||||||
|
pass the same policy at connection time. Signing is required by default; enable
|
||||||
|
SMB encryption on the profile where the server supports it.
|
||||||
|
|
||||||
Override the connector response limits with
|
Override the connector response limits with
|
||||||
`GOVOPLAN_CONNECTOR_MAX_STRUCTURED_RESPONSE_BYTES` and
|
`GOVOPLAN_CONNECTOR_MAX_STRUCTURED_RESPONSE_BYTES` and
|
||||||
`GOVOPLAN_CONNECTOR_MAX_FILE_TRANSFER_BYTES`. The smaller applicable limit wins
|
`GOVOPLAN_CONNECTOR_MAX_FILE_TRANSFER_BYTES`. The smaller applicable limit wins
|
||||||
when an import is also subject to `FILE_UPLOAD_MAX_BYTES`.
|
when an import is also subject to `FILE_UPLOAD_MAX_BYTES`.
|
||||||
|
|
||||||
Never work around a connector pinning failure by adding a raw IP, disabling TLS,
|
Never work around a connector pinning failure by adding a raw IP, disabling TLS,
|
||||||
or enabling private networks. SMB may redirect through DFS, and user-configured
|
or enabling private networks. A failure means the peer policy rejected an actual
|
||||||
S3 connectors may perform their own redirects or endpoint discovery; those
|
connection destination or the installed SDK no longer exposes the verified
|
||||||
connector transports remain disabled until every connection peer can be
|
transport seam. The separately configured platform S3 backend is trusted only by
|
||||||
governed. The separately configured platform S3 backend is trusted only by the
|
the deployment owner and is not selectable by a user or connector profile.
|
||||||
deployment owner and is not selectable by a user or connector profile.
|
|
||||||
|
|
||||||
### Backup and restore
|
### Backup and restore
|
||||||
|
|
||||||
@@ -509,10 +628,13 @@ snapshots so database references and objects represent the same recovery point.
|
|||||||
The integrity API verifies a restored set, but it does not replace a coordinated
|
The integrity API verifies a restored set, but it does not replace a coordinated
|
||||||
backup.
|
backup.
|
||||||
|
|
||||||
Create a scan with `POST /api/v1/files/integrity/scans`, then call
|
Operators normally use **Administration > File integrity**. The equivalent API
|
||||||
`POST /api/v1/files/integrity/scans/{scan_id}/run` until it reports
|
creates a scan with `POST /api/v1/files/integrity/scans`, then calls
|
||||||
`completed`. Each call advances at most the persisted batch size, so a stopped
|
`POST /api/v1/files/integrity/scans/{scan_id}/run` with the scan's current
|
||||||
operator or worker can resume from the committed blob/object cursors.
|
`expected_revision` until it reports `completed`. Each call advances at most
|
||||||
|
the persisted batch size, so a stopped operator or worker can resume from the
|
||||||
|
committed blob/object cursors. Concurrent or stale actions receive `409` before
|
||||||
|
the storage backend is invoked; reload the scan and inspect the newer state.
|
||||||
|
|
||||||
Findings distinguish:
|
Findings distinguish:
|
||||||
|
|
||||||
@@ -524,22 +646,38 @@ Findings distinguish:
|
|||||||
|
|
||||||
Missing or corrupt blobs fail closed for ordinary downloads and Campaign
|
Missing or corrupt blobs fail closed for ordinary downloads and Campaign
|
||||||
attachment materialization. After restoring the expected bytes, use the finding
|
attachment materialization. After restoring the expected bytes, use the finding
|
||||||
`recheck` action first in dry-run mode and then apply it. Orphan cleanup is also
|
`recheck` action with its current `expected_revision`. Orphan cleanup starts
|
||||||
dry-run by default, rechecks that no database reference exists, remains scoped
|
with a dry-run preview and requires separate destructive confirmation. The
|
||||||
to the scanned tenant prefix, and is idempotent. Both applied and dry-run
|
confirmation reuses the finding revision from that preview, rechecks that no
|
||||||
actions emit audit evidence.
|
database reference exists, remains scoped to the scanned tenant prefix, and is
|
||||||
|
idempotent. Both applied and dry-run actions emit audit evidence. A shared
|
||||||
|
reference blocks orphan cleanup. Managed-asset purge separately enforces asset
|
||||||
|
retention, legal hold, active shares, and retained Campaign/Form evidence before
|
||||||
|
releasing a blob for fenced garbage collection.
|
||||||
|
|
||||||
### Recovery ledger for object effects
|
### Recovery ledger for object effects
|
||||||
|
|
||||||
Every managed blob creation or integrity repair starts a Core recovery
|
On PostgreSQL, every managed blob creation or integrity repair starts a Core
|
||||||
operation in an independent committed transaction before Files protects or
|
recovery operation in an independent committed transaction before Files
|
||||||
writes bytes. The operation records tenant/blob identifiers, an opaque object
|
protects or writes bytes. The operation records tenant/blob identifiers, an opaque object
|
||||||
locator or locator digest, semantic SHA-256/size evidence, the recovery mode,
|
locator or locator digest, semantic SHA-256/size evidence, the recovery mode,
|
||||||
and a distributed lease fence. It never records file contents, ZIP passwords,
|
and a distributed lease fence. It never records file contents, ZIP passwords,
|
||||||
connector credentials, or a newly uploaded filename. New object keys are opaque;
|
connector credentials, or a newly uploaded filename. New object keys are opaque;
|
||||||
legacy filename-bearing keys remain readable but repair operations record only
|
legacy filename-bearing keys remain readable but repair operations record only
|
||||||
their digest and recover through the blob ID.
|
their digest and recover through the blob ID.
|
||||||
|
|
||||||
|
SQLite is a supported local-development database but permits only one writer.
|
||||||
|
Files therefore uses an explicit reduced-durability mode there: recovery intent
|
||||||
|
and its lease are written in the caller transaction, while a process-local
|
||||||
|
fence prevents competing effects in the same runtime. Commit makes the intent
|
||||||
|
durable before independent verification. A handled rollback reconstructs a
|
||||||
|
durable recovery operation and verifies compensation or forward completion.
|
||||||
|
A hard process loss before commit can leave an object without a surviving
|
||||||
|
ledger row, so SQLite is not a production recovery profile; after such a loss,
|
||||||
|
run a complete Files integrity scan and reconcile every reported orphan before
|
||||||
|
resuming writes. PostgreSQL retains the independent pre-effect durability
|
||||||
|
guarantee.
|
||||||
|
|
||||||
The Files business transaction then creates or updates the blob, version, and
|
The Files business transaction then creates or updates the blob, version, and
|
||||||
asset rows. Its actual SQLAlchemy commit or rollback settles every pending
|
asset rows. Its actual SQLAlchemy commit or rollback settles every pending
|
||||||
operation:
|
operation:
|
||||||
@@ -560,6 +698,22 @@ and the durable finding state are verified afterward. If the caller transaction
|
|||||||
rolls back after deletion, Files may forward-complete only that existing
|
rolls back after deletion, Files may forward-complete only that existing
|
||||||
finding after rechecking that the key is still unreferenced.
|
finding after rechecking that the key is still unreferenced.
|
||||||
|
|
||||||
|
Hard purge uses an irreversible Core recovery plan with an approval reference
|
||||||
|
and a tenant-wide purge lease. The preview hash binds target IDs, deletion
|
||||||
|
state, lifecycle revisions, holds, deadlines, blockers, and blob IDs. Asset and
|
||||||
|
version removal plus audit evidence commit together. Released objects are not
|
||||||
|
deleted by that transaction. Blob GC later takes `files:blob:<tenant>:<blob>`,
|
||||||
|
the same distributed resource fence as upload/repair, locks and rechecks the
|
||||||
|
blob, and verifies object absence before removing metadata. Provider or database
|
||||||
|
ambiguity remains recovery-required or outcome-unknown in Ops.
|
||||||
|
|
||||||
|
Explicit S3 connector writes use forward recovery because the provider cannot
|
||||||
|
join the database transaction. The durable request contains only tenant,
|
||||||
|
profile, opaque target digest, content SHA-256/size, and expected revision. A
|
||||||
|
conditional create or overwrite is followed by a metadata probe for the exact
|
||||||
|
content and operation markers. Never retry an unresolved target with another
|
||||||
|
request key; reconcile the owning Files operation from provider evidence first.
|
||||||
|
|
||||||
Archive preview and confirmation use bounded process-local temporary staging.
|
Archive preview and confirmation use bounded process-local temporary staging.
|
||||||
Staging is not authoritative and is removed on every handled exit; extracted
|
Staging is not authoritative and is removed on every handled exit; extracted
|
||||||
members enter the same per-blob recovery boundary as direct uploads. A hard
|
members enter the same per-blob recovery boundary as direct uploads. A hard
|
||||||
@@ -567,11 +721,9 @@ process loss may leave a temporary OS file for normal host temporary-file
|
|||||||
cleanup, but cannot make that staging path a managed Files object.
|
cleanup, but cannot make that staging path a managed Files object.
|
||||||
|
|
||||||
Use the Ops recovery-operation view to inspect `files` operations. Do not retry
|
Use the Ops recovery-operation view to inspect `files` operations. Do not retry
|
||||||
a busy or unresolved blob blindly: first verify the FileBlob row, object hash,
|
a busy or unresolved blob or connector path blindly: first verify the FileBlob
|
||||||
integrity state, and any Encryption envelope named by the blob. Hard purge,
|
row and object hash, or the remote request/content markers and revision, plus
|
||||||
legal hold, and two-way remote connector mutation are not implemented yet, so
|
any Encryption envelope named by a managed blob.
|
||||||
they cannot claim recovery-ledger adoption; their owning work remains tracked
|
|
||||||
separately.
|
|
||||||
|
|
||||||
After restore:
|
After restore:
|
||||||
|
|
||||||
@@ -613,7 +765,7 @@ Use these symptoms as routing hints:
|
|||||||
| `Stored object does not exist` | Database/blob restore mismatch, wrong root, or missing shared storage |
|
| `Stored object does not exist` | Database/blob restore mismatch, wrong root, or missing shared storage |
|
||||||
| `Stored secret cannot be decrypted` | Wrong or rotated `MASTER_KEY_B64` |
|
| `Stored secret cannot be decrypted` | Wrong or rotated `MASTER_KEY_B64` |
|
||||||
| Private/non-public endpoint blocked | Deployment-wide egress policy is public-only or DNS returned a forbidden answer |
|
| Private/non-public endpoint blocked | Deployment-wide egress policy is public-only or DNS returned a forbidden answer |
|
||||||
| SDK cannot pin redirects/referrals | Expected fail-closed S3/SMB boundary, not a transient connector outage |
|
| SDK peer-pinning seam is unavailable | Optional S3/SMB SDK is incompatible; keep access fail-closed and validate the supported dependency range before upgrade |
|
||||||
| Connector response exceeds limit | Remote payload exceeds connector or upload limit |
|
| Connector response exceeds limit | Remote payload exceeds connector or upload limit |
|
||||||
| Profile is not visible | Scope, disabled state, campaign access, or policy mismatch |
|
| Profile is not visible | Scope, disabled state, campaign access, or policy mismatch |
|
||||||
| Group removal is vetoed | The group still owns a file/folder/connector space or is a share target |
|
| Group removal is vetoed | The group still owns a file/folder/connector space or is a share target |
|
||||||
@@ -629,11 +781,26 @@ public HTTP API. They must not import Files ORM models or storage helpers.
|
|||||||
| --- | --- |
|
| --- | --- |
|
||||||
| `files.access` (`0.1.6`) | Explain resource access provenance for managed files, explicit folders, and virtual folders |
|
| `files.access` (`0.1.6`) | Explain resource access provenance for managed files, explicit folders, and virtual folders |
|
||||||
| `files.campaign_attachments` (`0.1.6`) | Resolve managed attachment matches, prepare frozen campaign snapshots, annotate built messages, share assets with a campaign, and record/mark exact attachment use |
|
| `files.campaign_attachments` (`0.1.6`) | Resolve managed attachment matches, prepare frozen campaign snapshots, annotate built messages, share assets with a campaign, and record/mark exact attachment use |
|
||||||
|
| `records.source.files` (`1.0.0`) | Recheck current Files access and resolve one exact, integrity-approved managed file version for Records filing |
|
||||||
|
| `forms_runtime.evidence.files` (`1.0.0`) | Issue a one-time managed attachment grant and re-verify the exact file/version/checksum at Form submission |
|
||||||
|
|
||||||
Files requires Core principal resolution and permission evaluation. Campaign is
|
Files requires Core principal resolution and permission evaluation. Campaign is
|
||||||
an optional dependency; when installed, Files consumes the optional
|
an optional dependency; when installed, Files consumes the optional
|
||||||
`campaigns.access` interface to verify campaign existence and access. Missing
|
`campaigns.access` interface to verify campaign existence and access. Missing
|
||||||
optional Campaign support fails explicitly rather than bypassing the check.
|
optional Campaign support fails explicitly rather than bypassing the check.
|
||||||
|
Records is also optional. When enabled, the source capability returns the
|
||||||
|
requested `FileVersion` identity, path snapshot, content metadata, SHA-256,
|
||||||
|
integrity/protection state, and launch link. It rejects mutable aliases,
|
||||||
|
cross-tenant requests, missing access, and quarantined or failed blobs. Records
|
||||||
|
stores the filing decision; Files continues to own the version and bytes.
|
||||||
|
Forms Runtime is optional as well. Its public or authenticated participant never
|
||||||
|
receives general Files access through this integration. Forms Runtime asks for a
|
||||||
|
purpose-bound grant, Files verifies an active same-tenant user custodian, stores
|
||||||
|
only the token digest, accepts one bounded upload, and returns an immutable
|
||||||
|
`EvidenceReference`. Draft save and final submit independently recheck the exact
|
||||||
|
Form instance/revision, grant, asset, version, checksum, deletion state, and
|
||||||
|
integrity state. An idempotent replay returns the existing grant without
|
||||||
|
reproducing its bearer secret.
|
||||||
|
|
||||||
### API families
|
### API families
|
||||||
|
|
||||||
@@ -646,13 +813,28 @@ All routes below are under `/api/v1/files`.
|
|||||||
| File access | `GET /{file_id}`, `GET /{file_id}/download`, `DELETE /{file_id}`, `POST /bulk-delete` |
|
| File access | `GET /{file_id}`, `GET /{file_id}/download`, `DELETE /{file_id}`, `POST /bulk-delete` |
|
||||||
| Organization | `POST /bulk-rename`, `POST /transfer`, `POST /archive.zip`, `POST /resolve-patterns` |
|
| Organization | `POST /bulk-rename`, `POST /transfer`, `POST /archive.zip`, `POST /resolve-patterns` |
|
||||||
| Sharing | `POST /{file_id}/shares`, `POST /bulk-shares` |
|
| Sharing | `POST /{file_id}/shares`, `POST /bulk-shares` |
|
||||||
| Connector spaces | `GET/POST /connector-spaces`, `PATCH/DELETE /connector-spaces/{space_id}` |
|
| Connector spaces | `GET/POST /connector-spaces`, `PATCH/DELETE /connector-spaces/{space_id}`, `POST /connector-spaces/{space_id}/restore`, `POST /connector-spaces/{space_id}/write-back` |
|
||||||
| Connector catalog/discovery | `GET /connectors/providers`, `POST /connectors/discover` |
|
| Connector catalog/discovery | `GET /connectors/providers`, `POST /connectors/discover` |
|
||||||
| Connector profiles | `GET/POST /connectors/profiles`, `GET/PATCH/DELETE /connectors/profiles/{profile_id}` |
|
| Connector profiles | `GET/POST /connectors/profiles`, `GET/PATCH/DELETE /connectors/profiles/{profile_id}` |
|
||||||
| Browse/import/sync | `GET /connectors/profiles/{profile_id}/browse`, `POST /connectors/profiles/{profile_id}/import`, `POST /connectors/profiles/{profile_id}/sync` |
|
| Browse/import/sync | `GET /connectors/profiles/{profile_id}/browse`, `POST /connectors/profiles/{profile_id}/import`, `POST /connectors/profiles/{profile_id}/sync` |
|
||||||
|
| Restore and lifecycle | `POST /assets/{file_id}/restore`, `POST /folders/restore`, `PATCH /{file_id}/lifecycle` |
|
||||||
|
| Governed erasure | `POST /purge/preview`, `POST /purge/execute`, `POST /purge/blobs` |
|
||||||
| Credentials | `GET/POST /connectors/credentials`, `GET/PATCH/DELETE /connectors/credentials/{credential_id}` |
|
| Credentials | `GET/POST /connectors/credentials`, `GET/PATCH/DELETE /connectors/credentials/{credential_id}` |
|
||||||
| Policy | `GET/PUT /connectors/policies/{scope_type}`, `POST /connector-policy/evaluate` |
|
| Policy | `GET/PUT /connectors/policies/{scope_type}`, `POST /connector-policy/evaluate` |
|
||||||
| Incremental connector settings | `GET /connectors/settings/delta` |
|
| Incremental connector settings | `GET /connectors/settings/delta` |
|
||||||
|
| Form evidence | `POST /form-evidence/upload` with a short-lived `X-Form-Evidence-Token` issued by Forms Runtime |
|
||||||
|
|
||||||
|
The Files workspace exposes **Remove space** only for connector spaces and only
|
||||||
|
to actors with file-organization authority over the owning user
|
||||||
|
or group space. Confirmation explains the exact boundary: removal soft-deletes
|
||||||
|
the local connector-space definition and makes that virtual view disappear. It
|
||||||
|
does not mutate or delete remote provider content, previously imported managed
|
||||||
|
files, their metadata or shares, the connector profile, credentials, or remote
|
||||||
|
object references. Those retained objects therefore do not block removal and
|
||||||
|
the connector location can be linked again later. User and group managed spaces
|
||||||
|
are intrinsic ownership scopes rather than removable records, so they never
|
||||||
|
offer this action. A missing, already removed, cross-tenant, or inaccessible
|
||||||
|
connector space fails through the backend lookup and owner-access checks.
|
||||||
|
|
||||||
Consumers should use cursor/watermark contracts instead of assuming an
|
Consumers should use cursor/watermark contracts instead of assuming an
|
||||||
unbounded complete list. The default full-list page size is 500 and public page
|
unbounded complete list. The default full-list page size is 500 and public page
|
||||||
@@ -690,6 +872,9 @@ Files baseline indiscriminately.
|
|||||||
- Upload, archive extraction, connector response, and S3 stream code use bounded
|
- Upload, archive extraction, connector response, and S3 stream code use bounded
|
||||||
reads. Archive previews are sealed and short-lived; ZIP passwords are
|
reads. Archive previews are sealed and short-lived; ZIP passwords are
|
||||||
request-only.
|
request-only.
|
||||||
|
- Public Form evidence uploads require a custom-header bearer grant that is
|
||||||
|
stored only as SHA-256, expires after at most 15 minutes, is bound to one
|
||||||
|
exact submission and user custodian, and can create only one managed file.
|
||||||
- Connector HTTP sockets use connection-time DNS/IP validation and pinning,
|
- Connector HTTP sockets use connection-time DNS/IP validation and pinning,
|
||||||
redirects are refused, and unsafe SDK transports fail before client creation.
|
redirects are refused, and unsafe SDK transports fail before client creation.
|
||||||
- Database-managed connector passwords/tokens are encrypted; responses redact
|
- Database-managed connector passwords/tokens are encrypted; responses redact
|
||||||
@@ -762,6 +947,9 @@ The word "delete" has different meanings by object type:
|
|||||||
| Connector profile | Immediately disables the tombstone and clears credential links/material/references and private metadata |
|
| Connector profile | Immediately disables the tombstone and clears credential links/material/references and private metadata |
|
||||||
| Legacy `secret_ref` | Detached and audited as an unowned external reference; no provider deletion is attempted or claimed |
|
| Legacy `secret_ref` | Detached and audited as an unowned external reference; no provider deletion is attempted or claimed |
|
||||||
| Module retirement | Scrubs/audits credential material, then drops Files database tables; blob-backend cleanup is an operator responsibility |
|
| Module retirement | Scrubs/audits credential material, then drops Files database tables; blob-backend cleanup is an operator responsibility |
|
||||||
|
| Governed hard purge | Removes only preview-matched, soft-deleted asset/version rows without active retention, legal hold, active shares, Campaign evidence, or Form evidence |
|
||||||
|
| Blob garbage collection | Deletes an exact managed object only after a fresh zero-reference check under the shared blob fence; metadata follows verified object absence |
|
||||||
|
| Connector write-back | Explicit conditional S3 create/overwrite only; automatic provider delete, rename, move, and ACL propagation remain disabled |
|
||||||
|
|
||||||
Credential/profile scrubbing and its audit event use the same database
|
Credential/profile scrubbing and its audit event use the same database
|
||||||
transaction. If audit creation fails, the deletion rolls back. Repeating a
|
transaction. If audit creation fails, the deletion rolls back. Repeating a
|
||||||
@@ -769,16 +957,19 @@ delete against an already scrubbed tombstone does not recreate secret evidence.
|
|||||||
|
|
||||||
### Retention boundary
|
### Retention boundary
|
||||||
|
|
||||||
File versions and blobs are effectively retained indefinitely today. Although a
|
Each asset has an enforceable retained-until value, legal-hold flag, reason, and
|
||||||
blob has `ref_count` and `retained_until` fields, no complete retention-policy,
|
optimistic lifecycle revision. These controls block hard purge; they do not
|
||||||
legal-hold, hard-purge, or garbage-collection service enforces them. There is
|
automatically schedule it. Restore, purge preview/execute, and blob GC are
|
||||||
also no supported user restore endpoint for soft-deleted assets/folders.
|
explicit authorized operations. Blob `retained_until` remains an additional
|
||||||
Integrity reconciliation is operator-triggered and is not a retention or
|
storage-level safeguard and must also have expired before automated collection
|
||||||
automatic garbage-collection policy.
|
is introduced. Campaign and Form evidence are hard blockers rather than
|
||||||
|
silently cascaded references.
|
||||||
|
|
||||||
Do not promise erasure, timed retention, legal hold, or self-service recovery
|
Do not equate soft deletion with erasure. Erasure is complete only after the
|
||||||
from the current soft-delete behavior. Those require an explicit, auditable
|
approved purge removes asset/version metadata, bounded GC verifies that no
|
||||||
retention/purge design that preserves campaign and other evidence references.
|
version references the blob and removes its bytes/metadata, and any owning
|
||||||
|
Encryption retention/key-custody consequence has been handled under that
|
||||||
|
module's policy.
|
||||||
|
|
||||||
## Acceptance scenarios
|
## Acceptance scenarios
|
||||||
|
|
||||||
@@ -817,9 +1008,11 @@ bypass the same rule.
|
|||||||
### Pinned connector transport
|
### Pinned connector transport
|
||||||
|
|
||||||
Given public-only mode and DNS returns any private address, the connection must
|
Given public-only mode and DNS returns any private address, the connection must
|
||||||
be rejected before a socket opens. Given private mode, the HTTP connection must
|
be rejected before a socket opens. Given private mode, every HTTP, S3, and SMB
|
||||||
still use the validated address and reject redirects. SMB and S3 must fail
|
connection must still use an address from the answer validated for that exact
|
||||||
before their SDK clients connect until all SDK-managed peers can be pinned.
|
attempt. Botocore retries, redirects, endpoint discovery, and aliases, plus SMB
|
||||||
|
reconnects and DFS referrals, must pass through the pinned factories. A changed
|
||||||
|
or unsupported SDK seam must fail before a usable client/session is returned.
|
||||||
|
|
||||||
### Imported evidence and sync
|
### Imported evidence and sync
|
||||||
|
|
||||||
@@ -855,18 +1048,20 @@ returning different content or credentials.
|
|||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| Managed storage | Core local/S3 backend, exact managed-Garage or explicitly trusted HTTPS external S3, state-profile validation, fallback local read roots, tenant blob deduplication, checksums, bounded resumable integrity scans, quarantine, dry-run-first orphan cleanup, and Core-ledger verification/forward recovery | Scheduled scan execution and deployment-specific S3 HA/backup automation |
|
| Managed storage | Core local/S3 backend, exact managed-Garage or explicitly trusted HTTPS external S3, state-profile validation, fallback local read roots, tenant blob deduplication, checksums, bounded resumable integrity scans, quarantine, dry-run-first orphan cleanup, and Core-ledger verification/forward recovery | Scheduled scan execution and deployment-specific S3 HA/backup automation |
|
||||||
| Upload | Bounded direct upload, drag-and-drop UI, archive preview/selective extraction, password-protected ZIP support, explicit conflicts, opaque new object keys, and rollback compensation | Malware scanning, quotas, type policy, resumable/chunked upload |
|
| Upload | Bounded direct upload, drag-and-drop UI, archive preview/selective extraction, password-protected ZIP support, explicit conflicts, opaque new object keys, and rollback compensation | Malware scanning, quotas, type policy, resumable/chunked upload |
|
||||||
| Organization | Folders, bulk rename preview/apply, move/copy, drag-and-drop, ZIP download, pattern resolution | General file-history UI and user-driven append-version/restore |
|
| Organization | Folders, bulk rename preview/apply, move/copy, drag-and-drop, ZIP download, pattern resolution, and API restoration preserving versions/provenance | General file-history UI and user-driven append-version UI |
|
||||||
| Sharing | User/group/tenant/campaign grants, expiry, idempotent revocation, searchable share-management UI, and campaign linkage display | Richer policy-driven share lifecycles |
|
| Sharing | User/group/tenant/campaign grants, expiry, idempotent revocation, searchable share-management UI, and campaign linkage display | Richer policy-driven share lifecycles |
|
||||||
| Deletion/retention | Soft-delete assets/folders/spaces; immediate audited connector-secret scrubbing | File restore API, hard purge, retention policy, legal hold, and blob GC ([#38](https://git.add-ideas.de/GovOPlaN/govoplan-files/issues/38)) |
|
| Deletion/retention | Soft-delete and restore assets/folders/spaces; optimistic retention and legal-hold controls; preview-bound, approval-referenced hard purge; reference-checked blob GC; immediate audited connector-secret scrubbing | Automatic time-based purge scheduling and richer lifecycle administration UI |
|
||||||
|
| Privacy requests | Tenant-scoped bounded DSAR metadata search; retained/manual/revoke/detach planning; idempotent share revocation and mutable actor-reference detachment; explicit separation from byte purge | Content-specific automated redaction and policy-specific approval remain manual or belong to the owning process |
|
||||||
| Connector governance | Scoped profiles/credentials/policies, effective source explanation, separate credentials, linked user/group spaces | Provider-owned external secret lifecycle; API `secret_ref` remains rejected |
|
| Connector governance | Scoped profiles/credentials/policies, effective source explanation, separate credentials, linked user/group spaces | Provider-owned external secret lifecycle; API `secret_ref` remains rejected |
|
||||||
| HTTP connectors | Pinned, bounded, no-redirect Seafile and WebDAV/Nextcloud browse/import/manual sync | Background/folder sync, remote mutation, long-running transfer workers |
|
| HTTP connectors | Pinned, bounded, no-redirect Seafile and WebDAV/Nextcloud browse/import/manual sync | Background/folder sync, remote mutation, long-running transfer workers |
|
||||||
| SMB and S3 connectors | Provider descriptors and browse/import logic | Live access only after SDK connections plus DFS referrals/redirects are validated and pinned ([SMB #35](https://git.add-ideas.de/GovOPlaN/govoplan-files/issues/35), [S3 #34](https://git.add-ideas.de/GovOPlaN/govoplan-files/issues/34)) |
|
| SMB and S3 connectors | Provider descriptors, browse/import/manual sync, pinned SDK transports, redirect/retry/referral transport-contract tests, and explicit conditional S3 write-back with Core-ledger recovery | Live topology smoke evidence, provider-specific OAuth, additional provider writes, and background indexing remain separate deployment or connector-module concerns |
|
||||||
| Other providers | Reserved SharePoint/OneDrive keys and NFS/local descriptors | Graph/OAuth/provider paging, NFS deployment integration, DMS connectors |
|
| Other providers | Reserved SharePoint/OneDrive keys and NFS/local descriptors | Graph/OAuth/provider paging, NFS deployment integration, DMS connectors |
|
||||||
| Connector spaces | User/group link, browse, manual selected-file sync, edit/disable/delete | Background sync, remote writes/deletes, full conflict-reporting jobs |
|
| Connector spaces | User/group link, browse, manual selected-file sync, edit/disable/delete/restore, read-only default, and opt-in S3 two-way mode | Background sync and automatic remote delete/rename/move/ACL propagation |
|
||||||
| Profile capabilities | Stored and displayed | Enforce capability flags as an independent operation gate |
|
| Profile capabilities | Stored, displayed, and enforced for explicit connector write-back | Broader provider-specific capability negotiation |
|
||||||
| Audit | Connector discovery/import/sync/access and connector deletion; campaign exact-use evidence | Dedicated canonical audit events for every ordinary Files mutation |
|
| Audit | Connector discovery/import/sync/access and connector deletion; campaign exact-use evidence | Dedicated canonical audit events for every ordinary Files mutation |
|
||||||
| Preview | File metadata and attachment download | Dedicated safe content-preview service |
|
| Preview | File metadata and attachment download | Dedicated safe content-preview service |
|
||||||
| Campaign | Stable capability-based frozen attachments and sent-use evidence | Campaign-specific process state remains in Campaign |
|
| Campaign | Stable capability-based frozen attachments and sent-use evidence | Campaign-specific process state remains in Campaign |
|
||||||
|
| Forms Runtime | One-time managed attachment grants plus exact-version, checksum, deletion, tenant, submission, and integrity verification | Malware scanning and advanced/qualified signature providers remain separate assurance depth |
|
||||||
| Collaboration | Governed input/output snapshots | Co-editing, comments, review, presence, locks, and semantic document versions belong to Documents/workflow/provider modules |
|
| Collaboration | Governed input/output snapshots | Co-editing, comments, review, presence, locks, and semantic document versions belong to Documents/workflow/provider modules |
|
||||||
|
|
||||||
## Release and change checklist
|
## Release and change checklist
|
||||||
@@ -883,14 +1078,19 @@ Before releasing Files:
|
|||||||
6. Exercise upload, ZIP bounds, conflict handling, download, and soft deletion.
|
6. Exercise upload, ZIP bounds, conflict handling, download, and soft deletion.
|
||||||
7. Exercise connector policy explanation and one pinned HTTP provider where
|
7. Exercise connector policy explanation and one pinned HTTP provider where
|
||||||
configured; verify the deployment-managed or trusted external S3 backend if
|
configured; verify the deployment-managed or trusted external S3 backend if
|
||||||
selected, and verify user-configured S3 and SMB connector peers still fail
|
selected, and verify one configured S3 and SMB connector while recording the
|
||||||
closed.
|
actual target topology and private-network policy.
|
||||||
8. Verify credential deletion scrubs dependents and produces audit evidence.
|
8. Verify credential deletion scrubs dependents and produces audit evidence.
|
||||||
9. Verify a campaign attachment snapshot still identifies its exact version and
|
9. Verify a campaign attachment snapshot still identifies its exact version and
|
||||||
checksum after the current file changes.
|
checksum after the current file changes.
|
||||||
10. Exercise a committed upload, a rolled-back upload, object tamper detection,
|
10. Exercise a committed upload, a rolled-back upload, object tamper detection,
|
||||||
and applied orphan cleanup; inspect their `files` operations in Ops.
|
and applied orphan cleanup; inspect their `files` operations in Ops.
|
||||||
11. Update the implemented/planned table whenever a boundary changes.
|
11. Exercise a Form evidence grant, token replay, wrong-submission reference,
|
||||||
|
expired token, unsupported media type, and quarantined-file rejection.
|
||||||
|
12. Exercise a Files DSAR search/plan, repeat an approved reversible action, and
|
||||||
|
confirm retained file content can only be erased through the separate purge
|
||||||
|
authority and recovery path.
|
||||||
|
13. Update the implemented/planned table whenever a boundary changes.
|
||||||
|
|
||||||
## Related documents
|
## Related documents
|
||||||
|
|
||||||
|
|||||||
+8
-8
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@govoplan/files-webui",
|
"name": "@govoplan/files-webui",
|
||||||
"version": "0.1.9",
|
"version": "0.1.19",
|
||||||
"private": true,
|
"private": true,
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"main": "webui/src/index.ts",
|
"main": "webui/src/index.ts",
|
||||||
@@ -19,14 +19,14 @@
|
|||||||
"LICENSE"
|
"LICENSE"
|
||||||
],
|
],
|
||||||
"peerDependencies": {
|
"peerDependencies": {
|
||||||
"@govoplan/core-webui": "^0.1.9",
|
"@vitejs/plugin-react": "^5.2.0",
|
||||||
"lucide-react": "^1.23.0",
|
"vite": "^7.3.6",
|
||||||
"react": "^19.0.0",
|
|
||||||
"react-dom": "^19.0.0",
|
|
||||||
"react-router-dom": "^7.1.1",
|
|
||||||
"@vitejs/plugin-react": "^4.3.4",
|
|
||||||
"typescript": "^5.7.2",
|
"typescript": "^5.7.2",
|
||||||
"vite": "^6.0.6"
|
"react": ">=19.2.7 <20",
|
||||||
|
"react-dom": ">=19.2.7 <20",
|
||||||
|
"react-router": ">=8.3.0 <9",
|
||||||
|
"lucide-react": "^1.23.0",
|
||||||
|
"@govoplan/core-webui": "^0.1.18"
|
||||||
},
|
},
|
||||||
"peerDependenciesMeta": {
|
"peerDependenciesMeta": {
|
||||||
"@govoplan/core-webui": {
|
"@govoplan/core-webui": {
|
||||||
|
|||||||
+2
-2
@@ -4,14 +4,14 @@ build-backend = "setuptools.build_meta"
|
|||||||
|
|
||||||
[project]
|
[project]
|
||||||
name = "govoplan-files"
|
name = "govoplan-files"
|
||||||
version = "0.1.9"
|
version = "0.1.19"
|
||||||
description = "GovOPlaN files module with backend and WebUI integration."
|
description = "GovOPlaN files module with backend and WebUI integration."
|
||||||
readme = "README.md"
|
readme = "README.md"
|
||||||
requires-python = ">=3.12"
|
requires-python = ">=3.12"
|
||||||
license = { file = "LICENSE" }
|
license = { file = "LICENSE" }
|
||||||
authors = [{ name = "GovOPlaN" }]
|
authors = [{ name = "GovOPlaN" }]
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"govoplan-core>=0.1.14",
|
"govoplan-core>=0.1.19",
|
||||||
"defusedxml>=0.7,<1",
|
"defusedxml>=0.7,<1",
|
||||||
"pyzipper>=0.3.6,<1",
|
"pyzipper>=0.3.6,<1",
|
||||||
"python-multipart>=0.0.31,<1",
|
"python-multipart>=0.0.31,<1",
|
||||||
|
|||||||
@@ -5,18 +5,32 @@ import binascii
|
|||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from sqlalchemy import or_
|
from sqlalchemy import or_
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from govoplan_core.audit.logging import audit_from_principal
|
||||||
|
from govoplan_core.auth import ApiPrincipal, has_scope
|
||||||
from govoplan_core.core.access import AccessDecisionProvenance, PrincipalRef
|
from govoplan_core.core.access import AccessDecisionProvenance, PrincipalRef
|
||||||
from govoplan_core.core.files import FileAccessProvider
|
from govoplan_core.core.files import FileAccessProvider
|
||||||
from govoplan_core.core.files import (
|
from govoplan_core.core.files import (
|
||||||
|
ManagedTabularFile,
|
||||||
|
ManagedTabularFileAccessError,
|
||||||
|
ManagedTabularFileContent,
|
||||||
|
ManagedTabularFileNotFoundError,
|
||||||
|
ManagedTabularFileProvider,
|
||||||
|
ManagedTabularFileUnavailableError,
|
||||||
|
ManagedTabularFileValidationError,
|
||||||
|
PostboxFileReferenceRef,
|
||||||
|
PostboxFileReferenceRequest,
|
||||||
|
PostboxFileReferenceProvider,
|
||||||
ManagedArtifactRef,
|
ManagedArtifactRef,
|
||||||
ManagedArtifactStore,
|
ManagedArtifactStore,
|
||||||
ManagedArtifactWriteRequest,
|
ManagedArtifactWriteRequest,
|
||||||
)
|
)
|
||||||
from govoplan_core.core.modules import ModuleContext
|
from govoplan_core.core.modules import ModuleContext
|
||||||
from govoplan_core.security.module_permissions import scopes_grant_compatible
|
from govoplan_core.security.module_permissions import scopes_grant_compatible
|
||||||
from govoplan_files.backend.db.models import FileAsset, FileFolder, FileShare
|
from govoplan_files.backend.db.models import FileAsset, FileBlob, FileFolder, FileShare, FileVersion
|
||||||
from govoplan_files.backend.runtime import configure_runtime
|
from govoplan_files.backend.runtime import configure_runtime
|
||||||
|
from govoplan_files.backend.storage.common import FileStorageError
|
||||||
from govoplan_files.backend.storage.campaign_attachments import (
|
from govoplan_files.backend.storage.campaign_attachments import (
|
||||||
annotate_built_messages_with_managed_files,
|
annotate_built_messages_with_managed_files,
|
||||||
managed_match_payloads,
|
managed_match_payloads,
|
||||||
@@ -28,6 +42,9 @@ from govoplan_files.backend.storage.campaign_usage import record_campaign_attach
|
|||||||
from govoplan_files.backend.storage.files import (
|
from govoplan_files.backend.storage.files import (
|
||||||
create_file_asset,
|
create_file_asset,
|
||||||
current_version_and_blob,
|
current_version_and_blob,
|
||||||
|
get_asset_for_user,
|
||||||
|
list_recent_assets_for_user,
|
||||||
|
read_asset_version_bytes,
|
||||||
sync_file_asset_from_source,
|
sync_file_asset_from_source,
|
||||||
)
|
)
|
||||||
from govoplan_files.backend.storage.paths import normalize_folder
|
from govoplan_files.backend.storage.paths import normalize_folder
|
||||||
@@ -146,6 +163,137 @@ def artifact_store_capability(context: ModuleContext) -> FilesArtifactStore:
|
|||||||
return FilesArtifactStore()
|
return FilesArtifactStore()
|
||||||
|
|
||||||
|
|
||||||
|
class FilesPostboxReferenceService(PostboxFileReferenceProvider):
|
||||||
|
"""Resolve Postbox evidence without turning message access into file access."""
|
||||||
|
|
||||||
|
def resolve_postbox_references(
|
||||||
|
self,
|
||||||
|
session: object,
|
||||||
|
principal: object,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
requests: tuple[PostboxFileReferenceRequest, ...],
|
||||||
|
) -> tuple[PostboxFileReferenceRef, ...]:
|
||||||
|
if not hasattr(session, "get"):
|
||||||
|
raise TypeError("Postbox file resolution requires a SQLAlchemy session.")
|
||||||
|
principal_tenant = str(getattr(principal, "tenant_id", "") or "")
|
||||||
|
can_download = bool(
|
||||||
|
hasattr(principal, "has") and principal.has("files:file:download")
|
||||||
|
)
|
||||||
|
user = getattr(principal, "user", None)
|
||||||
|
user_id = str(getattr(user, "id", "") or "")
|
||||||
|
is_admin = bool(
|
||||||
|
hasattr(principal, "has") and principal.has("files:file:admin")
|
||||||
|
)
|
||||||
|
results: list[PostboxFileReferenceRef] = []
|
||||||
|
for request in requests:
|
||||||
|
if principal_tenant != tenant_id:
|
||||||
|
results.append(_unavailable_postbox_reference(request, "tenant_mismatch"))
|
||||||
|
continue
|
||||||
|
if not can_download or not user_id:
|
||||||
|
results.append(
|
||||||
|
_unavailable_postbox_reference(
|
||||||
|
request,
|
||||||
|
"download_permission_missing",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
reference_type = request.reference_type.strip().casefold()
|
||||||
|
version: FileVersion | None = None
|
||||||
|
if reference_type in {"file", "file_asset", "files:file"}:
|
||||||
|
results.append(
|
||||||
|
_unavailable_postbox_reference(request, "exact_version_required")
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
elif reference_type in {"file_version", "files:file_version"}:
|
||||||
|
version = session.get(FileVersion, request.reference_id) # type: ignore[attr-defined]
|
||||||
|
asset = (
|
||||||
|
session.get(FileAsset, version.file_asset_id) # type: ignore[attr-defined]
|
||||||
|
if version is not None
|
||||||
|
else None
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
results.append(
|
||||||
|
_unavailable_postbox_reference(
|
||||||
|
request,
|
||||||
|
"unsupported_reference_type",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
if (
|
||||||
|
asset is None
|
||||||
|
or version is None
|
||||||
|
or asset.tenant_id != tenant_id
|
||||||
|
or version.tenant_id != tenant_id
|
||||||
|
or asset.deleted_at is not None
|
||||||
|
):
|
||||||
|
results.append(_unavailable_postbox_reference(request, "file_not_found"))
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
get_asset_for_user(
|
||||||
|
session, # type: ignore[arg-type]
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
user_id=user_id,
|
||||||
|
asset_id=asset.id,
|
||||||
|
is_admin=is_admin,
|
||||||
|
)
|
||||||
|
except FileStorageError:
|
||||||
|
results.append(
|
||||||
|
_unavailable_postbox_reference(request, "file_access_denied")
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
blob = session.get(FileBlob, version.blob_id) # type: ignore[attr-defined]
|
||||||
|
if blob is None or blob.tenant_id != tenant_id:
|
||||||
|
results.append(
|
||||||
|
_unavailable_postbox_reference(request, "file_payload_missing")
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
results.append(
|
||||||
|
PostboxFileReferenceRef(
|
||||||
|
reference_type=request.reference_type,
|
||||||
|
reference_id=request.reference_id,
|
||||||
|
available=True,
|
||||||
|
reason_code="available",
|
||||||
|
file_asset_id=asset.id,
|
||||||
|
file_version_id=version.id,
|
||||||
|
filename=version.filename_at_upload or asset.filename,
|
||||||
|
content_type=version.content_type or blob.content_type,
|
||||||
|
size_bytes=version.size_bytes,
|
||||||
|
sha256=version.checksum_sha256,
|
||||||
|
download_path=(
|
||||||
|
f"/api/v1/files/{asset.id}/versions/{version.id}/download"
|
||||||
|
),
|
||||||
|
provenance={
|
||||||
|
"module": "files",
|
||||||
|
"postbox_id": request.postbox_id,
|
||||||
|
"message_id": request.message_id,
|
||||||
|
"display_path": asset.display_path,
|
||||||
|
"exact_version": True,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return tuple(results)
|
||||||
|
|
||||||
|
|
||||||
|
def _unavailable_postbox_reference(
|
||||||
|
request: PostboxFileReferenceRequest,
|
||||||
|
reason_code: str,
|
||||||
|
) -> PostboxFileReferenceRef:
|
||||||
|
return PostboxFileReferenceRef(
|
||||||
|
reference_type=request.reference_type,
|
||||||
|
reference_id=request.reference_id,
|
||||||
|
available=False,
|
||||||
|
reason_code=reason_code,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def postbox_reference_capability(
|
||||||
|
context: ModuleContext,
|
||||||
|
) -> FilesPostboxReferenceService:
|
||||||
|
configure_runtime(registry=context.registry, settings=context.settings)
|
||||||
|
return FilesPostboxReferenceService()
|
||||||
|
|
||||||
|
|
||||||
class FilesAccessService(FileAccessProvider):
|
class FilesAccessService(FileAccessProvider):
|
||||||
def explain_resource_provenance(
|
def explain_resource_provenance(
|
||||||
self,
|
self,
|
||||||
@@ -284,6 +432,228 @@ def access_capability(context: ModuleContext) -> FilesAccessService:
|
|||||||
return FilesAccessService()
|
return FilesAccessService()
|
||||||
|
|
||||||
|
|
||||||
|
class FilesManagedTabularFileService(ManagedTabularFileProvider):
|
||||||
|
"""Expose authorized immutable CSV/XLSX versions to optional consumers."""
|
||||||
|
|
||||||
|
def list_tabular_files(
|
||||||
|
self,
|
||||||
|
session: object,
|
||||||
|
principal: object,
|
||||||
|
*,
|
||||||
|
query: str = "",
|
||||||
|
limit: int = 100,
|
||||||
|
) -> tuple[ManagedTabularFile, ...]:
|
||||||
|
db, api_principal, user_id = _tabular_context(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
required_scope="files:file:read",
|
||||||
|
)
|
||||||
|
normalized_query = str(query or "").strip().casefold()
|
||||||
|
requested_limit = max(1, min(int(limit), 100))
|
||||||
|
assets = list_recent_assets_for_user(
|
||||||
|
db,
|
||||||
|
tenant_id=api_principal.tenant_id,
|
||||||
|
user_id=user_id,
|
||||||
|
limit=min(500, requested_limit * 5),
|
||||||
|
is_admin=has_scope(api_principal, "files:file:admin"),
|
||||||
|
)
|
||||||
|
results: list[ManagedTabularFile] = []
|
||||||
|
for asset in assets:
|
||||||
|
if normalized_query and normalized_query not in (
|
||||||
|
f"{asset.filename} {asset.display_path} {asset.description or ''}"
|
||||||
|
).casefold():
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
version, blob = current_version_and_blob(db, asset)
|
||||||
|
except FileStorageError:
|
||||||
|
continue
|
||||||
|
if not _is_tabular_file(version.filename_at_upload, version.content_type):
|
||||||
|
continue
|
||||||
|
results.append(_managed_tabular_file(asset, version, blob))
|
||||||
|
if len(results) >= requested_limit:
|
||||||
|
break
|
||||||
|
return tuple(results)
|
||||||
|
|
||||||
|
def get_tabular_file(
|
||||||
|
self,
|
||||||
|
session: object,
|
||||||
|
principal: object,
|
||||||
|
*,
|
||||||
|
file_asset_id: str,
|
||||||
|
file_version_id: str | None = None,
|
||||||
|
) -> ManagedTabularFile | None:
|
||||||
|
db, api_principal, user_id = _tabular_context(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
required_scope="files:file:read",
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
asset = get_asset_for_user(
|
||||||
|
db,
|
||||||
|
tenant_id=api_principal.tenant_id,
|
||||||
|
user_id=user_id,
|
||||||
|
asset_id=file_asset_id,
|
||||||
|
is_admin=has_scope(api_principal, "files:file:admin"),
|
||||||
|
)
|
||||||
|
except FileStorageError:
|
||||||
|
return None
|
||||||
|
if file_version_id:
|
||||||
|
version = db.get(FileVersion, file_version_id)
|
||||||
|
if (
|
||||||
|
version is None
|
||||||
|
or version.file_asset_id != asset.id
|
||||||
|
or version.tenant_id != api_principal.tenant_id
|
||||||
|
):
|
||||||
|
return None
|
||||||
|
blob = db.get(FileBlob, version.blob_id)
|
||||||
|
if blob is None or blob.tenant_id != api_principal.tenant_id:
|
||||||
|
return None
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
version, blob = current_version_and_blob(db, asset)
|
||||||
|
except FileStorageError as exc:
|
||||||
|
raise ManagedTabularFileUnavailableError(
|
||||||
|
"Managed file version metadata is unavailable."
|
||||||
|
) from exc
|
||||||
|
if not _is_tabular_file(version.filename_at_upload, version.content_type):
|
||||||
|
return None
|
||||||
|
return _managed_tabular_file(asset, version, blob)
|
||||||
|
|
||||||
|
def read_tabular_file(
|
||||||
|
self,
|
||||||
|
session: object,
|
||||||
|
principal: object,
|
||||||
|
*,
|
||||||
|
file_asset_id: str,
|
||||||
|
file_version_id: str,
|
||||||
|
max_bytes: int,
|
||||||
|
) -> ManagedTabularFileContent:
|
||||||
|
db, api_principal, user_id = _tabular_context(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
required_scope="files:file:download",
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
asset = get_asset_for_user(
|
||||||
|
db,
|
||||||
|
tenant_id=api_principal.tenant_id,
|
||||||
|
user_id=user_id,
|
||||||
|
asset_id=file_asset_id,
|
||||||
|
is_admin=has_scope(api_principal, "files:file:admin"),
|
||||||
|
)
|
||||||
|
except FileStorageError as exc:
|
||||||
|
raise ManagedTabularFileNotFoundError(
|
||||||
|
"Managed tabular file not found."
|
||||||
|
) from exc
|
||||||
|
metadata = self.get_tabular_file(
|
||||||
|
db,
|
||||||
|
api_principal,
|
||||||
|
file_asset_id=asset.id,
|
||||||
|
file_version_id=file_version_id,
|
||||||
|
)
|
||||||
|
if metadata is None:
|
||||||
|
raise ManagedTabularFileNotFoundError(
|
||||||
|
"Managed tabular file version not found."
|
||||||
|
)
|
||||||
|
effective_max = max(1, int(max_bytes))
|
||||||
|
if metadata.size_bytes > effective_max:
|
||||||
|
raise ManagedTabularFileValidationError(
|
||||||
|
f"Managed tabular files are limited to {effective_max:,} bytes for this operation."
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
payload, version, blob = read_asset_version_bytes(
|
||||||
|
db,
|
||||||
|
asset,
|
||||||
|
file_version_id,
|
||||||
|
)
|
||||||
|
except FileStorageError as exc:
|
||||||
|
raise ManagedTabularFileUnavailableError(
|
||||||
|
"Managed tabular file content is unavailable or failed integrity verification."
|
||||||
|
) from exc
|
||||||
|
if len(payload) > effective_max:
|
||||||
|
raise ManagedTabularFileValidationError(
|
||||||
|
f"Managed tabular files are limited to {effective_max:,} bytes for this operation."
|
||||||
|
)
|
||||||
|
result = _managed_tabular_file(asset, version, blob)
|
||||||
|
audit_from_principal(
|
||||||
|
db,
|
||||||
|
api_principal,
|
||||||
|
action="files.tabular_content.read",
|
||||||
|
object_type="file_version",
|
||||||
|
object_id=version.id,
|
||||||
|
details={
|
||||||
|
"file_asset_id": asset.id,
|
||||||
|
"size_bytes": version.size_bytes,
|
||||||
|
"checksum_sha256": version.checksum_sha256,
|
||||||
|
"consumer": "tabular_content",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
return ManagedTabularFileContent(file=result, payload=payload)
|
||||||
|
|
||||||
|
|
||||||
|
def managed_tabular_file_capability(
|
||||||
|
context: ModuleContext,
|
||||||
|
) -> FilesManagedTabularFileService:
|
||||||
|
configure_runtime(registry=context.registry, settings=context.settings)
|
||||||
|
return FilesManagedTabularFileService()
|
||||||
|
|
||||||
|
|
||||||
|
def _tabular_context(
|
||||||
|
session: object,
|
||||||
|
principal: object,
|
||||||
|
*,
|
||||||
|
required_scope: str,
|
||||||
|
) -> tuple[Session, ApiPrincipal, str]:
|
||||||
|
if not isinstance(session, Session):
|
||||||
|
raise TypeError("Managed tabular file access requires a SQLAlchemy session.")
|
||||||
|
if not isinstance(principal, ApiPrincipal):
|
||||||
|
raise ManagedTabularFileAccessError(
|
||||||
|
"Managed tabular file access requires a tenant API principal."
|
||||||
|
)
|
||||||
|
if not (
|
||||||
|
has_scope(principal, required_scope)
|
||||||
|
or has_scope(principal, "files:file:admin")
|
||||||
|
):
|
||||||
|
raise ManagedTabularFileAccessError(
|
||||||
|
f"Managed tabular file access requires {required_scope}."
|
||||||
|
)
|
||||||
|
user_id = str(getattr(principal.user, "id", "") or "").strip()
|
||||||
|
if not principal.tenant_id or not user_id:
|
||||||
|
raise ManagedTabularFileAccessError(
|
||||||
|
"Managed tabular file access requires a tenant user principal."
|
||||||
|
)
|
||||||
|
return session, principal, user_id
|
||||||
|
|
||||||
|
|
||||||
|
def _managed_tabular_file(
|
||||||
|
asset: FileAsset,
|
||||||
|
version: FileVersion,
|
||||||
|
blob: FileBlob,
|
||||||
|
) -> ManagedTabularFile:
|
||||||
|
return ManagedTabularFile(
|
||||||
|
file_asset_id=asset.id,
|
||||||
|
file_version_id=version.id,
|
||||||
|
filename=version.filename_at_upload or asset.filename,
|
||||||
|
display_path=version.display_path_at_upload or asset.display_path,
|
||||||
|
content_type=version.content_type or blob.content_type,
|
||||||
|
size_bytes=version.size_bytes,
|
||||||
|
sha256=version.checksum_sha256,
|
||||||
|
updated_at=version.created_at,
|
||||||
|
current_version=asset.current_version_id == version.id,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _is_tabular_file(filename: str, content_type: str | None) -> bool:
|
||||||
|
normalized_name = str(filename or "").strip().casefold()
|
||||||
|
normalized_type = str(content_type or "").split(";", 1)[0].strip().casefold()
|
||||||
|
return normalized_name.endswith((".csv", ".xlsx")) or normalized_type in {
|
||||||
|
"text/csv",
|
||||||
|
"application/csv",
|
||||||
|
"application/vnd.ms-excel",
|
||||||
|
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def virtual_folder_resource_id(*, tenant_id: str, owner_type: str, owner_id: str, path: str) -> str:
|
def virtual_folder_resource_id(*, tenant_id: str, owner_type: str, owner_id: str, path: str) -> str:
|
||||||
normalized_path = normalize_folder(path)
|
normalized_path = normalize_folder(path)
|
||||||
encoded_path = base64.urlsafe_b64encode(normalized_path.encode("utf-8")).decode("ascii").rstrip("=")
|
encoded_path = base64.urlsafe_b64encode(normalized_path.encode("utf-8")).decode("ascii").rstrip("=")
|
||||||
|
|||||||
@@ -56,6 +56,10 @@ def _record_asset_change(session: OrmSession, asset: FileAsset) -> None:
|
|||||||
"filename",
|
"filename",
|
||||||
"description",
|
"description",
|
||||||
"deleted_at",
|
"deleted_at",
|
||||||
|
"retained_until",
|
||||||
|
"legal_hold",
|
||||||
|
"lifecycle_revision",
|
||||||
|
"lifecycle_reason",
|
||||||
"metadata_",
|
"metadata_",
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
@@ -79,6 +83,9 @@ def _record_asset_change(session: OrmSession, asset: FileAsset) -> None:
|
|||||||
"previous_path": previous_value(asset, "display_path"),
|
"previous_path": previous_value(asset, "display_path"),
|
||||||
"filename": asset.filename,
|
"filename": asset.filename,
|
||||||
"deleted_at": _isoformat(asset.deleted_at),
|
"deleted_at": _isoformat(asset.deleted_at),
|
||||||
|
"retained_until": _isoformat(asset.retained_until),
|
||||||
|
"legal_hold": asset.legal_hold,
|
||||||
|
"lifecycle_revision": asset.lifecycle_revision,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,600 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from collections.abc import Mapping
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
from urllib.parse import urlsplit
|
||||||
|
|
||||||
|
from govoplan_core.core.configuration_packages import (
|
||||||
|
ConfigurationApplyResult,
|
||||||
|
ConfigurationDiagnostic,
|
||||||
|
ConfigurationExportResult,
|
||||||
|
ConfigurationExportSelection,
|
||||||
|
ConfigurationPackageFragment,
|
||||||
|
ConfigurationPlanItem,
|
||||||
|
ConfigurationPreflightContext,
|
||||||
|
ConfigurationPreflightResult,
|
||||||
|
ConfigurationProvider,
|
||||||
|
ConfigurationProviderDescription,
|
||||||
|
)
|
||||||
|
from govoplan_core.core.infrastructure_capabilities import (
|
||||||
|
InfrastructureCapability,
|
||||||
|
InfrastructureCapabilityReceipt,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.runtime import settings as runtime_settings
|
||||||
|
|
||||||
|
|
||||||
|
FILES_CONFIGURATION_CAPABILITY = "files.configuration"
|
||||||
|
MANAGED_STORAGE_FRAGMENT = "managed_storage"
|
||||||
|
_PAYLOAD_KEYS = frozenset(
|
||||||
|
{"capability_id", "expected_backend", "expected_source"}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class FilesConfigurationProvider(ConfigurationProvider):
|
||||||
|
module_id = "files"
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
settings: object | None = None,
|
||||||
|
environment: Mapping[str, str] | None = None,
|
||||||
|
) -> None:
|
||||||
|
self._settings = runtime_settings if settings is None else settings
|
||||||
|
self._environment = os.environ if environment is None else environment
|
||||||
|
|
||||||
|
def describe(self) -> ConfigurationProviderDescription:
|
||||||
|
return ConfigurationProviderDescription(
|
||||||
|
module_id=self.module_id,
|
||||||
|
fragment_types=(MANAGED_STORAGE_FRAGMENT,),
|
||||||
|
schema_refs={
|
||||||
|
MANAGED_STORAGE_FRAGMENT: "govoplan/files/configuration/managed-storage.v1"
|
||||||
|
},
|
||||||
|
exported_scopes=("system",),
|
||||||
|
)
|
||||||
|
|
||||||
|
def preflight(
|
||||||
|
self,
|
||||||
|
fragment: ConfigurationPackageFragment,
|
||||||
|
context: ConfigurationPreflightContext,
|
||||||
|
) -> ConfigurationPreflightResult:
|
||||||
|
if fragment.fragment_type != MANAGED_STORAGE_FRAGMENT:
|
||||||
|
return ConfigurationPreflightResult(
|
||||||
|
diagnostics=(_unsupported(fragment),),
|
||||||
|
plan=(_blocked_plan(fragment, "Fragment type is unsupported."),),
|
||||||
|
)
|
||||||
|
diagnostics, binding_ref = self._binding_diagnostics(fragment, context)
|
||||||
|
blocked = any(item.severity == "blocker" for item in diagnostics)
|
||||||
|
return ConfigurationPreflightResult(
|
||||||
|
diagnostics=tuple(diagnostics),
|
||||||
|
plan=(
|
||||||
|
ConfigurationPlanItem(
|
||||||
|
action="blocked" if blocked else "skip",
|
||||||
|
module_id="files",
|
||||||
|
fragment_type=MANAGED_STORAGE_FRAGMENT,
|
||||||
|
fragment_id=fragment.fragment_id or binding_ref,
|
||||||
|
summary=(
|
||||||
|
"Managed storage does not match the deployment receipt."
|
||||||
|
if blocked
|
||||||
|
else "Deployment-owned managed storage already matches the receipt; no module state is rewritten."
|
||||||
|
),
|
||||||
|
),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
def apply(
|
||||||
|
self,
|
||||||
|
fragment: ConfigurationPackageFragment,
|
||||||
|
supplied_data: Mapping[str, Any],
|
||||||
|
context: ConfigurationPreflightContext,
|
||||||
|
) -> ConfigurationApplyResult:
|
||||||
|
del supplied_data
|
||||||
|
if fragment.fragment_type != MANAGED_STORAGE_FRAGMENT:
|
||||||
|
return ConfigurationApplyResult(diagnostics=(_unsupported(fragment),))
|
||||||
|
diagnostics, _binding_ref = self._binding_diagnostics(fragment, context)
|
||||||
|
return ConfigurationApplyResult(
|
||||||
|
diagnostics=tuple(
|
||||||
|
item for item in diagnostics if item.severity == "blocker"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
def export(
|
||||||
|
self,
|
||||||
|
selection: ConfigurationExportSelection,
|
||||||
|
context: ConfigurationPreflightContext,
|
||||||
|
) -> ConfigurationExportResult:
|
||||||
|
del selection
|
||||||
|
receipt = context.infrastructure_receipt
|
||||||
|
if context.infrastructure_receipt_error:
|
||||||
|
return ConfigurationExportResult(
|
||||||
|
diagnostics=(
|
||||||
|
_receipt_error(
|
||||||
|
context.infrastructure_receipt_error,
|
||||||
|
object_ref="files.storage",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if receipt is None:
|
||||||
|
return ConfigurationExportResult(
|
||||||
|
diagnostics=(
|
||||||
|
ConfigurationDiagnostic(
|
||||||
|
severity="blocker",
|
||||||
|
code="infrastructure_receipt_missing",
|
||||||
|
message="Files managed-storage export requires the deployment capability receipt.",
|
||||||
|
module_id="files",
|
||||||
|
object_ref="files.storage",
|
||||||
|
resolution="Mount the installer-generated receipt before exporting deployment configuration.",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
capability = receipt.capability("files.storage")
|
||||||
|
if capability is None:
|
||||||
|
return ConfigurationExportResult(
|
||||||
|
diagnostics=(
|
||||||
|
ConfigurationDiagnostic(
|
||||||
|
severity="blocker",
|
||||||
|
code="infrastructure_capability_missing",
|
||||||
|
message="The deployment receipt does not declare managed file storage.",
|
||||||
|
module_id="files",
|
||||||
|
object_ref="files.storage",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return ConfigurationExportResult(
|
||||||
|
fragments=(
|
||||||
|
ConfigurationPackageFragment(
|
||||||
|
module_id="files",
|
||||||
|
fragment_type=MANAGED_STORAGE_FRAGMENT,
|
||||||
|
fragment_id=f"{receipt.installation_id}:files.storage",
|
||||||
|
payload={
|
||||||
|
"capability_id": "files.storage",
|
||||||
|
"expected_backend": _expected_backend(capability.source),
|
||||||
|
"expected_source": capability.source,
|
||||||
|
},
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
def health(
|
||||||
|
self,
|
||||||
|
import_result: ConfigurationApplyResult,
|
||||||
|
context: ConfigurationPreflightContext,
|
||||||
|
) -> tuple[ConfigurationDiagnostic, ...]:
|
||||||
|
del context
|
||||||
|
return tuple(
|
||||||
|
item for item in import_result.diagnostics if item.severity == "blocker"
|
||||||
|
)
|
||||||
|
|
||||||
|
def _binding_diagnostics(
|
||||||
|
self,
|
||||||
|
fragment: ConfigurationPackageFragment,
|
||||||
|
context: ConfigurationPreflightContext,
|
||||||
|
) -> tuple[list[ConfigurationDiagnostic], str]:
|
||||||
|
diagnostics: list[ConfigurationDiagnostic] = []
|
||||||
|
payload = _files_fragment_payload(fragment, diagnostics)
|
||||||
|
if payload is None:
|
||||||
|
return diagnostics, fragment.fragment_id or "files.storage"
|
||||||
|
receipt_state = _files_receipt_capability(
|
||||||
|
fragment,
|
||||||
|
context,
|
||||||
|
payload,
|
||||||
|
diagnostics,
|
||||||
|
)
|
||||||
|
if receipt_state is None:
|
||||||
|
return diagnostics, fragment.fragment_id or "files.storage"
|
||||||
|
_receipt, capability, binding_ref = receipt_state
|
||||||
|
expected_backend = _expected_backend(capability.source)
|
||||||
|
diagnostics.extend(
|
||||||
|
_package_expectation_diagnostics(
|
||||||
|
payload,
|
||||||
|
capability,
|
||||||
|
expected_backend,
|
||||||
|
binding_ref,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
active_backend = _normalized_backend(
|
||||||
|
getattr(self._settings, "file_storage_backend", "local")
|
||||||
|
)
|
||||||
|
if active_backend != expected_backend:
|
||||||
|
diagnostics.append(
|
||||||
|
ConfigurationDiagnostic(
|
||||||
|
severity="blocker",
|
||||||
|
code="files_storage_runtime_mismatch",
|
||||||
|
message=(
|
||||||
|
f"Files runtime uses {active_backend!r}, while the deployment receipt declares {expected_backend!r}."
|
||||||
|
),
|
||||||
|
module_id="files",
|
||||||
|
object_ref=binding_ref,
|
||||||
|
resolution="Reconcile deployment environment and receipt before starting or importing Files configuration.",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return diagnostics, binding_ref
|
||||||
|
diagnostics.extend(
|
||||||
|
self._runtime_storage_diagnostics(
|
||||||
|
expected_backend,
|
||||||
|
capability,
|
||||||
|
binding_ref,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if "files" not in capability.dependent_modules:
|
||||||
|
diagnostics.append(
|
||||||
|
ConfigurationDiagnostic(
|
||||||
|
severity="info",
|
||||||
|
code="infrastructure_consumer_not_declared",
|
||||||
|
message="Files is active but was not selected as a receipt consumer when the deployment plan was generated.",
|
||||||
|
module_id="files",
|
||||||
|
object_ref=binding_ref,
|
||||||
|
resolution="Regenerate the deployment plan so removal-impact inventory includes Files.",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return diagnostics, binding_ref
|
||||||
|
|
||||||
|
def _runtime_storage_diagnostics(
|
||||||
|
self,
|
||||||
|
expected_backend: str,
|
||||||
|
capability: InfrastructureCapability,
|
||||||
|
binding_ref: str,
|
||||||
|
) -> list[ConfigurationDiagnostic]:
|
||||||
|
if expected_backend == "s3":
|
||||||
|
return self._s3_diagnostics(
|
||||||
|
capability.endpoint,
|
||||||
|
capability.secret_refs,
|
||||||
|
capability.source,
|
||||||
|
binding_ref,
|
||||||
|
)
|
||||||
|
root = Path(
|
||||||
|
str(getattr(self._settings, "file_storage_local_root", "") or "")
|
||||||
|
)
|
||||||
|
if root.is_absolute():
|
||||||
|
return []
|
||||||
|
return [
|
||||||
|
ConfigurationDiagnostic(
|
||||||
|
severity="blocker",
|
||||||
|
code="files_storage_local_root_not_durable",
|
||||||
|
message="Receipt-bound local file storage requires an absolute deployment-managed path.",
|
||||||
|
module_id="files",
|
||||||
|
object_ref=binding_ref,
|
||||||
|
resolution="Set FILE_STORAGE_LOCAL_ROOT to the mounted persistent-volume path.",
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
|
def _s3_diagnostics(
|
||||||
|
self,
|
||||||
|
endpoint: Mapping[str, object],
|
||||||
|
secret_refs: tuple[str, ...],
|
||||||
|
source: str,
|
||||||
|
object_ref: str,
|
||||||
|
) -> list[ConfigurationDiagnostic]:
|
||||||
|
diagnostics: list[ConfigurationDiagnostic] = []
|
||||||
|
endpoint_url = str(
|
||||||
|
getattr(self._settings, "file_storage_s3_endpoint_url", "")
|
||||||
|
or getattr(self._settings, "s3_endpoint_url", "")
|
||||||
|
or ""
|
||||||
|
).strip()
|
||||||
|
runtime_endpoint = _redacted_endpoint(
|
||||||
|
endpoint_url,
|
||||||
|
default_port=int(endpoint.get("port") or 443),
|
||||||
|
)
|
||||||
|
receipt_endpoint = {
|
||||||
|
key: endpoint.get(key) for key in ("scheme", "host", "port") if endpoint.get(key) is not None
|
||||||
|
}
|
||||||
|
if receipt_endpoint and runtime_endpoint != receipt_endpoint:
|
||||||
|
diagnostics.append(
|
||||||
|
ConfigurationDiagnostic(
|
||||||
|
severity="blocker",
|
||||||
|
code="files_storage_endpoint_mismatch",
|
||||||
|
message="Files S3 endpoint does not match the sanitized deployment receipt endpoint.",
|
||||||
|
module_id="files",
|
||||||
|
object_ref=object_ref,
|
||||||
|
resolution="Reconcile FILE_STORAGE_S3_ENDPOINT_URL and the deployment plan before apply.",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
bucket = str(
|
||||||
|
getattr(self._settings, "file_storage_s3_bucket", "")
|
||||||
|
or getattr(self._settings, "s3_bucket", "")
|
||||||
|
or ""
|
||||||
|
).strip()
|
||||||
|
if not bucket:
|
||||||
|
diagnostics.append(
|
||||||
|
ConfigurationDiagnostic(
|
||||||
|
severity="blocker",
|
||||||
|
code="files_storage_bucket_missing",
|
||||||
|
message="Files S3 storage has no configured bucket.",
|
||||||
|
module_id="files",
|
||||||
|
object_ref=object_ref,
|
||||||
|
resolution="Set FILE_STORAGE_S3_BUCKET in the deployment environment.",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
for reference in secret_refs:
|
||||||
|
variable = reference.removeprefix("env:")
|
||||||
|
if not variable.startswith("FILE_STORAGE_"):
|
||||||
|
continue
|
||||||
|
if not str(self._environment.get(variable, "")).strip():
|
||||||
|
diagnostics.append(
|
||||||
|
ConfigurationDiagnostic(
|
||||||
|
severity="blocker",
|
||||||
|
code="files_storage_secret_reference_unresolved",
|
||||||
|
message=f"Required storage secret reference {reference} is not available to the runtime.",
|
||||||
|
module_id="files",
|
||||||
|
object_ref=object_ref,
|
||||||
|
resolution="Provide the referenced environment secret without copying its value into the package.",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
deployment_managed = bool(
|
||||||
|
getattr(
|
||||||
|
self._settings,
|
||||||
|
"file_storage_s3_deployment_managed",
|
||||||
|
False,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
endpoint_trusted = bool(
|
||||||
|
getattr(self._settings, "file_storage_s3_endpoint_trusted", False)
|
||||||
|
)
|
||||||
|
if source == "installer-managed-garage" and not deployment_managed:
|
||||||
|
diagnostics.append(
|
||||||
|
ConfigurationDiagnostic(
|
||||||
|
severity="blocker",
|
||||||
|
code="files_storage_management_boundary_mismatch",
|
||||||
|
message="Garage is installer-managed in the receipt but not marked deployment-managed in Files runtime.",
|
||||||
|
module_id="files",
|
||||||
|
object_ref=object_ref,
|
||||||
|
resolution="Set FILE_STORAGE_S3_DEPLOYMENT_MANAGED=true for the installer-managed Garage endpoint.",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if source == "operator-supplied-s3" and not endpoint_trusted:
|
||||||
|
diagnostics.append(
|
||||||
|
ConfigurationDiagnostic(
|
||||||
|
severity="blocker",
|
||||||
|
code="files_storage_trust_boundary_missing",
|
||||||
|
message="External S3 storage must be explicitly marked as a trusted deployment endpoint.",
|
||||||
|
module_id="files",
|
||||||
|
object_ref=object_ref,
|
||||||
|
resolution="Review the endpoint and set FILE_STORAGE_S3_ENDPOINT_TRUSTED=true in deployment configuration.",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return diagnostics
|
||||||
|
|
||||||
|
|
||||||
|
def _files_fragment_payload(
|
||||||
|
fragment: ConfigurationPackageFragment,
|
||||||
|
diagnostics: list[ConfigurationDiagnostic],
|
||||||
|
) -> Mapping[str, Any] | None:
|
||||||
|
payload = fragment.payload
|
||||||
|
if not isinstance(payload, Mapping):
|
||||||
|
diagnostics.append(
|
||||||
|
_invalid(fragment, "Managed-storage payload must be an object.")
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
unknown = sorted(set(payload) - _PAYLOAD_KEYS)
|
||||||
|
if not unknown:
|
||||||
|
return payload
|
||||||
|
secret_like = any(
|
||||||
|
marker in key.casefold()
|
||||||
|
for key in unknown
|
||||||
|
for marker in ("password", "secret", "token", "credential", "access_key")
|
||||||
|
)
|
||||||
|
diagnostics.append(
|
||||||
|
ConfigurationDiagnostic(
|
||||||
|
severity="blocker",
|
||||||
|
code=(
|
||||||
|
"files_configuration_secret_forbidden"
|
||||||
|
if secret_like
|
||||||
|
else "files_configuration_payload_invalid"
|
||||||
|
),
|
||||||
|
message=(
|
||||||
|
"Managed-storage fragments accept receipt references and non-secret expectations only."
|
||||||
|
if secret_like
|
||||||
|
else f"Managed-storage payload contains unsupported fields: {', '.join(unknown)}."
|
||||||
|
),
|
||||||
|
module_id="files",
|
||||||
|
object_ref=fragment.fragment_id or MANAGED_STORAGE_FRAGMENT,
|
||||||
|
resolution="Keep storage credentials in deployment environment references, never in a configuration package.",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return payload
|
||||||
|
|
||||||
|
|
||||||
|
def _files_receipt_capability(
|
||||||
|
fragment: ConfigurationPackageFragment,
|
||||||
|
context: ConfigurationPreflightContext,
|
||||||
|
payload: Mapping[str, Any],
|
||||||
|
diagnostics: list[ConfigurationDiagnostic],
|
||||||
|
) -> tuple[
|
||||||
|
InfrastructureCapabilityReceipt,
|
||||||
|
InfrastructureCapability,
|
||||||
|
str,
|
||||||
|
] | None:
|
||||||
|
if context.infrastructure_receipt_error:
|
||||||
|
diagnostics.append(
|
||||||
|
_receipt_error(
|
||||||
|
context.infrastructure_receipt_error,
|
||||||
|
object_ref=fragment.fragment_id or "files.storage",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
receipt = context.infrastructure_receipt
|
||||||
|
if receipt is None:
|
||||||
|
diagnostics.append(
|
||||||
|
ConfigurationDiagnostic(
|
||||||
|
severity="blocker",
|
||||||
|
code="infrastructure_receipt_missing",
|
||||||
|
message="Files managed-storage configuration requires the deployment capability receipt.",
|
||||||
|
module_id="files",
|
||||||
|
object_ref=fragment.fragment_id or "files.storage",
|
||||||
|
resolution="Mount the installer-generated receipt and rerun preflight.",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
binding_ref = fragment.fragment_id or f"{receipt.installation_id}:files.storage"
|
||||||
|
capability_id = _text(payload.get("capability_id")) or "files.storage"
|
||||||
|
if capability_id != "files.storage":
|
||||||
|
diagnostics.append(
|
||||||
|
_invalid(
|
||||||
|
fragment,
|
||||||
|
"Files managed-storage fragments must reference capability 'files.storage'.",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
capability = receipt.capability(capability_id)
|
||||||
|
if capability is None:
|
||||||
|
diagnostics.append(
|
||||||
|
ConfigurationDiagnostic(
|
||||||
|
severity="blocker",
|
||||||
|
code="infrastructure_capability_missing",
|
||||||
|
message="The deployment receipt does not declare managed file storage.",
|
||||||
|
module_id="files",
|
||||||
|
object_ref=binding_ref,
|
||||||
|
resolution="Regenerate the receipt from a deployment profile that declares managed storage.",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
state_diagnostic = _storage_capability_state_diagnostic(capability, binding_ref)
|
||||||
|
if state_diagnostic is not None:
|
||||||
|
diagnostics.append(state_diagnostic)
|
||||||
|
return None
|
||||||
|
return receipt, capability, binding_ref
|
||||||
|
|
||||||
|
|
||||||
|
def _storage_capability_state_diagnostic(
|
||||||
|
capability: InfrastructureCapability,
|
||||||
|
binding_ref: str,
|
||||||
|
) -> ConfigurationDiagnostic | None:
|
||||||
|
if capability.state == "unavailable":
|
||||||
|
return ConfigurationDiagnostic(
|
||||||
|
severity="blocker",
|
||||||
|
code="infrastructure_capability_unavailable",
|
||||||
|
message="The deployment receipt states that managed file storage is unavailable.",
|
||||||
|
module_id="files",
|
||||||
|
object_ref=binding_ref,
|
||||||
|
resolution="Select local, managed Garage, or external S3 storage in the deployment profile.",
|
||||||
|
)
|
||||||
|
if capability.state == "available_unconfigured":
|
||||||
|
return ConfigurationDiagnostic(
|
||||||
|
severity="blocker",
|
||||||
|
code="infrastructure_capability_unconfigured",
|
||||||
|
message="Managed file storage is available but has not been bound by the deployment runtime.",
|
||||||
|
module_id="files",
|
||||||
|
object_ref=binding_ref,
|
||||||
|
resolution="Complete the deployment-owned storage configuration before importing Files configuration.",
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _package_expectation_diagnostics(
|
||||||
|
payload: Mapping[str, Any],
|
||||||
|
capability: InfrastructureCapability,
|
||||||
|
expected_backend: str,
|
||||||
|
binding_ref: str,
|
||||||
|
) -> list[ConfigurationDiagnostic]:
|
||||||
|
diagnostics: list[ConfigurationDiagnostic] = []
|
||||||
|
package_backend = (
|
||||||
|
_text(payload.get("expected_backend")) or expected_backend
|
||||||
|
).casefold()
|
||||||
|
if package_backend != expected_backend:
|
||||||
|
diagnostics.append(
|
||||||
|
ConfigurationDiagnostic(
|
||||||
|
severity="blocker",
|
||||||
|
code="infrastructure_backend_mismatch",
|
||||||
|
message="The package storage backend expectation conflicts with the deployment receipt.",
|
||||||
|
module_id="files",
|
||||||
|
object_ref=binding_ref,
|
||||||
|
resolution="Use the receipt backend or regenerate the deployment package after review.",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
package_source = _text(payload.get("expected_source"))
|
||||||
|
if package_source and package_source != capability.source:
|
||||||
|
diagnostics.append(
|
||||||
|
ConfigurationDiagnostic(
|
||||||
|
severity="blocker",
|
||||||
|
code="infrastructure_source_mismatch",
|
||||||
|
message="The package storage source expectation conflicts with the deployment receipt.",
|
||||||
|
module_id="files",
|
||||||
|
object_ref=binding_ref,
|
||||||
|
resolution="Review the provider replacement and regenerate the package from the active receipt.",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return diagnostics
|
||||||
|
|
||||||
|
|
||||||
|
def _expected_backend(source: str) -> str:
|
||||||
|
return "local" if source == "host-local" else "s3"
|
||||||
|
|
||||||
|
|
||||||
|
def _normalized_backend(value: object) -> str:
|
||||||
|
clean = str(value or "local").strip().casefold()
|
||||||
|
if clean in {"local", "filesystem", "fs"}:
|
||||||
|
return "local"
|
||||||
|
if clean in {"s3", "garage"}:
|
||||||
|
return "s3"
|
||||||
|
return clean
|
||||||
|
|
||||||
|
|
||||||
|
def _redacted_endpoint(value: str, *, default_port: int) -> dict[str, object]:
|
||||||
|
try:
|
||||||
|
parsed = urlsplit(value)
|
||||||
|
host = parsed.hostname
|
||||||
|
if not parsed.scheme or not host:
|
||||||
|
return {"reference": "unresolved"}
|
||||||
|
port = parsed.port or default_port
|
||||||
|
except ValueError:
|
||||||
|
return {"reference": "unresolved"}
|
||||||
|
return {"scheme": parsed.scheme, "host": host, "port": port}
|
||||||
|
|
||||||
|
|
||||||
|
def _text(value: object) -> str | None:
|
||||||
|
if value is None:
|
||||||
|
return None
|
||||||
|
text = str(value).strip()
|
||||||
|
return text or None
|
||||||
|
|
||||||
|
|
||||||
|
def _receipt_error(error: str, *, object_ref: str) -> ConfigurationDiagnostic:
|
||||||
|
return ConfigurationDiagnostic(
|
||||||
|
severity="blocker",
|
||||||
|
code="infrastructure_receipt_invalid",
|
||||||
|
message=f"The deployment capability receipt is invalid: {error}",
|
||||||
|
module_id="files",
|
||||||
|
object_ref=object_ref,
|
||||||
|
resolution="Repair or regenerate the deployment receipt before importing Files configuration.",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _invalid(
|
||||||
|
fragment: ConfigurationPackageFragment,
|
||||||
|
message: str,
|
||||||
|
) -> ConfigurationDiagnostic:
|
||||||
|
return ConfigurationDiagnostic(
|
||||||
|
severity="blocker",
|
||||||
|
code="files_configuration_payload_invalid",
|
||||||
|
message=message,
|
||||||
|
module_id="files",
|
||||||
|
object_ref=fragment.fragment_id or fragment.fragment_type,
|
||||||
|
resolution="Review the Files configuration-package fragment and rerun preflight.",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _unsupported(fragment: ConfigurationPackageFragment) -> ConfigurationDiagnostic:
|
||||||
|
return ConfigurationDiagnostic(
|
||||||
|
severity="blocker",
|
||||||
|
code="fragment_type_unsupported",
|
||||||
|
message=f"Files configuration does not support fragment type {fragment.fragment_type!r}.",
|
||||||
|
module_id="files",
|
||||||
|
object_ref=fragment.fragment_id or fragment.fragment_type,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _blocked_plan(
|
||||||
|
fragment: ConfigurationPackageFragment,
|
||||||
|
summary: str,
|
||||||
|
) -> ConfigurationPlanItem:
|
||||||
|
return ConfigurationPlanItem(
|
||||||
|
action="blocked",
|
||||||
|
module_id="files",
|
||||||
|
fragment_type=fragment.fragment_type,
|
||||||
|
fragment_id=fragment.fragment_id,
|
||||||
|
summary=summary,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["FILES_CONFIGURATION_CAPABILITY", "FilesConfigurationProvider"]
|
||||||
@@ -4,7 +4,18 @@ import uuid
|
|||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from sqlalchemy import Boolean, DateTime, ForeignKey, Index, Integer, JSON, String, Text, UniqueConstraint, text
|
from sqlalchemy import (
|
||||||
|
Boolean,
|
||||||
|
DateTime,
|
||||||
|
ForeignKey,
|
||||||
|
Index,
|
||||||
|
Integer,
|
||||||
|
JSON,
|
||||||
|
String,
|
||||||
|
Text,
|
||||||
|
UniqueConstraint,
|
||||||
|
text,
|
||||||
|
)
|
||||||
from sqlalchemy.orm import Mapped, mapped_column
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
|
||||||
from govoplan_core.db.base import Base, TimestampMixin
|
from govoplan_core.db.base import Base, TimestampMixin
|
||||||
@@ -33,17 +44,29 @@ class FileBlob(Base, TimestampMixin):
|
|||||||
storage_key: Mapped[str] = mapped_column(String(1000), nullable=False)
|
storage_key: Mapped[str] = mapped_column(String(1000), nullable=False)
|
||||||
checksum_sha256: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
checksum_sha256: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||||
size_bytes: Mapped[int] = mapped_column(Integer, nullable=False)
|
size_bytes: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||||
protection_discriminator: Mapped[str] = mapped_column(String(320), default="plaintext", nullable=False, index=True)
|
protection_discriminator: Mapped[str] = mapped_column(
|
||||||
encryption_envelope_id: Mapped[str | None] = mapped_column(String(255), nullable=True, index=True)
|
String(320), default="plaintext", nullable=False, index=True
|
||||||
storage_checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
)
|
||||||
|
encryption_envelope_id: Mapped[str | None] = mapped_column(
|
||||||
|
String(255), nullable=True, index=True
|
||||||
|
)
|
||||||
|
storage_checksum_sha256: Mapped[str | None] = mapped_column(
|
||||||
|
String(64), nullable=True
|
||||||
|
)
|
||||||
storage_size_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
storage_size_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||||
content_type: Mapped[str | None] = mapped_column(String(255))
|
content_type: Mapped[str | None] = mapped_column(String(255))
|
||||||
ref_count: Mapped[int] = mapped_column(Integer, default=1, nullable=False)
|
ref_count: Mapped[int] = mapped_column(Integer, default=1, nullable=False)
|
||||||
retained_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
retained_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
integrity_status: Mapped[str] = mapped_column(String(30), default="unchecked", nullable=False, index=True)
|
integrity_status: Mapped[str] = mapped_column(
|
||||||
integrity_checked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
|
String(30), default="unchecked", nullable=False, index=True
|
||||||
|
)
|
||||||
|
integrity_checked_at: Mapped[datetime | None] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=True, index=True
|
||||||
|
)
|
||||||
integrity_failure: Mapped[str | None] = mapped_column(String(100), nullable=True)
|
integrity_failure: Mapped[str | None] = mapped_column(String(100), nullable=True)
|
||||||
quarantined_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
|
quarantined_at: Mapped[datetime | None] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=True, index=True
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class FileIntegrityScan(Base, TimestampMixin):
|
class FileIntegrityScan(Base, TimestampMixin):
|
||||||
@@ -53,20 +76,35 @@ class FileIntegrityScan(Base, TimestampMixin):
|
|||||||
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||||
storage_backend: Mapped[str] = mapped_column(String(50), nullable=False)
|
storage_backend: Mapped[str] = mapped_column(String(50), nullable=False)
|
||||||
storage_prefix: Mapped[str] = mapped_column(String(1000), nullable=False)
|
storage_prefix: Mapped[str] = mapped_column(String(1000), nullable=False)
|
||||||
status: Mapped[str] = mapped_column(String(30), default="pending", nullable=False, index=True)
|
status: Mapped[str] = mapped_column(
|
||||||
|
String(30), default="pending", nullable=False, index=True
|
||||||
|
)
|
||||||
|
revision: Mapped[int] = mapped_column(Integer, default=1, nullable=False)
|
||||||
phase: Mapped[str] = mapped_column(String(30), default="blobs", nullable=False)
|
phase: Mapped[str] = mapped_column(String(30), default="blobs", nullable=False)
|
||||||
verify_checksums: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False)
|
verify_checksums: Mapped[bool] = mapped_column(
|
||||||
|
Boolean, default=True, nullable=False
|
||||||
|
)
|
||||||
batch_size: Mapped[int] = mapped_column(Integer, default=100, nullable=False)
|
batch_size: Mapped[int] = mapped_column(Integer, default=100, nullable=False)
|
||||||
blob_cursor: Mapped[str | None] = mapped_column(String(36), nullable=True)
|
blob_cursor: Mapped[str | None] = mapped_column(String(36), nullable=True)
|
||||||
object_cursor: Mapped[str | None] = mapped_column(String(1000), nullable=True)
|
object_cursor: Mapped[str | None] = mapped_column(String(1000), nullable=True)
|
||||||
scanned_blob_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
|
scanned_blob_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
|
||||||
verified_blob_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
|
verified_blob_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
|
||||||
quarantined_blob_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
|
quarantined_blob_count: Mapped[int] = mapped_column(
|
||||||
scanned_object_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
|
Integer, default=0, nullable=False
|
||||||
|
)
|
||||||
|
scanned_object_count: Mapped[int] = mapped_column(
|
||||||
|
Integer, default=0, nullable=False
|
||||||
|
)
|
||||||
orphan_object_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
|
orphan_object_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
|
||||||
created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
|
created_by_user_id: Mapped[str | None] = mapped_column(
|
||||||
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True
|
||||||
completed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
)
|
||||||
|
started_at: Mapped[datetime | None] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=True
|
||||||
|
)
|
||||||
|
completed_at: Mapped[datetime | None] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=True
|
||||||
|
)
|
||||||
last_error: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
last_error: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||||
|
|
||||||
|
|
||||||
@@ -77,18 +115,35 @@ class FileIntegrityFinding(Base, TimestampMixin):
|
|||||||
)
|
)
|
||||||
|
|
||||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||||
scan_id: Mapped[str] = mapped_column(ForeignKey("file_integrity_scans.id", ondelete="CASCADE"), nullable=False, index=True)
|
scan_id: Mapped[str] = mapped_column(
|
||||||
|
ForeignKey("file_integrity_scans.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||||
kind: Mapped[str] = mapped_column(String(40), nullable=False, index=True)
|
kind: Mapped[str] = mapped_column(String(40), nullable=False, index=True)
|
||||||
state: Mapped[str] = mapped_column(String(30), default="open", nullable=False, index=True)
|
state: Mapped[str] = mapped_column(
|
||||||
blob_id: Mapped[str | None] = mapped_column(ForeignKey("file_blobs.id", ondelete="SET NULL"), nullable=True, index=True)
|
String(30), default="open", nullable=False, index=True
|
||||||
|
)
|
||||||
|
revision: Mapped[int] = mapped_column(Integer, default=1, nullable=False)
|
||||||
|
blob_id: Mapped[str | None] = mapped_column(
|
||||||
|
ForeignKey("file_blobs.id", ondelete="SET NULL"), nullable=True, index=True
|
||||||
|
)
|
||||||
storage_key: Mapped[str] = mapped_column(String(1000), nullable=False)
|
storage_key: Mapped[str] = mapped_column(String(1000), nullable=False)
|
||||||
expected_size_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
expected_size_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||||
observed_size_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
observed_size_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||||
expected_checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
expected_checksum_sha256: Mapped[str | None] = mapped_column(
|
||||||
observed_checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
String(64), nullable=True
|
||||||
resolved_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
)
|
||||||
resolved_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
|
observed_checksum_sha256: Mapped[str | None] = mapped_column(
|
||||||
|
String(64), nullable=True
|
||||||
|
)
|
||||||
|
resolved_at: Mapped[datetime | None] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=True
|
||||||
|
)
|
||||||
|
resolved_by_user_id: Mapped[str | None] = mapped_column(
|
||||||
|
ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class FileFolder(Base, TimestampMixin):
|
class FileFolder(Base, TimestampMixin):
|
||||||
@@ -96,14 +151,18 @@ class FileFolder(Base, TimestampMixin):
|
|||||||
__table_args__ = (
|
__table_args__ = (
|
||||||
Index(
|
Index(
|
||||||
"uq_file_folders_active_user_path",
|
"uq_file_folders_active_user_path",
|
||||||
"tenant_id", "owner_user_id", "path",
|
"tenant_id",
|
||||||
|
"owner_user_id",
|
||||||
|
"path",
|
||||||
unique=True,
|
unique=True,
|
||||||
sqlite_where=text("owner_type = 'user' AND deleted_at IS NULL"),
|
sqlite_where=text("owner_type = 'user' AND deleted_at IS NULL"),
|
||||||
postgresql_where=text("owner_type = 'user' AND deleted_at IS NULL"),
|
postgresql_where=text("owner_type = 'user' AND deleted_at IS NULL"),
|
||||||
),
|
),
|
||||||
Index(
|
Index(
|
||||||
"uq_file_folders_active_group_path",
|
"uq_file_folders_active_group_path",
|
||||||
"tenant_id", "owner_group_id", "path",
|
"tenant_id",
|
||||||
|
"owner_group_id",
|
||||||
|
"path",
|
||||||
unique=True,
|
unique=True,
|
||||||
sqlite_where=text("owner_type = 'group' AND deleted_at IS NULL"),
|
sqlite_where=text("owner_type = 'group' AND deleted_at IS NULL"),
|
||||||
postgresql_where=text("owner_type = 'group' AND deleted_at IS NULL"),
|
postgresql_where=text("owner_type = 'group' AND deleted_at IS NULL"),
|
||||||
@@ -113,12 +172,22 @@ class FileFolder(Base, TimestampMixin):
|
|||||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||||
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||||
owner_type: Mapped[str] = mapped_column(String(20), nullable=False, index=True)
|
owner_type: Mapped[str] = mapped_column(String(20), nullable=False, index=True)
|
||||||
owner_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
|
owner_user_id: Mapped[str | None] = mapped_column(
|
||||||
owner_group_id: Mapped[str | None] = mapped_column(ForeignKey("access_groups.id", ondelete="SET NULL"), nullable=True, index=True)
|
ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True
|
||||||
|
)
|
||||||
|
owner_group_id: Mapped[str | None] = mapped_column(
|
||||||
|
ForeignKey("access_groups.id", ondelete="SET NULL"), nullable=True, index=True
|
||||||
|
)
|
||||||
path: Mapped[str] = mapped_column(String(1000), nullable=False, index=True)
|
path: Mapped[str] = mapped_column(String(1000), nullable=False, index=True)
|
||||||
created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
|
created_by_user_id: Mapped[str | None] = mapped_column(
|
||||||
deleted_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
|
ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True
|
||||||
metadata_: Mapped[dict[str, Any] | None] = mapped_column("metadata", JSON, nullable=True)
|
)
|
||||||
|
deleted_at: Mapped[datetime | None] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=True, index=True
|
||||||
|
)
|
||||||
|
metadata_: Mapped[dict[str, Any] | None] = mapped_column(
|
||||||
|
"metadata", JSON, nullable=True
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class FileAsset(Base, TimestampMixin):
|
class FileAsset(Base, TimestampMixin):
|
||||||
@@ -127,48 +196,159 @@ class FileAsset(Base, TimestampMixin):
|
|||||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||||
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||||
owner_type: Mapped[str] = mapped_column(String(20), nullable=False, index=True)
|
owner_type: Mapped[str] = mapped_column(String(20), nullable=False, index=True)
|
||||||
owner_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
|
owner_user_id: Mapped[str | None] = mapped_column(
|
||||||
owner_group_id: Mapped[str | None] = mapped_column(ForeignKey("access_groups.id", ondelete="SET NULL"), nullable=True, index=True)
|
ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True
|
||||||
current_version_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
|
)
|
||||||
|
owner_group_id: Mapped[str | None] = mapped_column(
|
||||||
|
ForeignKey("access_groups.id", ondelete="SET NULL"), nullable=True, index=True
|
||||||
|
)
|
||||||
|
current_version_id: Mapped[str | None] = mapped_column(
|
||||||
|
String(36), nullable=True, index=True
|
||||||
|
)
|
||||||
display_path: Mapped[str] = mapped_column(String(1000), nullable=False, index=True)
|
display_path: Mapped[str] = mapped_column(String(1000), nullable=False, index=True)
|
||||||
filename: Mapped[str] = mapped_column(String(500), nullable=False, index=True)
|
filename: Mapped[str] = mapped_column(String(500), nullable=False, index=True)
|
||||||
description: Mapped[str | None] = mapped_column(Text)
|
description: Mapped[str | None] = mapped_column(Text)
|
||||||
created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
|
created_by_user_id: Mapped[str | None] = mapped_column(
|
||||||
deleted_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
|
ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True
|
||||||
metadata_: Mapped[dict[str, Any] | None] = mapped_column("metadata", JSON, nullable=True)
|
)
|
||||||
|
deleted_at: Mapped[datetime | None] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=True, index=True
|
||||||
|
)
|
||||||
|
retained_until: Mapped[datetime | None] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=True, index=True
|
||||||
|
)
|
||||||
|
legal_hold: Mapped[bool] = mapped_column(
|
||||||
|
Boolean, default=False, nullable=False, index=True
|
||||||
|
)
|
||||||
|
lifecycle_revision: Mapped[int] = mapped_column(
|
||||||
|
Integer, default=1, nullable=False
|
||||||
|
)
|
||||||
|
lifecycle_reason: Mapped[str | None] = mapped_column(String(500), nullable=True)
|
||||||
|
metadata_: Mapped[dict[str, Any] | None] = mapped_column(
|
||||||
|
"metadata", JSON, nullable=True
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class FileVersion(Base, TimestampMixin):
|
class FileVersion(Base, TimestampMixin):
|
||||||
__tablename__ = "file_versions"
|
__tablename__ = "file_versions"
|
||||||
__table_args__ = (UniqueConstraint("file_asset_id", "version_number", name="uq_file_versions_asset_number"),)
|
__table_args__ = (
|
||||||
|
UniqueConstraint(
|
||||||
|
"file_asset_id", "version_number", name="uq_file_versions_asset_number"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||||
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||||
file_asset_id: Mapped[str] = mapped_column(ForeignKey("file_assets.id", ondelete="CASCADE"), nullable=False, index=True)
|
file_asset_id: Mapped[str] = mapped_column(
|
||||||
blob_id: Mapped[str] = mapped_column(ForeignKey("file_blobs.id", ondelete="RESTRICT"), nullable=False, index=True)
|
ForeignKey("file_assets.id", ondelete="CASCADE"), nullable=False, index=True
|
||||||
|
)
|
||||||
|
blob_id: Mapped[str] = mapped_column(
|
||||||
|
ForeignKey("file_blobs.id", ondelete="RESTRICT"), nullable=False, index=True
|
||||||
|
)
|
||||||
version_number: Mapped[int] = mapped_column(Integer, nullable=False)
|
version_number: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||||
filename_at_upload: Mapped[str] = mapped_column(String(500), nullable=False)
|
filename_at_upload: Mapped[str] = mapped_column(String(500), nullable=False)
|
||||||
display_path_at_upload: Mapped[str] = mapped_column(String(1000), nullable=False)
|
display_path_at_upload: Mapped[str] = mapped_column(String(1000), nullable=False)
|
||||||
content_type: Mapped[str | None] = mapped_column(String(255))
|
content_type: Mapped[str | None] = mapped_column(String(255))
|
||||||
size_bytes: Mapped[int] = mapped_column(Integer, nullable=False)
|
size_bytes: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||||
checksum_sha256: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
checksum_sha256: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||||
created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
|
created_by_user_id: Mapped[str | None] = mapped_column(
|
||||||
|
ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class FileFormEvidenceGrant(Base, TimestampMixin):
|
||||||
|
__tablename__ = "file_form_evidence_grants"
|
||||||
|
__table_args__ = (
|
||||||
|
UniqueConstraint(
|
||||||
|
"tenant_id",
|
||||||
|
"idempotency_key",
|
||||||
|
name="uq_file_form_evidence_grants_idempotency",
|
||||||
|
),
|
||||||
|
Index(
|
||||||
|
"ix_file_form_evidence_grants_form",
|
||||||
|
"tenant_id",
|
||||||
|
"form_instance_id",
|
||||||
|
"form_definition_id",
|
||||||
|
"form_definition_revision",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||||
|
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||||
|
form_instance_id: Mapped[str] = mapped_column(
|
||||||
|
String(36), nullable=False, index=True
|
||||||
|
)
|
||||||
|
form_definition_id: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||||
|
form_definition_revision: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||||
|
token_sha256: Mapped[str] = mapped_column(String(64), nullable=False, unique=True)
|
||||||
|
idempotency_key: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||||
|
request_sha256: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||||
|
custodian_user_id: Mapped[str] = mapped_column(
|
||||||
|
ForeignKey("access_users.id", ondelete="RESTRICT"),
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
evidence_kind: Mapped[str] = mapped_column(String(30), nullable=False)
|
||||||
|
purpose: Mapped[str] = mapped_column(String(500), nullable=False)
|
||||||
|
status: Mapped[str] = mapped_column(
|
||||||
|
String(30), default="issued", nullable=False, index=True
|
||||||
|
)
|
||||||
|
expires_at: Mapped[datetime] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=False, index=True
|
||||||
|
)
|
||||||
|
max_size_bytes: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||||
|
allowed_content_types: Mapped[list[str]] = mapped_column(
|
||||||
|
JSON, default=list, nullable=False
|
||||||
|
)
|
||||||
|
file_asset_id: Mapped[str | None] = mapped_column(
|
||||||
|
ForeignKey("file_assets.id", ondelete="RESTRICT"), nullable=True, index=True
|
||||||
|
)
|
||||||
|
file_version_id: Mapped[str | None] = mapped_column(
|
||||||
|
ForeignKey("file_versions.id", ondelete="RESTRICT"), nullable=True, index=True
|
||||||
|
)
|
||||||
|
uploaded_at: Mapped[datetime | None] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=True
|
||||||
|
)
|
||||||
|
revoked_at: Mapped[datetime | None] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=True
|
||||||
|
)
|
||||||
|
metadata_: Mapped[dict[str, Any]] = mapped_column(
|
||||||
|
"metadata", JSON, default=dict, nullable=False
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class FileShare(Base, TimestampMixin):
|
class FileShare(Base, TimestampMixin):
|
||||||
__tablename__ = "file_shares"
|
__tablename__ = "file_shares"
|
||||||
__table_args__ = (UniqueConstraint("file_asset_id", "target_type", "target_id", "revoked_at", name="uq_file_shares_active_target"),)
|
__table_args__ = (
|
||||||
|
UniqueConstraint(
|
||||||
|
"file_asset_id",
|
||||||
|
"target_type",
|
||||||
|
"target_id",
|
||||||
|
"revoked_at",
|
||||||
|
name="uq_file_shares_active_target",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||||
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||||
file_asset_id: Mapped[str] = mapped_column(ForeignKey("file_assets.id", ondelete="CASCADE"), nullable=False, index=True)
|
file_asset_id: Mapped[str] = mapped_column(
|
||||||
|
ForeignKey("file_assets.id", ondelete="CASCADE"), nullable=False, index=True
|
||||||
|
)
|
||||||
target_type: Mapped[str] = mapped_column(String(20), nullable=False, index=True)
|
target_type: Mapped[str] = mapped_column(String(20), nullable=False, index=True)
|
||||||
target_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
target_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||||
permission: Mapped[str] = mapped_column(String(20), default="read", nullable=False)
|
permission: Mapped[str] = mapped_column(String(20), default="read", nullable=False)
|
||||||
created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
|
created_by_user_id: Mapped[str | None] = mapped_column(
|
||||||
expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
|
ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True
|
||||||
revoked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
|
)
|
||||||
revoked_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
|
expires_at: Mapped[datetime | None] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=True, index=True
|
||||||
|
)
|
||||||
|
revoked_at: Mapped[datetime | None] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=True, index=True
|
||||||
|
)
|
||||||
|
revoked_by_user_id: Mapped[str | None] = mapped_column(
|
||||||
|
ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class FileConnectorProfile(Base, TimestampMixin):
|
class FileConnectorProfile(Base, TimestampMixin):
|
||||||
@@ -179,15 +359,23 @@ class FileConnectorProfile(Base, TimestampMixin):
|
|||||||
|
|
||||||
id: Mapped[str] = mapped_column(String(255), primary_key=True)
|
id: Mapped[str] = mapped_column(String(255), primary_key=True)
|
||||||
tenant_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
|
tenant_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
|
||||||
scope_type: Mapped[str] = mapped_column(String(20), default="tenant", nullable=False, index=True)
|
scope_type: Mapped[str] = mapped_column(
|
||||||
|
String(20), default="tenant", nullable=False, index=True
|
||||||
|
)
|
||||||
scope_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
|
scope_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
|
||||||
label: Mapped[str] = mapped_column(String(255), nullable=False)
|
label: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||||
provider: Mapped[str] = mapped_column(String(50), nullable=False, index=True)
|
provider: Mapped[str] = mapped_column(String(50), nullable=False, index=True)
|
||||||
endpoint_url: Mapped[str | None] = mapped_column(String(1000), nullable=True)
|
endpoint_url: Mapped[str | None] = mapped_column(String(1000), nullable=True)
|
||||||
base_path: Mapped[str | None] = mapped_column(String(1000), nullable=True)
|
base_path: Mapped[str | None] = mapped_column(String(1000), nullable=True)
|
||||||
enabled: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False, index=True)
|
enabled: Mapped[bool] = mapped_column(
|
||||||
credential_profile_id: Mapped[str | None] = mapped_column(String(255), nullable=True, index=True)
|
Boolean, default=True, nullable=False, index=True
|
||||||
credential_mode: Mapped[str] = mapped_column(String(30), default="none", nullable=False)
|
)
|
||||||
|
credential_profile_id: Mapped[str | None] = mapped_column(
|
||||||
|
String(255), nullable=True, index=True
|
||||||
|
)
|
||||||
|
credential_mode: Mapped[str] = mapped_column(
|
||||||
|
String(30), default="none", nullable=False
|
||||||
|
)
|
||||||
username: Mapped[str | None] = mapped_column(String(320), nullable=True)
|
username: Mapped[str | None] = mapped_column(String(320), nullable=True)
|
||||||
password_encrypted: Mapped[str | None] = mapped_column(Text, nullable=True)
|
password_encrypted: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
token_encrypted: Mapped[str | None] = mapped_column(Text, nullable=True)
|
token_encrypted: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
@@ -196,9 +384,15 @@ class FileConnectorProfile(Base, TimestampMixin):
|
|||||||
secret_ref: Mapped[str | None] = mapped_column(String(1000), nullable=True)
|
secret_ref: Mapped[str | None] = mapped_column(String(1000), nullable=True)
|
||||||
capabilities: Mapped[list[str] | None] = mapped_column(JSON, nullable=True)
|
capabilities: Mapped[list[str] | None] = mapped_column(JSON, nullable=True)
|
||||||
policy: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
|
policy: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
|
||||||
metadata_: Mapped[dict[str, Any] | None] = mapped_column("metadata", JSON, nullable=True)
|
metadata_: Mapped[dict[str, Any] | None] = mapped_column(
|
||||||
created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
|
"metadata", JSON, nullable=True
|
||||||
updated_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
|
)
|
||||||
|
created_by_user_id: Mapped[str | None] = mapped_column(
|
||||||
|
ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True
|
||||||
|
)
|
||||||
|
updated_by_user_id: Mapped[str | None] = mapped_column(
|
||||||
|
ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class FileConnectorCredential(Base, TimestampMixin):
|
class FileConnectorCredential(Base, TimestampMixin):
|
||||||
@@ -209,12 +403,18 @@ class FileConnectorCredential(Base, TimestampMixin):
|
|||||||
|
|
||||||
id: Mapped[str] = mapped_column(String(255), primary_key=True)
|
id: Mapped[str] = mapped_column(String(255), primary_key=True)
|
||||||
tenant_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
|
tenant_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
|
||||||
scope_type: Mapped[str] = mapped_column(String(20), default="tenant", nullable=False, index=True)
|
scope_type: Mapped[str] = mapped_column(
|
||||||
|
String(20), default="tenant", nullable=False, index=True
|
||||||
|
)
|
||||||
scope_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
|
scope_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
|
||||||
label: Mapped[str] = mapped_column(String(255), nullable=False)
|
label: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||||
provider: Mapped[str | None] = mapped_column(String(50), nullable=True, index=True)
|
provider: Mapped[str | None] = mapped_column(String(50), nullable=True, index=True)
|
||||||
enabled: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False, index=True)
|
enabled: Mapped[bool] = mapped_column(
|
||||||
credential_mode: Mapped[str] = mapped_column(String(30), default="none", nullable=False)
|
Boolean, default=True, nullable=False, index=True
|
||||||
|
)
|
||||||
|
credential_mode: Mapped[str] = mapped_column(
|
||||||
|
String(30), default="none", nullable=False
|
||||||
|
)
|
||||||
username: Mapped[str | None] = mapped_column(String(320), nullable=True)
|
username: Mapped[str | None] = mapped_column(String(320), nullable=True)
|
||||||
password_encrypted: Mapped[str | None] = mapped_column(Text, nullable=True)
|
password_encrypted: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
token_encrypted: Mapped[str | None] = mapped_column(Text, nullable=True)
|
token_encrypted: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
@@ -222,15 +422,26 @@ class FileConnectorCredential(Base, TimestampMixin):
|
|||||||
token_env: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
token_env: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||||
secret_ref: Mapped[str | None] = mapped_column(String(1000), nullable=True)
|
secret_ref: Mapped[str | None] = mapped_column(String(1000), nullable=True)
|
||||||
policy: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
|
policy: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
|
||||||
metadata_: Mapped[dict[str, Any] | None] = mapped_column("metadata", JSON, nullable=True)
|
metadata_: Mapped[dict[str, Any] | None] = mapped_column(
|
||||||
created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
|
"metadata", JSON, nullable=True
|
||||||
updated_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
|
)
|
||||||
|
created_by_user_id: Mapped[str | None] = mapped_column(
|
||||||
|
ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True
|
||||||
|
)
|
||||||
|
updated_by_user_id: Mapped[str | None] = mapped_column(
|
||||||
|
ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class FileConnectorPolicy(Base, TimestampMixin):
|
class FileConnectorPolicy(Base, TimestampMixin):
|
||||||
__tablename__ = "file_connector_policies"
|
__tablename__ = "file_connector_policies"
|
||||||
__table_args__ = (
|
__table_args__ = (
|
||||||
UniqueConstraint("tenant_id", "scope_type", "scope_id", name="uq_file_connector_policies_scope"),
|
UniqueConstraint(
|
||||||
|
"tenant_id",
|
||||||
|
"scope_type",
|
||||||
|
"scope_id",
|
||||||
|
name="uq_file_connector_policies_scope",
|
||||||
|
),
|
||||||
Index("ix_file_connector_policies_scope", "scope_type", "scope_id"),
|
Index("ix_file_connector_policies_scope", "scope_type", "scope_id"),
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -239,24 +450,38 @@ class FileConnectorPolicy(Base, TimestampMixin):
|
|||||||
scope_type: Mapped[str] = mapped_column(String(20), nullable=False, index=True)
|
scope_type: Mapped[str] = mapped_column(String(20), nullable=False, index=True)
|
||||||
scope_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
|
scope_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
|
||||||
policy: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict, nullable=False)
|
policy: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict, nullable=False)
|
||||||
created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
|
created_by_user_id: Mapped[str | None] = mapped_column(
|
||||||
updated_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
|
ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True
|
||||||
|
)
|
||||||
|
updated_by_user_id: Mapped[str | None] = mapped_column(
|
||||||
|
ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class FileConnectorSpace(Base, TimestampMixin):
|
class FileConnectorSpace(Base, TimestampMixin):
|
||||||
__tablename__ = "file_connector_spaces"
|
__tablename__ = "file_connector_spaces"
|
||||||
__table_args__ = (
|
__table_args__ = (
|
||||||
Index("ix_file_connector_spaces_owner", "tenant_id", "owner_type", "owner_user_id", "owner_group_id"),
|
Index(
|
||||||
|
"ix_file_connector_spaces_owner",
|
||||||
|
"tenant_id",
|
||||||
|
"owner_type",
|
||||||
|
"owner_user_id",
|
||||||
|
"owner_group_id",
|
||||||
|
),
|
||||||
Index(
|
Index(
|
||||||
"uq_file_connector_spaces_active_user_label",
|
"uq_file_connector_spaces_active_user_label",
|
||||||
"tenant_id", "owner_user_id", "label",
|
"tenant_id",
|
||||||
|
"owner_user_id",
|
||||||
|
"label",
|
||||||
unique=True,
|
unique=True,
|
||||||
sqlite_where=text("owner_type = 'user' AND deleted_at IS NULL"),
|
sqlite_where=text("owner_type = 'user' AND deleted_at IS NULL"),
|
||||||
postgresql_where=text("owner_type = 'user' AND deleted_at IS NULL"),
|
postgresql_where=text("owner_type = 'user' AND deleted_at IS NULL"),
|
||||||
),
|
),
|
||||||
Index(
|
Index(
|
||||||
"uq_file_connector_spaces_active_group_label",
|
"uq_file_connector_spaces_active_group_label",
|
||||||
"tenant_id", "owner_group_id", "label",
|
"tenant_id",
|
||||||
|
"owner_group_id",
|
||||||
|
"label",
|
||||||
unique=True,
|
unique=True,
|
||||||
sqlite_where=text("owner_type = 'group' AND deleted_at IS NULL"),
|
sqlite_where=text("owner_type = 'group' AND deleted_at IS NULL"),
|
||||||
postgresql_where=text("owner_type = 'group' AND deleted_at IS NULL"),
|
postgresql_where=text("owner_type = 'group' AND deleted_at IS NULL"),
|
||||||
@@ -266,41 +491,81 @@ class FileConnectorSpace(Base, TimestampMixin):
|
|||||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||||
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||||
owner_type: Mapped[str] = mapped_column(String(20), nullable=False, index=True)
|
owner_type: Mapped[str] = mapped_column(String(20), nullable=False, index=True)
|
||||||
owner_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
|
owner_user_id: Mapped[str | None] = mapped_column(
|
||||||
owner_group_id: Mapped[str | None] = mapped_column(ForeignKey("access_groups.id", ondelete="SET NULL"), nullable=True, index=True)
|
ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True
|
||||||
|
)
|
||||||
|
owner_group_id: Mapped[str | None] = mapped_column(
|
||||||
|
ForeignKey("access_groups.id", ondelete="SET NULL"), nullable=True, index=True
|
||||||
|
)
|
||||||
label: Mapped[str] = mapped_column(String(255), nullable=False)
|
label: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||||
connector_profile_id: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
|
connector_profile_id: Mapped[str] = mapped_column(
|
||||||
|
String(255), nullable=False, index=True
|
||||||
|
)
|
||||||
provider: Mapped[str] = mapped_column(String(50), nullable=False, index=True)
|
provider: Mapped[str] = mapped_column(String(50), nullable=False, index=True)
|
||||||
library_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
library_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||||
remote_path: Mapped[str] = mapped_column(String(1000), default="", nullable=False)
|
remote_path: Mapped[str] = mapped_column(String(1000), default="", nullable=False)
|
||||||
sync_mode: Mapped[str] = mapped_column(String(30), default="manual", nullable=False)
|
sync_mode: Mapped[str] = mapped_column(String(30), default="manual", nullable=False)
|
||||||
read_only: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False)
|
read_only: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False)
|
||||||
is_active: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False, index=True)
|
is_active: Mapped[bool] = mapped_column(
|
||||||
created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
|
Boolean, default=True, nullable=False, index=True
|
||||||
deleted_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
|
)
|
||||||
metadata_: Mapped[dict[str, Any] | None] = mapped_column("metadata", JSON, nullable=True)
|
created_by_user_id: Mapped[str | None] = mapped_column(
|
||||||
|
ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True
|
||||||
|
)
|
||||||
|
deleted_at: Mapped[datetime | None] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=True, index=True
|
||||||
|
)
|
||||||
|
metadata_: Mapped[dict[str, Any] | None] = mapped_column(
|
||||||
|
"metadata", JSON, nullable=True
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class CampaignAttachmentUse(Base, TimestampMixin):
|
class CampaignAttachmentUse(Base, TimestampMixin):
|
||||||
__tablename__ = "campaign_attachment_uses"
|
__tablename__ = "campaign_attachment_uses"
|
||||||
__table_args__ = (UniqueConstraint("campaign_job_id", "file_version_id", "filename_used", "use_stage", name="uq_campaign_attachment_uses_job_file_stage"),)
|
__table_args__ = (
|
||||||
|
UniqueConstraint(
|
||||||
|
"campaign_job_id",
|
||||||
|
"file_version_id",
|
||||||
|
"filename_used",
|
||||||
|
"use_stage",
|
||||||
|
name="uq_campaign_attachment_uses_job_file_stage",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||||
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||||
campaign_id: Mapped[str] = mapped_column(ForeignKey("campaigns.id", ondelete="CASCADE"), nullable=False, index=True)
|
campaign_id: Mapped[str] = mapped_column(
|
||||||
campaign_version_id: Mapped[str] = mapped_column(ForeignKey("campaign_versions.id", ondelete="CASCADE"), nullable=False, index=True)
|
ForeignKey("campaigns.id", ondelete="CASCADE"), nullable=False, index=True
|
||||||
campaign_job_id: Mapped[str | None] = mapped_column(ForeignKey("campaign_jobs.id", ondelete="SET NULL"), nullable=True, index=True)
|
)
|
||||||
|
campaign_version_id: Mapped[str] = mapped_column(
|
||||||
|
ForeignKey("campaign_versions.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
campaign_job_id: Mapped[str | None] = mapped_column(
|
||||||
|
ForeignKey("campaign_jobs.id", ondelete="SET NULL"), nullable=True, index=True
|
||||||
|
)
|
||||||
entry_index: Mapped[int | None] = mapped_column(Integer)
|
entry_index: Mapped[int | None] = mapped_column(Integer)
|
||||||
entry_id: Mapped[str | None] = mapped_column(String(255), index=True)
|
entry_id: Mapped[str | None] = mapped_column(String(255), index=True)
|
||||||
file_asset_id: Mapped[str] = mapped_column(ForeignKey("file_assets.id", ondelete="RESTRICT"), nullable=False, index=True)
|
file_asset_id: Mapped[str] = mapped_column(
|
||||||
file_version_id: Mapped[str] = mapped_column(ForeignKey("file_versions.id", ondelete="RESTRICT"), nullable=False, index=True)
|
ForeignKey("file_assets.id", ondelete="RESTRICT"), nullable=False, index=True
|
||||||
file_blob_id: Mapped[str] = mapped_column(ForeignKey("file_blobs.id", ondelete="RESTRICT"), nullable=False, index=True)
|
)
|
||||||
|
file_version_id: Mapped[str] = mapped_column(
|
||||||
|
ForeignKey("file_versions.id", ondelete="RESTRICT"), nullable=False, index=True
|
||||||
|
)
|
||||||
|
file_blob_id: Mapped[str] = mapped_column(
|
||||||
|
ForeignKey("file_blobs.id", ondelete="RESTRICT"), nullable=False, index=True
|
||||||
|
)
|
||||||
filename_used: Mapped[str] = mapped_column(String(500), nullable=False)
|
filename_used: Mapped[str] = mapped_column(String(500), nullable=False)
|
||||||
checksum_sha256: Mapped[str] = mapped_column(String(64), nullable=False)
|
checksum_sha256: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||||
size_bytes: Mapped[int] = mapped_column(Integer, nullable=False)
|
size_bytes: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||||
content_type: Mapped[str | None] = mapped_column(String(255))
|
content_type: Mapped[str | None] = mapped_column(String(255))
|
||||||
use_stage: Mapped[str] = mapped_column(String(20), default="built", nullable=False, index=True)
|
use_stage: Mapped[str] = mapped_column(
|
||||||
used_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
|
String(20), default="built", nullable=False, index=True
|
||||||
|
)
|
||||||
|
used_at: Mapped[datetime | None] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=True, index=True
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
@@ -312,6 +577,7 @@ __all__ = [
|
|||||||
"FileConnectorProfile",
|
"FileConnectorProfile",
|
||||||
"FileConnectorSpace",
|
"FileConnectorSpace",
|
||||||
"FileFolder",
|
"FileFolder",
|
||||||
|
"FileFormEvidenceGrant",
|
||||||
"FileShare",
|
"FileShare",
|
||||||
"FileVersion",
|
"FileVersion",
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -0,0 +1,879 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from collections.abc import Sequence
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
|
from sqlalchemy import inspect, or_
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from govoplan_core.core.dsar import (
|
||||||
|
DsarErasureActionRef,
|
||||||
|
DsarExecutionResultRef,
|
||||||
|
DsarRecordRef,
|
||||||
|
DsarSubjectRef,
|
||||||
|
dsar_capability_name,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.db.models import (
|
||||||
|
CampaignAttachmentUse,
|
||||||
|
FileAsset,
|
||||||
|
FileConnectorCredential,
|
||||||
|
FileConnectorPolicy,
|
||||||
|
FileConnectorProfile,
|
||||||
|
FileConnectorSpace,
|
||||||
|
FileFolder,
|
||||||
|
FileFormEvidenceGrant,
|
||||||
|
FileIntegrityFinding,
|
||||||
|
FileIntegrityScan,
|
||||||
|
FileShare,
|
||||||
|
FileVersion,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
FILES_DSAR_CAPABILITY = dsar_capability_name("files")
|
||||||
|
_MAX_RECORDS = 5_000
|
||||||
|
|
||||||
|
|
||||||
|
class FilesDsarProvider:
|
||||||
|
provider_id = "files"
|
||||||
|
module_id = "files"
|
||||||
|
|
||||||
|
def search_subject(
|
||||||
|
self,
|
||||||
|
session: object,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
subject: DsarSubjectRef,
|
||||||
|
) -> Sequence[DsarRecordRef]:
|
||||||
|
db = _session(session)
|
||||||
|
subject_user_id = _subject_user_id(subject)
|
||||||
|
if subject_user_id is None:
|
||||||
|
return ()
|
||||||
|
|
||||||
|
records: list[DsarRecordRef] = []
|
||||||
|
form_evidence_available = _has_table(db, FileFormEvidenceGrant)
|
||||||
|
campaign_evidence_available = _has_table(db, CampaignAttachmentUse)
|
||||||
|
retention_reasons: dict[str, str | None] = {}
|
||||||
|
|
||||||
|
def retention_reason(asset: FileAsset) -> str | None:
|
||||||
|
if asset.id not in retention_reasons:
|
||||||
|
retention_reasons[asset.id] = _asset_retention_reason(
|
||||||
|
db,
|
||||||
|
asset,
|
||||||
|
form_evidence_available=form_evidence_available,
|
||||||
|
campaign_evidence_available=campaign_evidence_available,
|
||||||
|
)
|
||||||
|
return retention_reasons[asset.id]
|
||||||
|
|
||||||
|
def append(record: DsarRecordRef) -> None:
|
||||||
|
if len(records) >= _MAX_RECORDS:
|
||||||
|
raise ValueError(
|
||||||
|
"Files DSAR match limit exceeded; narrow the subject selectors."
|
||||||
|
)
|
||||||
|
records.append(record)
|
||||||
|
|
||||||
|
assets = _bounded_rows(
|
||||||
|
db.query(FileAsset)
|
||||||
|
.filter(
|
||||||
|
FileAsset.tenant_id == tenant_id,
|
||||||
|
or_(
|
||||||
|
FileAsset.owner_user_id == subject_user_id,
|
||||||
|
FileAsset.created_by_user_id == subject_user_id,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.order_by(FileAsset.id)
|
||||||
|
)
|
||||||
|
for asset in assets:
|
||||||
|
match_fields = _matching_fields(
|
||||||
|
asset,
|
||||||
|
subject_user_id,
|
||||||
|
("owner_user_id", "created_by_user_id"),
|
||||||
|
)
|
||||||
|
asset_retention_reason = retention_reason(asset)
|
||||||
|
append(
|
||||||
|
_record(
|
||||||
|
"file_asset",
|
||||||
|
asset.id,
|
||||||
|
"managed_file",
|
||||||
|
asset.filename,
|
||||||
|
{
|
||||||
|
"match_fields": match_fields,
|
||||||
|
"owner_type": asset.owner_type,
|
||||||
|
"display_path": asset.display_path,
|
||||||
|
"filename": asset.filename,
|
||||||
|
"description": asset.description,
|
||||||
|
"deleted_at": _iso(asset.deleted_at),
|
||||||
|
"retained_until": _iso(asset.retained_until),
|
||||||
|
"legal_hold": asset.legal_hold,
|
||||||
|
"lifecycle_reason": asset.lifecycle_reason,
|
||||||
|
"lifecycle_revision": asset.lifecycle_revision,
|
||||||
|
},
|
||||||
|
observed_at=asset.updated_at,
|
||||||
|
immutable=asset_retention_reason is not None,
|
||||||
|
retention_reason=asset_retention_reason,
|
||||||
|
source_path=f"/files?file={asset.id}",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
version_rows = _bounded_rows(
|
||||||
|
db.query(FileVersion, FileAsset)
|
||||||
|
.join(FileAsset, FileAsset.id == FileVersion.file_asset_id)
|
||||||
|
.filter(
|
||||||
|
FileVersion.tenant_id == tenant_id,
|
||||||
|
FileAsset.tenant_id == tenant_id,
|
||||||
|
or_(
|
||||||
|
FileAsset.owner_user_id == subject_user_id,
|
||||||
|
FileVersion.created_by_user_id == subject_user_id,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.order_by(FileVersion.id)
|
||||||
|
)
|
||||||
|
for version, asset in version_rows:
|
||||||
|
asset_retention_reason = retention_reason(asset)
|
||||||
|
append(
|
||||||
|
_record(
|
||||||
|
"file_version",
|
||||||
|
version.id,
|
||||||
|
"managed_file_version",
|
||||||
|
version.filename_at_upload,
|
||||||
|
{
|
||||||
|
"match_fields": (
|
||||||
|
["asset.owner_user_id"]
|
||||||
|
if asset.owner_user_id == subject_user_id
|
||||||
|
else []
|
||||||
|
)
|
||||||
|
+ (
|
||||||
|
["created_by_user_id"]
|
||||||
|
if version.created_by_user_id == subject_user_id
|
||||||
|
else []
|
||||||
|
),
|
||||||
|
"file_asset_id": version.file_asset_id,
|
||||||
|
"version_number": version.version_number,
|
||||||
|
"filename_at_upload": version.filename_at_upload,
|
||||||
|
"display_path_at_upload": version.display_path_at_upload,
|
||||||
|
"content_type": version.content_type,
|
||||||
|
"size_bytes": version.size_bytes,
|
||||||
|
"checksum_sha256": version.checksum_sha256,
|
||||||
|
"created_at": _iso(version.created_at),
|
||||||
|
},
|
||||||
|
observed_at=version.updated_at,
|
||||||
|
immutable=asset_retention_reason is not None,
|
||||||
|
retention_reason=asset_retention_reason,
|
||||||
|
source_path=(
|
||||||
|
f"/api/v1/files/{version.file_asset_id}/versions/"
|
||||||
|
f"{version.id}/download"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
for folder in _bounded_rows(
|
||||||
|
db.query(FileFolder)
|
||||||
|
.filter(
|
||||||
|
FileFolder.tenant_id == tenant_id,
|
||||||
|
or_(
|
||||||
|
FileFolder.owner_user_id == subject_user_id,
|
||||||
|
FileFolder.created_by_user_id == subject_user_id,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.order_by(FileFolder.id)
|
||||||
|
):
|
||||||
|
append(
|
||||||
|
_record(
|
||||||
|
"file_folder",
|
||||||
|
folder.id,
|
||||||
|
"managed_folder",
|
||||||
|
folder.path,
|
||||||
|
{
|
||||||
|
"match_fields": _matching_fields(
|
||||||
|
folder,
|
||||||
|
subject_user_id,
|
||||||
|
("owner_user_id", "created_by_user_id"),
|
||||||
|
),
|
||||||
|
"owner_type": folder.owner_type,
|
||||||
|
"path": folder.path,
|
||||||
|
"deleted_at": _iso(folder.deleted_at),
|
||||||
|
},
|
||||||
|
observed_at=folder.updated_at,
|
||||||
|
source_path="/files",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
for share in _bounded_rows(
|
||||||
|
db.query(FileShare)
|
||||||
|
.filter(
|
||||||
|
FileShare.tenant_id == tenant_id,
|
||||||
|
or_(
|
||||||
|
(FileShare.target_type == "user")
|
||||||
|
& (FileShare.target_id == subject_user_id),
|
||||||
|
FileShare.created_by_user_id == subject_user_id,
|
||||||
|
FileShare.revoked_by_user_id == subject_user_id,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.order_by(FileShare.id)
|
||||||
|
):
|
||||||
|
target_matches = (
|
||||||
|
share.target_type == "user" and share.target_id == subject_user_id
|
||||||
|
)
|
||||||
|
match_fields = _matching_fields(
|
||||||
|
share,
|
||||||
|
subject_user_id,
|
||||||
|
("created_by_user_id", "revoked_by_user_id"),
|
||||||
|
)
|
||||||
|
if target_matches:
|
||||||
|
match_fields.insert(0, "target_id")
|
||||||
|
append(
|
||||||
|
_record(
|
||||||
|
"file_share",
|
||||||
|
share.id,
|
||||||
|
"file_access_evidence",
|
||||||
|
f"Share for file {share.file_asset_id}",
|
||||||
|
{
|
||||||
|
"match_fields": match_fields,
|
||||||
|
"file_asset_id": share.file_asset_id,
|
||||||
|
"target_type": share.target_type,
|
||||||
|
"target_is_subject": target_matches,
|
||||||
|
"permission": share.permission,
|
||||||
|
"expires_at": _iso(share.expires_at),
|
||||||
|
"revoked_at": _iso(share.revoked_at),
|
||||||
|
},
|
||||||
|
observed_at=share.updated_at,
|
||||||
|
immutable=True,
|
||||||
|
retention_reason=(
|
||||||
|
"File-sharing history is institutional access evidence."
|
||||||
|
),
|
||||||
|
source_path=f"/files?file={share.file_asset_id}",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
if form_evidence_available:
|
||||||
|
evidence_rows = _bounded_rows(
|
||||||
|
db.query(FileFormEvidenceGrant)
|
||||||
|
.outerjoin(
|
||||||
|
FileAsset,
|
||||||
|
FileAsset.id == FileFormEvidenceGrant.file_asset_id,
|
||||||
|
)
|
||||||
|
.filter(
|
||||||
|
FileFormEvidenceGrant.tenant_id == tenant_id,
|
||||||
|
or_(
|
||||||
|
FileFormEvidenceGrant.custodian_user_id == subject_user_id,
|
||||||
|
FileAsset.owner_user_id == subject_user_id,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.order_by(FileFormEvidenceGrant.id)
|
||||||
|
)
|
||||||
|
for grant in evidence_rows:
|
||||||
|
append(
|
||||||
|
_record(
|
||||||
|
"file_form_evidence",
|
||||||
|
grant.id,
|
||||||
|
"form_evidence",
|
||||||
|
f"Form evidence {grant.form_definition_id}",
|
||||||
|
{
|
||||||
|
"match_fields": (
|
||||||
|
["custodian_user_id"]
|
||||||
|
if grant.custodian_user_id == subject_user_id
|
||||||
|
else ["asset.owner_user_id"]
|
||||||
|
),
|
||||||
|
"form_instance_id": grant.form_instance_id,
|
||||||
|
"form_definition_id": grant.form_definition_id,
|
||||||
|
"form_definition_revision": grant.form_definition_revision,
|
||||||
|
"evidence_kind": grant.evidence_kind,
|
||||||
|
"purpose": grant.purpose,
|
||||||
|
"status": grant.status,
|
||||||
|
"expires_at": _iso(grant.expires_at),
|
||||||
|
"file_asset_id": grant.file_asset_id,
|
||||||
|
"file_version_id": grant.file_version_id,
|
||||||
|
},
|
||||||
|
observed_at=grant.updated_at,
|
||||||
|
immutable=True,
|
||||||
|
retention_reason=(
|
||||||
|
"Submitted Form attachment evidence follows the owning "
|
||||||
|
"process retention and cannot be erased through Files alone."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
if campaign_evidence_available:
|
||||||
|
campaign_rows = _bounded_rows(
|
||||||
|
db.query(CampaignAttachmentUse)
|
||||||
|
.join(FileAsset, FileAsset.id == CampaignAttachmentUse.file_asset_id)
|
||||||
|
.filter(
|
||||||
|
CampaignAttachmentUse.tenant_id == tenant_id,
|
||||||
|
FileAsset.owner_user_id == subject_user_id,
|
||||||
|
)
|
||||||
|
.order_by(CampaignAttachmentUse.id)
|
||||||
|
)
|
||||||
|
for use in campaign_rows:
|
||||||
|
append(
|
||||||
|
_record(
|
||||||
|
"campaign_attachment_use",
|
||||||
|
use.id,
|
||||||
|
"delivery_evidence",
|
||||||
|
use.filename_used,
|
||||||
|
{
|
||||||
|
"match_fields": ["asset.owner_user_id"],
|
||||||
|
"campaign_id": use.campaign_id,
|
||||||
|
"campaign_version_id": use.campaign_version_id,
|
||||||
|
"campaign_job_id": use.campaign_job_id,
|
||||||
|
"file_asset_id": use.file_asset_id,
|
||||||
|
"file_version_id": use.file_version_id,
|
||||||
|
"filename_used": use.filename_used,
|
||||||
|
"checksum_sha256": use.checksum_sha256,
|
||||||
|
"size_bytes": use.size_bytes,
|
||||||
|
"use_stage": use.use_stage,
|
||||||
|
"used_at": _iso(use.used_at),
|
||||||
|
},
|
||||||
|
observed_at=use.updated_at,
|
||||||
|
immutable=True,
|
||||||
|
retention_reason=(
|
||||||
|
"Campaign attachment use is immutable delivery evidence."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
self._append_configuration_references(
|
||||||
|
db,
|
||||||
|
append=append,
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
subject_user_id=subject_user_id,
|
||||||
|
)
|
||||||
|
self._append_integrity_references(
|
||||||
|
db,
|
||||||
|
append=append,
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
subject_user_id=subject_user_id,
|
||||||
|
)
|
||||||
|
return tuple(records)
|
||||||
|
|
||||||
|
def plan_erasure(
|
||||||
|
self,
|
||||||
|
session: object,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
subject: DsarSubjectRef,
|
||||||
|
records: Sequence[DsarRecordRef],
|
||||||
|
) -> Sequence[DsarErasureActionRef]:
|
||||||
|
del session
|
||||||
|
subject_user_id = _subject_user_id(subject)
|
||||||
|
if subject_user_id is None:
|
||||||
|
return ()
|
||||||
|
actions: list[DsarErasureActionRef] = []
|
||||||
|
for record in records:
|
||||||
|
if record.provider_id != self.provider_id or record.module_id != self.module_id:
|
||||||
|
raise ValueError("Files DSAR received a foreign provider record.")
|
||||||
|
match_fields = {
|
||||||
|
str(value) for value in record.data.get("match_fields", ())
|
||||||
|
}
|
||||||
|
if record.immutable_evidence:
|
||||||
|
actions.append(
|
||||||
|
_action(
|
||||||
|
f"files:retain:{record.resource_type}:{record.resource_id}",
|
||||||
|
"retain",
|
||||||
|
record,
|
||||||
|
f"Retain {record.title}",
|
||||||
|
record.retention_reason
|
||||||
|
or "Institutional evidence must be retained.",
|
||||||
|
executable=False,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
elif record.resource_type in {
|
||||||
|
"file_asset",
|
||||||
|
"file_version",
|
||||||
|
"file_folder",
|
||||||
|
}:
|
||||||
|
actions.append(
|
||||||
|
_action(
|
||||||
|
f"files:review:{record.resource_type}:{record.resource_id}",
|
||||||
|
"manual_review",
|
||||||
|
record,
|
||||||
|
f"Review {record.title}",
|
||||||
|
(
|
||||||
|
"Managed file content, names, paths, ownership, and shared "
|
||||||
|
"references require a case decision. Approved byte erasure "
|
||||||
|
"must use the separate Files purge workflow."
|
||||||
|
),
|
||||||
|
executable=False,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
record.resource_type == "file_share"
|
||||||
|
and "target_id" in match_fields
|
||||||
|
and record.data.get("revoked_at") is None
|
||||||
|
):
|
||||||
|
actions.append(
|
||||||
|
_action(
|
||||||
|
f"files:revoke:file_share:{record.resource_id}",
|
||||||
|
"revoke",
|
||||||
|
record,
|
||||||
|
"Revoke active file share",
|
||||||
|
"The active user-targeted share can be revoked without deleting file evidence.",
|
||||||
|
executable=True,
|
||||||
|
metadata={"subject_user_id": subject_user_id},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
for field_name in sorted(
|
||||||
|
match_fields.intersection(_detachable_fields(record.resource_type))
|
||||||
|
):
|
||||||
|
actions.append(
|
||||||
|
_action(
|
||||||
|
(
|
||||||
|
f"files:detach:{record.resource_type}:"
|
||||||
|
f"{field_name}:{record.resource_id}"
|
||||||
|
),
|
||||||
|
"detach",
|
||||||
|
record,
|
||||||
|
f"Detach {field_name.replace('_', ' ')}",
|
||||||
|
(
|
||||||
|
"Remove the mutable subject reference while preserving the "
|
||||||
|
"governed resource and DSAR evidence."
|
||||||
|
),
|
||||||
|
executable=True,
|
||||||
|
metadata={
|
||||||
|
"field": field_name,
|
||||||
|
"subject_user_id": subject_user_id,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
action_ids = [action.action_id for action in actions]
|
||||||
|
if len(action_ids) != len(set(action_ids)):
|
||||||
|
raise ValueError("Files DSAR produced duplicate action ids.")
|
||||||
|
return tuple(actions)
|
||||||
|
|
||||||
|
def execute_erasure(
|
||||||
|
self,
|
||||||
|
session: object,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
subject: DsarSubjectRef,
|
||||||
|
actions: Sequence[DsarErasureActionRef],
|
||||||
|
request_id: str,
|
||||||
|
) -> Sequence[DsarExecutionResultRef]:
|
||||||
|
db = _session(session)
|
||||||
|
subject_user_id = _subject_user_id(subject)
|
||||||
|
if subject_user_id is None:
|
||||||
|
return tuple(
|
||||||
|
_blocked(action, "Files requires a direct membership subject reference.")
|
||||||
|
for action in actions
|
||||||
|
)
|
||||||
|
results: list[DsarExecutionResultRef] = []
|
||||||
|
for action in actions:
|
||||||
|
if (
|
||||||
|
action.provider_id != self.provider_id
|
||||||
|
or action.module_id != self.module_id
|
||||||
|
or action.metadata.get("subject_user_id") != subject_user_id
|
||||||
|
):
|
||||||
|
results.append(_blocked(action, "The Files DSAR action is stale or invalid."))
|
||||||
|
continue
|
||||||
|
if action.action_id.startswith("files:revoke:file_share:"):
|
||||||
|
results.append(
|
||||||
|
_revoke_share(
|
||||||
|
db,
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
subject_user_id=subject_user_id,
|
||||||
|
action=action,
|
||||||
|
request_id=request_id,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
elif action.action_id.startswith("files:detach:"):
|
||||||
|
results.append(
|
||||||
|
_detach_reference(
|
||||||
|
db,
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
subject_user_id=subject_user_id,
|
||||||
|
action=action,
|
||||||
|
request_id=request_id,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
results.append(_blocked(action, "Files does not execute this action kind."))
|
||||||
|
db.flush()
|
||||||
|
return tuple(results)
|
||||||
|
|
||||||
|
def _append_configuration_references(
|
||||||
|
self,
|
||||||
|
db: Session,
|
||||||
|
*,
|
||||||
|
append: object,
|
||||||
|
tenant_id: str,
|
||||||
|
subject_user_id: str,
|
||||||
|
) -> None:
|
||||||
|
configurations = (
|
||||||
|
(FileConnectorProfile, "connector_profile", ("created_by_user_id", "updated_by_user_id")),
|
||||||
|
(FileConnectorCredential, "connector_credential", ("created_by_user_id", "updated_by_user_id")),
|
||||||
|
(FileConnectorPolicy, "connector_policy", ("created_by_user_id", "updated_by_user_id")),
|
||||||
|
(FileConnectorSpace, "connector_space", ("owner_user_id", "created_by_user_id")),
|
||||||
|
)
|
||||||
|
for model, resource_type, fields in configurations:
|
||||||
|
if not _has_table(db, model):
|
||||||
|
continue
|
||||||
|
conditions = [getattr(model, field) == subject_user_id for field in fields]
|
||||||
|
query = db.query(model).filter(or_(*conditions))
|
||||||
|
if hasattr(model, "tenant_id"):
|
||||||
|
query = query.filter(model.tenant_id == tenant_id)
|
||||||
|
for row in _bounded_rows(query.order_by(model.id)):
|
||||||
|
match_fields = _matching_fields(row, subject_user_id, fields)
|
||||||
|
data: dict[str, object] = {
|
||||||
|
"match_fields": match_fields,
|
||||||
|
"label": getattr(row, "label", None),
|
||||||
|
"provider": getattr(row, "provider", None),
|
||||||
|
}
|
||||||
|
if isinstance(row, FileConnectorCredential):
|
||||||
|
data["credential_mode"] = row.credential_mode
|
||||||
|
elif isinstance(row, FileConnectorSpace):
|
||||||
|
data.update(
|
||||||
|
{
|
||||||
|
"remote_path": row.remote_path,
|
||||||
|
"sync_mode": row.sync_mode,
|
||||||
|
"read_only": row.read_only,
|
||||||
|
"deleted_at": _iso(row.deleted_at),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
append( # type: ignore[operator]
|
||||||
|
_record(
|
||||||
|
resource_type,
|
||||||
|
row.id,
|
||||||
|
"connector_configuration_evidence",
|
||||||
|
getattr(row, "label", None) or resource_type.replace("_", " "),
|
||||||
|
data,
|
||||||
|
observed_at=row.updated_at,
|
||||||
|
immutable=True,
|
||||||
|
retention_reason=(
|
||||||
|
"Connector configuration history is institutional evidence; "
|
||||||
|
"credential secrets are excluded from the DSAR export."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
def _append_integrity_references(
|
||||||
|
self,
|
||||||
|
db: Session,
|
||||||
|
*,
|
||||||
|
append: object,
|
||||||
|
tenant_id: str,
|
||||||
|
subject_user_id: str,
|
||||||
|
) -> None:
|
||||||
|
if _has_table(db, FileIntegrityScan):
|
||||||
|
for scan in _bounded_rows(
|
||||||
|
db.query(FileIntegrityScan)
|
||||||
|
.filter(
|
||||||
|
FileIntegrityScan.tenant_id == tenant_id,
|
||||||
|
FileIntegrityScan.created_by_user_id == subject_user_id,
|
||||||
|
)
|
||||||
|
.order_by(FileIntegrityScan.id)
|
||||||
|
):
|
||||||
|
append( # type: ignore[operator]
|
||||||
|
_record(
|
||||||
|
"file_integrity_scan",
|
||||||
|
scan.id,
|
||||||
|
"storage_integrity_evidence",
|
||||||
|
f"Integrity scan {scan.id}",
|
||||||
|
{
|
||||||
|
"match_fields": ["created_by_user_id"],
|
||||||
|
"storage_backend": scan.storage_backend,
|
||||||
|
"status": scan.status,
|
||||||
|
"started_at": _iso(scan.started_at),
|
||||||
|
"completed_at": _iso(scan.completed_at),
|
||||||
|
},
|
||||||
|
observed_at=scan.updated_at,
|
||||||
|
immutable=True,
|
||||||
|
retention_reason="Storage integrity scans are operator evidence.",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if _has_table(db, FileIntegrityFinding):
|
||||||
|
for finding in _bounded_rows(
|
||||||
|
db.query(FileIntegrityFinding)
|
||||||
|
.filter(
|
||||||
|
FileIntegrityFinding.tenant_id == tenant_id,
|
||||||
|
FileIntegrityFinding.resolved_by_user_id == subject_user_id,
|
||||||
|
)
|
||||||
|
.order_by(FileIntegrityFinding.id)
|
||||||
|
):
|
||||||
|
append( # type: ignore[operator]
|
||||||
|
_record(
|
||||||
|
"file_integrity_finding",
|
||||||
|
finding.id,
|
||||||
|
"storage_integrity_evidence",
|
||||||
|
f"Integrity finding {finding.kind}",
|
||||||
|
{
|
||||||
|
"match_fields": ["resolved_by_user_id"],
|
||||||
|
"kind": finding.kind,
|
||||||
|
"state": finding.state,
|
||||||
|
"resolved_at": _iso(finding.resolved_at),
|
||||||
|
},
|
||||||
|
observed_at=finding.updated_at,
|
||||||
|
immutable=True,
|
||||||
|
retention_reason="Integrity resolution is operator evidence.",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _asset_retention_reason(
|
||||||
|
session: Session,
|
||||||
|
asset: FileAsset,
|
||||||
|
*,
|
||||||
|
form_evidence_available: bool,
|
||||||
|
campaign_evidence_available: bool,
|
||||||
|
) -> str | None:
|
||||||
|
reasons: list[str] = []
|
||||||
|
if asset.legal_hold:
|
||||||
|
reasons.append("The file is under legal hold.")
|
||||||
|
retained_until = _aware(asset.retained_until)
|
||||||
|
if retained_until is not None and retained_until > datetime.now(timezone.utc):
|
||||||
|
reasons.append(f"The file is retained until {retained_until.isoformat()}.")
|
||||||
|
if form_evidence_available:
|
||||||
|
has_form_evidence = (
|
||||||
|
session.query(FileFormEvidenceGrant.id)
|
||||||
|
.filter(
|
||||||
|
FileFormEvidenceGrant.tenant_id == asset.tenant_id,
|
||||||
|
FileFormEvidenceGrant.file_asset_id == asset.id,
|
||||||
|
)
|
||||||
|
.first()
|
||||||
|
is not None
|
||||||
|
)
|
||||||
|
if has_form_evidence:
|
||||||
|
reasons.append("The file is referenced by submitted Form evidence.")
|
||||||
|
if campaign_evidence_available:
|
||||||
|
has_campaign_evidence = (
|
||||||
|
session.query(CampaignAttachmentUse.id)
|
||||||
|
.filter(
|
||||||
|
CampaignAttachmentUse.tenant_id == asset.tenant_id,
|
||||||
|
CampaignAttachmentUse.file_asset_id == asset.id,
|
||||||
|
)
|
||||||
|
.first()
|
||||||
|
is not None
|
||||||
|
)
|
||||||
|
if has_campaign_evidence:
|
||||||
|
reasons.append("The file is referenced by Campaign delivery evidence.")
|
||||||
|
return " ".join(reasons) or None
|
||||||
|
|
||||||
|
|
||||||
|
_DETACHABLE_MODELS: dict[str, tuple[type[object], frozenset[str]]] = {
|
||||||
|
"file_asset": (FileAsset, frozenset({"created_by_user_id"})),
|
||||||
|
"file_version": (FileVersion, frozenset({"created_by_user_id"})),
|
||||||
|
"file_folder": (FileFolder, frozenset({"created_by_user_id"})),
|
||||||
|
"file_share": (
|
||||||
|
FileShare,
|
||||||
|
frozenset({"created_by_user_id", "revoked_by_user_id"}),
|
||||||
|
),
|
||||||
|
"connector_profile": (
|
||||||
|
FileConnectorProfile,
|
||||||
|
frozenset({"created_by_user_id", "updated_by_user_id"}),
|
||||||
|
),
|
||||||
|
"connector_credential": (
|
||||||
|
FileConnectorCredential,
|
||||||
|
frozenset({"created_by_user_id", "updated_by_user_id"}),
|
||||||
|
),
|
||||||
|
"connector_policy": (
|
||||||
|
FileConnectorPolicy,
|
||||||
|
frozenset({"created_by_user_id", "updated_by_user_id"}),
|
||||||
|
),
|
||||||
|
"connector_space": (FileConnectorSpace, frozenset({"created_by_user_id"})),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _detachable_fields(resource_type: str) -> frozenset[str]:
|
||||||
|
entry = _DETACHABLE_MODELS.get(resource_type)
|
||||||
|
return entry[1] if entry is not None else frozenset()
|
||||||
|
|
||||||
|
|
||||||
|
def _revoke_share(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
subject_user_id: str,
|
||||||
|
action: DsarErasureActionRef,
|
||||||
|
request_id: str,
|
||||||
|
) -> DsarExecutionResultRef:
|
||||||
|
row = (
|
||||||
|
session.query(FileShare)
|
||||||
|
.filter(FileShare.id == action.resource_id)
|
||||||
|
.with_for_update()
|
||||||
|
.one_or_none()
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
row is None
|
||||||
|
or row.tenant_id != tenant_id
|
||||||
|
or row.target_type != "user"
|
||||||
|
or row.target_id != subject_user_id
|
||||||
|
):
|
||||||
|
return _blocked(action, "The subject-targeted file share is no longer available.")
|
||||||
|
if row.revoked_at is not None:
|
||||||
|
return _result(
|
||||||
|
action,
|
||||||
|
"unchanged",
|
||||||
|
"The file share was already revoked.",
|
||||||
|
{"request_id": request_id, "revoked_at": _iso(row.revoked_at)},
|
||||||
|
)
|
||||||
|
row.revoked_at = datetime.now(timezone.utc)
|
||||||
|
row.revoked_by_user_id = None
|
||||||
|
return _result(
|
||||||
|
action,
|
||||||
|
"executed",
|
||||||
|
"The subject-targeted file share was revoked.",
|
||||||
|
{"request_id": request_id, "revoked_at": _iso(row.revoked_at)},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _detach_reference(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
subject_user_id: str,
|
||||||
|
action: DsarErasureActionRef,
|
||||||
|
request_id: str,
|
||||||
|
) -> DsarExecutionResultRef:
|
||||||
|
entry = _DETACHABLE_MODELS.get(action.resource_type)
|
||||||
|
field_name = str(action.metadata.get("field") or "")
|
||||||
|
if entry is None or field_name not in entry[1]:
|
||||||
|
return _blocked(action, "The requested Files subject reference is not detachable.")
|
||||||
|
model = entry[0]
|
||||||
|
row = (
|
||||||
|
session.query(model)
|
||||||
|
.filter(getattr(model, "id") == action.resource_id)
|
||||||
|
.with_for_update()
|
||||||
|
.one_or_none()
|
||||||
|
)
|
||||||
|
if row is None or getattr(row, "tenant_id", None) != tenant_id:
|
||||||
|
return _blocked(action, "The Files resource is no longer available.")
|
||||||
|
current = getattr(row, field_name)
|
||||||
|
if current is None:
|
||||||
|
return _result(
|
||||||
|
action,
|
||||||
|
"unchanged",
|
||||||
|
"The subject reference was already detached.",
|
||||||
|
{"request_id": request_id, "field": field_name},
|
||||||
|
)
|
||||||
|
if current != subject_user_id:
|
||||||
|
return _blocked(action, "The Files subject reference changed after planning.")
|
||||||
|
setattr(row, field_name, None)
|
||||||
|
return _result(
|
||||||
|
action,
|
||||||
|
"executed",
|
||||||
|
"The mutable subject reference was detached.",
|
||||||
|
{"request_id": request_id, "field": field_name},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _subject_user_id(subject: DsarSubjectRef) -> str | None:
|
||||||
|
candidates: list[str] = []
|
||||||
|
if subject.membership_id:
|
||||||
|
candidates.append(subject.membership_id)
|
||||||
|
for key, value in subject.external_references.items():
|
||||||
|
if key in {
|
||||||
|
"files.user",
|
||||||
|
"files.membership",
|
||||||
|
"access.membership",
|
||||||
|
"membership_id",
|
||||||
|
}:
|
||||||
|
candidates.append(value)
|
||||||
|
normalized = {value.strip() for value in candidates if value.strip()}
|
||||||
|
if len(normalized) != 1:
|
||||||
|
return None
|
||||||
|
return normalized.pop()
|
||||||
|
|
||||||
|
|
||||||
|
def _matching_fields(
|
||||||
|
row: object,
|
||||||
|
subject_user_id: str,
|
||||||
|
fields: Sequence[str],
|
||||||
|
) -> list[str]:
|
||||||
|
return [field for field in fields if getattr(row, field) == subject_user_id]
|
||||||
|
|
||||||
|
|
||||||
|
def _record(
|
||||||
|
resource_type: str,
|
||||||
|
resource_id: str,
|
||||||
|
category: str,
|
||||||
|
title: str,
|
||||||
|
data: dict[str, object],
|
||||||
|
*,
|
||||||
|
observed_at: datetime | None = None,
|
||||||
|
immutable: bool = False,
|
||||||
|
retention_reason: str | None = None,
|
||||||
|
source_path: str | None = None,
|
||||||
|
) -> DsarRecordRef:
|
||||||
|
return DsarRecordRef(
|
||||||
|
provider_id="files",
|
||||||
|
module_id="files",
|
||||||
|
resource_type=resource_type,
|
||||||
|
resource_id=resource_id,
|
||||||
|
category=category,
|
||||||
|
title=title,
|
||||||
|
data=data,
|
||||||
|
observed_at=observed_at,
|
||||||
|
immutable_evidence=immutable,
|
||||||
|
retention_reason=retention_reason,
|
||||||
|
source_path=source_path,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _action(
|
||||||
|
action_id: str,
|
||||||
|
kind: str,
|
||||||
|
record: DsarRecordRef,
|
||||||
|
title: str,
|
||||||
|
rationale: str,
|
||||||
|
*,
|
||||||
|
executable: bool,
|
||||||
|
metadata: dict[str, object] | None = None,
|
||||||
|
) -> DsarErasureActionRef:
|
||||||
|
return DsarErasureActionRef(
|
||||||
|
action_id=action_id,
|
||||||
|
provider_id="files",
|
||||||
|
module_id="files",
|
||||||
|
kind=kind, # type: ignore[arg-type]
|
||||||
|
resource_type=record.resource_type,
|
||||||
|
resource_id=record.resource_id,
|
||||||
|
title=title,
|
||||||
|
rationale=rationale,
|
||||||
|
executable=executable,
|
||||||
|
metadata=metadata or {},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _result(
|
||||||
|
action: DsarErasureActionRef,
|
||||||
|
status: str,
|
||||||
|
summary: str,
|
||||||
|
evidence: dict[str, object] | None = None,
|
||||||
|
) -> DsarExecutionResultRef:
|
||||||
|
return DsarExecutionResultRef(
|
||||||
|
action_id=action.action_id,
|
||||||
|
status=status, # type: ignore[arg-type]
|
||||||
|
summary=summary,
|
||||||
|
evidence=evidence or {},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _blocked(action: DsarErasureActionRef, summary: str) -> DsarExecutionResultRef:
|
||||||
|
return _result(action, "blocked", summary)
|
||||||
|
|
||||||
|
|
||||||
|
def _session(value: object) -> Session:
|
||||||
|
if not isinstance(value, Session):
|
||||||
|
raise TypeError("Files DSAR provider requires a SQLAlchemy session.")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def _bounded_rows(query: object) -> list[object]:
|
||||||
|
rows = query.limit(_MAX_RECORDS + 1).all() # type: ignore[attr-defined]
|
||||||
|
if len(rows) > _MAX_RECORDS:
|
||||||
|
raise ValueError("Files DSAR match limit exceeded; narrow the subject selectors.")
|
||||||
|
return rows
|
||||||
|
|
||||||
|
|
||||||
|
def _has_table(session: Session, model: type[object]) -> bool:
|
||||||
|
return inspect(session.connection()).has_table(model.__tablename__)
|
||||||
|
|
||||||
|
|
||||||
|
def _aware(value: datetime | None) -> datetime | None:
|
||||||
|
if value is None or value.tzinfo is not None:
|
||||||
|
return value
|
||||||
|
return value.replace(tzinfo=timezone.utc)
|
||||||
|
|
||||||
|
|
||||||
|
def _iso(value: datetime | None) -> str | None:
|
||||||
|
aware = _aware(value)
|
||||||
|
return aware.isoformat() if aware else None
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["FILES_DSAR_CAPABILITY", "FilesDsarProvider"]
|
||||||
@@ -0,0 +1,449 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from collections.abc import Mapping, Sequence
|
||||||
|
from datetime import UTC, datetime, timedelta
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import secrets
|
||||||
|
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from govoplan_core.core.access import CAPABILITY_ACCESS_DIRECTORY, AccessDirectory
|
||||||
|
from govoplan_core.core.form_evidence import (
|
||||||
|
FormEvidenceContractError,
|
||||||
|
FormEvidenceGrant,
|
||||||
|
FormEvidenceGrantRequest,
|
||||||
|
FormEvidenceInspection,
|
||||||
|
FormEvidenceInspectionRequest,
|
||||||
|
FormEvidenceState,
|
||||||
|
)
|
||||||
|
from govoplan_core.core.institutional import EvidenceReference
|
||||||
|
from govoplan_core.core.modules import ModuleContext
|
||||||
|
from govoplan_files.backend.db.models import (
|
||||||
|
FileAsset,
|
||||||
|
FileBlob,
|
||||||
|
FileFormEvidenceGrant,
|
||||||
|
FileVersion,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.runtime import configure_runtime
|
||||||
|
|
||||||
|
|
||||||
|
PROVIDER_ID = "files"
|
||||||
|
CAPABILITY_FORM_EVIDENCE_FILES = "forms_runtime.evidence.files"
|
||||||
|
MAX_GRANT_TTL = timedelta(minutes=15)
|
||||||
|
|
||||||
|
|
||||||
|
class FilesFormEvidenceProvider:
|
||||||
|
provider_id = PROVIDER_ID
|
||||||
|
|
||||||
|
def __init__(self, registry: object | None, settings: object) -> None:
|
||||||
|
self._registry = registry
|
||||||
|
self._settings = settings
|
||||||
|
|
||||||
|
def supported_kinds(self) -> Sequence[str]:
|
||||||
|
return ("document",)
|
||||||
|
|
||||||
|
def create_upload_grant(
|
||||||
|
self,
|
||||||
|
session: object,
|
||||||
|
principal: object,
|
||||||
|
*,
|
||||||
|
request: FormEvidenceGrantRequest,
|
||||||
|
) -> FormEvidenceGrant:
|
||||||
|
db = _session(session)
|
||||||
|
_assert_tenant(principal, request.tenant_id)
|
||||||
|
if request.evidence_kind != "document":
|
||||||
|
raise FormEvidenceContractError(
|
||||||
|
"Files can accept only document evidence for Forms Runtime."
|
||||||
|
)
|
||||||
|
custodian_user_id = _custodian_user_id(request.custodian_ref)
|
||||||
|
self._assert_active_custodian(
|
||||||
|
tenant_id=request.tenant_id,
|
||||||
|
user_id=custodian_user_id,
|
||||||
|
)
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
if request.expires_at <= now:
|
||||||
|
raise FormEvidenceContractError(
|
||||||
|
"Form evidence upload grant expiry must be in the future."
|
||||||
|
)
|
||||||
|
allowed_content_types = _content_types(request.allowed_content_types)
|
||||||
|
configured_max = int(getattr(self._settings, "file_upload_max_bytes"))
|
||||||
|
max_size_bytes = min(request.max_size_bytes or configured_max, configured_max)
|
||||||
|
request_sha256 = _request_sha256(
|
||||||
|
request,
|
||||||
|
custodian_user_id=custodian_user_id,
|
||||||
|
max_size_bytes=max_size_bytes,
|
||||||
|
allowed_content_types=allowed_content_types,
|
||||||
|
)
|
||||||
|
existing = (
|
||||||
|
db.query(FileFormEvidenceGrant)
|
||||||
|
.filter(
|
||||||
|
FileFormEvidenceGrant.tenant_id == request.tenant_id,
|
||||||
|
FileFormEvidenceGrant.idempotency_key == request.idempotency_key,
|
||||||
|
)
|
||||||
|
.one_or_none()
|
||||||
|
)
|
||||||
|
if existing is not None:
|
||||||
|
if not secrets.compare_digest(existing.request_sha256, request_sha256):
|
||||||
|
raise FormEvidenceContractError(
|
||||||
|
"Form evidence grant idempotency conflict."
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
existing.status in {"expired", "revoked"}
|
||||||
|
or _aware(existing.expires_at) <= now
|
||||||
|
):
|
||||||
|
raise FormEvidenceContractError(
|
||||||
|
"The existing Form evidence upload grant is no longer usable; "
|
||||||
|
"request a new grant with a new idempotency key."
|
||||||
|
)
|
||||||
|
return _grant_response(existing, upload_token=None, replayed=True)
|
||||||
|
|
||||||
|
metadata = _bounded_metadata(request.metadata)
|
||||||
|
remaining_attachments = metadata.get("remaining_attachments")
|
||||||
|
if isinstance(remaining_attachments, int):
|
||||||
|
existing_attachment_ids = set(metadata.get("existing_attachment_ids", ()))
|
||||||
|
active_grants = (
|
||||||
|
db.query(FileFormEvidenceGrant)
|
||||||
|
.filter(
|
||||||
|
FileFormEvidenceGrant.tenant_id == request.tenant_id,
|
||||||
|
FileFormEvidenceGrant.form_instance_id == request.instance_id,
|
||||||
|
FileFormEvidenceGrant.form_definition_id
|
||||||
|
== request.definition_ref.object_id,
|
||||||
|
FileFormEvidenceGrant.form_definition_revision
|
||||||
|
== str(request.definition_ref.version),
|
||||||
|
FileFormEvidenceGrant.status.in_(("issued", "uploaded")),
|
||||||
|
FileFormEvidenceGrant.expires_at > now,
|
||||||
|
)
|
||||||
|
.with_for_update()
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
outstanding = sum(
|
||||||
|
1
|
||||||
|
for item in active_grants
|
||||||
|
if not item.file_asset_id
|
||||||
|
or item.file_asset_id not in existing_attachment_ids
|
||||||
|
)
|
||||||
|
if outstanding >= remaining_attachments:
|
||||||
|
raise FormEvidenceContractError(
|
||||||
|
"This Form already has the maximum number of active attachment uploads."
|
||||||
|
)
|
||||||
|
|
||||||
|
upload_token = secrets.token_urlsafe(32)
|
||||||
|
grant = FileFormEvidenceGrant(
|
||||||
|
tenant_id=request.tenant_id,
|
||||||
|
form_instance_id=request.instance_id,
|
||||||
|
form_definition_id=request.definition_ref.object_id,
|
||||||
|
form_definition_revision=str(request.definition_ref.version),
|
||||||
|
token_sha256=_token_sha256(upload_token),
|
||||||
|
idempotency_key=request.idempotency_key,
|
||||||
|
request_sha256=request_sha256,
|
||||||
|
custodian_user_id=custodian_user_id,
|
||||||
|
evidence_kind=request.evidence_kind,
|
||||||
|
purpose=request.purpose,
|
||||||
|
status="issued",
|
||||||
|
expires_at=min(request.expires_at, now + MAX_GRANT_TTL),
|
||||||
|
max_size_bytes=max_size_bytes,
|
||||||
|
allowed_content_types=list(allowed_content_types),
|
||||||
|
metadata_=metadata,
|
||||||
|
)
|
||||||
|
db.add(grant)
|
||||||
|
db.flush()
|
||||||
|
return _grant_response(grant, upload_token=upload_token, replayed=False)
|
||||||
|
|
||||||
|
def inspect_evidence(
|
||||||
|
self,
|
||||||
|
session: object,
|
||||||
|
principal: object,
|
||||||
|
*,
|
||||||
|
request: FormEvidenceInspectionRequest,
|
||||||
|
) -> FormEvidenceInspection:
|
||||||
|
db = _session(session)
|
||||||
|
_assert_tenant(principal, request.tenant_id)
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
if request.evidence.owner_module != PROVIDER_ID:
|
||||||
|
return _inspection(
|
||||||
|
request.evidence,
|
||||||
|
state="rejected",
|
||||||
|
observed_at=now,
|
||||||
|
reason="The evidence owner does not match the Files provider.",
|
||||||
|
)
|
||||||
|
if request.evidence.kind != "document" or not request.evidence.version:
|
||||||
|
return _inspection(
|
||||||
|
request.evidence,
|
||||||
|
state="rejected",
|
||||||
|
observed_at=now,
|
||||||
|
reason="Files evidence requires an exact document version.",
|
||||||
|
)
|
||||||
|
grant = (
|
||||||
|
db.query(FileFormEvidenceGrant)
|
||||||
|
.filter(
|
||||||
|
FileFormEvidenceGrant.tenant_id == request.tenant_id,
|
||||||
|
FileFormEvidenceGrant.form_instance_id == request.instance_id,
|
||||||
|
FileFormEvidenceGrant.form_definition_id
|
||||||
|
== request.definition_ref.object_id,
|
||||||
|
FileFormEvidenceGrant.form_definition_revision
|
||||||
|
== str(request.definition_ref.version),
|
||||||
|
FileFormEvidenceGrant.file_asset_id == request.evidence.evidence_id,
|
||||||
|
FileFormEvidenceGrant.file_version_id == request.evidence.version,
|
||||||
|
)
|
||||||
|
.one_or_none()
|
||||||
|
)
|
||||||
|
if grant is None:
|
||||||
|
return _inspection(
|
||||||
|
request.evidence,
|
||||||
|
state="rejected",
|
||||||
|
observed_at=now,
|
||||||
|
reason="The document was not captured for this exact Form submission.",
|
||||||
|
)
|
||||||
|
if grant.status == "revoked":
|
||||||
|
return _inspection(
|
||||||
|
request.evidence,
|
||||||
|
state="revoked",
|
||||||
|
observed_at=now,
|
||||||
|
reason="The Form evidence grant was revoked.",
|
||||||
|
)
|
||||||
|
if grant.status != "uploaded":
|
||||||
|
return _inspection(
|
||||||
|
request.evidence,
|
||||||
|
state="pending",
|
||||||
|
observed_at=now,
|
||||||
|
retryable=True,
|
||||||
|
reason="The Form evidence upload has not completed.",
|
||||||
|
)
|
||||||
|
asset = db.get(FileAsset, grant.file_asset_id)
|
||||||
|
version = db.get(FileVersion, grant.file_version_id)
|
||||||
|
blob = db.get(FileBlob, version.blob_id) if version is not None else None
|
||||||
|
if asset is None or version is None or blob is None:
|
||||||
|
return _inspection(
|
||||||
|
request.evidence,
|
||||||
|
state="unavailable",
|
||||||
|
observed_at=now,
|
||||||
|
retryable=True,
|
||||||
|
reason="The managed document cannot currently be reconstructed.",
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
asset.tenant_id != request.tenant_id
|
||||||
|
or version.tenant_id != request.tenant_id
|
||||||
|
or blob.tenant_id != request.tenant_id
|
||||||
|
or version.file_asset_id != asset.id
|
||||||
|
):
|
||||||
|
return _inspection(
|
||||||
|
request.evidence,
|
||||||
|
state="rejected",
|
||||||
|
observed_at=now,
|
||||||
|
reason="The managed document crosses an evidence ownership boundary.",
|
||||||
|
)
|
||||||
|
if asset.deleted_at is not None:
|
||||||
|
return _inspection(
|
||||||
|
request.evidence,
|
||||||
|
state="revoked",
|
||||||
|
observed_at=now,
|
||||||
|
reason="The managed document is no longer active.",
|
||||||
|
)
|
||||||
|
if blob.quarantined_at is not None or blob.integrity_status == "quarantined":
|
||||||
|
return _inspection(
|
||||||
|
request.evidence,
|
||||||
|
state="rejected",
|
||||||
|
observed_at=now,
|
||||||
|
reason="The managed document failed its integrity gate.",
|
||||||
|
)
|
||||||
|
if blob.integrity_status != "verified":
|
||||||
|
return _inspection(
|
||||||
|
request.evidence,
|
||||||
|
state="pending",
|
||||||
|
observed_at=now,
|
||||||
|
retryable=True,
|
||||||
|
reason="The managed document is awaiting integrity verification.",
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
not request.evidence.checksum
|
||||||
|
or not secrets.compare_digest(
|
||||||
|
request.evidence.checksum,
|
||||||
|
version.checksum_sha256,
|
||||||
|
)
|
||||||
|
or not secrets.compare_digest(
|
||||||
|
version.checksum_sha256,
|
||||||
|
blob.checksum_sha256,
|
||||||
|
)
|
||||||
|
):
|
||||||
|
return _inspection(
|
||||||
|
request.evidence,
|
||||||
|
state="rejected",
|
||||||
|
observed_at=now,
|
||||||
|
reason="The managed document checksum does not match the evidence.",
|
||||||
|
)
|
||||||
|
return _inspection(
|
||||||
|
request.evidence,
|
||||||
|
state="accepted",
|
||||||
|
observed_at=now,
|
||||||
|
metadata={
|
||||||
|
"content_type": version.content_type,
|
||||||
|
"size_bytes": version.size_bytes,
|
||||||
|
"integrity_status": blob.integrity_status,
|
||||||
|
"grant_id": grant.id,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
def _assert_active_custodian(self, *, tenant_id: str, user_id: str) -> None:
|
||||||
|
registry = self._registry
|
||||||
|
if registry is None or not hasattr(registry, "has_capability"):
|
||||||
|
raise FormEvidenceContractError(
|
||||||
|
"The Access directory is unavailable for Form evidence custody."
|
||||||
|
)
|
||||||
|
if not registry.has_capability(CAPABILITY_ACCESS_DIRECTORY):
|
||||||
|
raise FormEvidenceContractError(
|
||||||
|
"The Access directory is unavailable for Form evidence custody."
|
||||||
|
)
|
||||||
|
directory = registry.require_capability(CAPABILITY_ACCESS_DIRECTORY)
|
||||||
|
if not isinstance(directory, AccessDirectory):
|
||||||
|
raise FormEvidenceContractError(
|
||||||
|
"The Access directory capability is invalid."
|
||||||
|
)
|
||||||
|
user = directory.get_user(user_id)
|
||||||
|
if user is None or user.tenant_id != tenant_id or user.status != "active":
|
||||||
|
raise FormEvidenceContractError(
|
||||||
|
"Form evidence requires an active same-tenant custodian."
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def create_files_form_evidence_provider(
|
||||||
|
context: ModuleContext,
|
||||||
|
) -> FilesFormEvidenceProvider:
|
||||||
|
configure_runtime(registry=context.registry, settings=context.settings)
|
||||||
|
return FilesFormEvidenceProvider(context.registry, context.settings)
|
||||||
|
|
||||||
|
|
||||||
|
def _grant_response(
|
||||||
|
grant: FileFormEvidenceGrant,
|
||||||
|
*,
|
||||||
|
upload_token: str | None,
|
||||||
|
replayed: bool,
|
||||||
|
) -> FormEvidenceGrant:
|
||||||
|
return FormEvidenceGrant(
|
||||||
|
provider_id=PROVIDER_ID,
|
||||||
|
grant_id=grant.id,
|
||||||
|
upload_token=upload_token,
|
||||||
|
upload_url="/api/v1/files/form-evidence/upload",
|
||||||
|
expires_at=_aware(grant.expires_at),
|
||||||
|
max_size_bytes=grant.max_size_bytes,
|
||||||
|
allowed_content_types=tuple(grant.allowed_content_types),
|
||||||
|
replayed=replayed,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _inspection(
|
||||||
|
reference: EvidenceReference,
|
||||||
|
*,
|
||||||
|
state: FormEvidenceState,
|
||||||
|
observed_at: datetime,
|
||||||
|
retryable: bool = False,
|
||||||
|
reason: str | None = None,
|
||||||
|
metadata: Mapping[str, object] | None = None,
|
||||||
|
) -> FormEvidenceInspection:
|
||||||
|
return FormEvidenceInspection(
|
||||||
|
provider_id=PROVIDER_ID,
|
||||||
|
reference=reference,
|
||||||
|
state=state,
|
||||||
|
observed_at=observed_at,
|
||||||
|
retryable=retryable,
|
||||||
|
reason=reason,
|
||||||
|
metadata=dict(metadata or {}),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _session(value: object) -> Session:
|
||||||
|
if not isinstance(value, Session):
|
||||||
|
raise TypeError("Files Form evidence requires a SQLAlchemy session.")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def _assert_tenant(principal: object, tenant_id: str) -> None:
|
||||||
|
principal_tenant = str(getattr(principal, "tenant_id", "") or "").strip()
|
||||||
|
if not principal_tenant or principal_tenant != tenant_id:
|
||||||
|
raise PermissionError("Form evidence cannot cross tenants.")
|
||||||
|
|
||||||
|
|
||||||
|
def _custodian_user_id(value: str | None) -> str:
|
||||||
|
clean = str(value or "").strip()
|
||||||
|
if not clean.startswith("user:") or len(clean) <= len("user:"):
|
||||||
|
raise FormEvidenceContractError(
|
||||||
|
"Files Form evidence requires a user custodian."
|
||||||
|
)
|
||||||
|
return clean.removeprefix("user:")
|
||||||
|
|
||||||
|
|
||||||
|
def _content_types(values: Sequence[str]) -> tuple[str, ...]:
|
||||||
|
cleaned = tuple(
|
||||||
|
dict.fromkeys(
|
||||||
|
value.split(";", 1)[0].strip().casefold()
|
||||||
|
for value in values
|
||||||
|
if value.strip()
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if len(cleaned) > 50 or any(len(value) > 255 for value in cleaned):
|
||||||
|
raise FormEvidenceContractError(
|
||||||
|
"Form evidence content-type restrictions are too large."
|
||||||
|
)
|
||||||
|
return cleaned
|
||||||
|
|
||||||
|
|
||||||
|
def _bounded_metadata(value: Mapping[str, object]) -> dict[str, object]:
|
||||||
|
remaining = value.get("remaining_attachments")
|
||||||
|
result: dict[str, object] = {}
|
||||||
|
if isinstance(remaining, int) and remaining >= 0:
|
||||||
|
result["remaining_attachments"] = remaining
|
||||||
|
raw_ids = value.get("existing_attachment_ids")
|
||||||
|
if isinstance(raw_ids, (list, tuple)):
|
||||||
|
clean_ids = tuple(
|
||||||
|
dict.fromkeys(
|
||||||
|
item.strip()
|
||||||
|
for item in raw_ids
|
||||||
|
if isinstance(item, str) and item.strip() and len(item.strip()) <= 255
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if len(clean_ids) <= 1000:
|
||||||
|
result["existing_attachment_ids"] = clean_ids
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def _request_sha256(
|
||||||
|
request: FormEvidenceGrantRequest,
|
||||||
|
*,
|
||||||
|
custodian_user_id: str,
|
||||||
|
max_size_bytes: int,
|
||||||
|
allowed_content_types: Sequence[str],
|
||||||
|
) -> str:
|
||||||
|
return hashlib.sha256(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"tenant_id": request.tenant_id,
|
||||||
|
"instance_id": request.instance_id,
|
||||||
|
"definition_ref": request.definition_ref.to_dict(),
|
||||||
|
"evidence_kind": request.evidence_kind,
|
||||||
|
"purpose": request.purpose,
|
||||||
|
"expires_at": request.expires_at.isoformat(),
|
||||||
|
"custodian_user_id": custodian_user_id,
|
||||||
|
"max_size_bytes": max_size_bytes,
|
||||||
|
"allowed_content_types": list(allowed_content_types),
|
||||||
|
"metadata": _bounded_metadata(request.metadata),
|
||||||
|
},
|
||||||
|
sort_keys=True,
|
||||||
|
separators=(",", ":"),
|
||||||
|
).encode("utf-8")
|
||||||
|
).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def _token_sha256(token: str) -> str:
|
||||||
|
return hashlib.sha256(token.encode("utf-8")).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def _aware(value: datetime) -> datetime:
|
||||||
|
return value if value.tzinfo is not None else value.replace(tzinfo=UTC)
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"CAPABILITY_FORM_EVIDENCE_FILES",
|
||||||
|
"FilesFormEvidenceProvider",
|
||||||
|
"PROVIDER_ID",
|
||||||
|
"create_files_form_evidence_provider",
|
||||||
|
]
|
||||||
File diff suppressed because it is too large
Load Diff
+147
@@ -0,0 +1,147 @@
|
|||||||
|
"""add purpose-bound Form evidence upload grants
|
||||||
|
|
||||||
|
Revision ID: a2b3c4d5e6f8
|
||||||
|
Revises: f1a2b3c4d5e7
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
|
||||||
|
revision = "a2b3c4d5e6f8"
|
||||||
|
down_revision = "f1a2b3c4d5e7"
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.create_table(
|
||||||
|
"file_form_evidence_grants",
|
||||||
|
sa.Column("id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("form_instance_id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("form_definition_id", sa.String(length=255), nullable=False),
|
||||||
|
sa.Column("form_definition_revision", sa.String(length=255), nullable=False),
|
||||||
|
sa.Column("token_sha256", sa.String(length=64), nullable=False),
|
||||||
|
sa.Column("idempotency_key", sa.String(length=255), nullable=False),
|
||||||
|
sa.Column("request_sha256", sa.String(length=64), nullable=False),
|
||||||
|
sa.Column("custodian_user_id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("evidence_kind", sa.String(length=30), nullable=False),
|
||||||
|
sa.Column("purpose", sa.String(length=500), nullable=False),
|
||||||
|
sa.Column("status", sa.String(length=30), nullable=False),
|
||||||
|
sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column("max_size_bytes", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("allowed_content_types", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("file_asset_id", sa.String(length=36), nullable=True),
|
||||||
|
sa.Column("file_version_id", sa.String(length=36), nullable=True),
|
||||||
|
sa.Column("uploaded_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column("revoked_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column("metadata", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["custodian_user_id"],
|
||||||
|
["access_users.id"],
|
||||||
|
ondelete="RESTRICT",
|
||||||
|
),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["file_asset_id"],
|
||||||
|
["file_assets.id"],
|
||||||
|
ondelete="RESTRICT",
|
||||||
|
),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["file_version_id"],
|
||||||
|
["file_versions.id"],
|
||||||
|
ondelete="RESTRICT",
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint("id"),
|
||||||
|
sa.UniqueConstraint(
|
||||||
|
"tenant_id",
|
||||||
|
"idempotency_key",
|
||||||
|
name="uq_file_form_evidence_grants_idempotency",
|
||||||
|
),
|
||||||
|
sa.UniqueConstraint("token_sha256"),
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_file_form_evidence_grants_tenant_id",
|
||||||
|
"file_form_evidence_grants",
|
||||||
|
["tenant_id"],
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_file_form_evidence_grants_form_instance_id",
|
||||||
|
"file_form_evidence_grants",
|
||||||
|
["form_instance_id"],
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_file_form_evidence_grants_custodian_user_id",
|
||||||
|
"file_form_evidence_grants",
|
||||||
|
["custodian_user_id"],
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_file_form_evidence_grants_status",
|
||||||
|
"file_form_evidence_grants",
|
||||||
|
["status"],
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_file_form_evidence_grants_expires_at",
|
||||||
|
"file_form_evidence_grants",
|
||||||
|
["expires_at"],
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_file_form_evidence_grants_file_asset_id",
|
||||||
|
"file_form_evidence_grants",
|
||||||
|
["file_asset_id"],
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_file_form_evidence_grants_file_version_id",
|
||||||
|
"file_form_evidence_grants",
|
||||||
|
["file_version_id"],
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_file_form_evidence_grants_form",
|
||||||
|
"file_form_evidence_grants",
|
||||||
|
[
|
||||||
|
"tenant_id",
|
||||||
|
"form_instance_id",
|
||||||
|
"form_definition_id",
|
||||||
|
"form_definition_revision",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_index(
|
||||||
|
"ix_file_form_evidence_grants_form",
|
||||||
|
table_name="file_form_evidence_grants",
|
||||||
|
)
|
||||||
|
op.drop_index(
|
||||||
|
"ix_file_form_evidence_grants_file_version_id",
|
||||||
|
table_name="file_form_evidence_grants",
|
||||||
|
)
|
||||||
|
op.drop_index(
|
||||||
|
"ix_file_form_evidence_grants_file_asset_id",
|
||||||
|
table_name="file_form_evidence_grants",
|
||||||
|
)
|
||||||
|
op.drop_index(
|
||||||
|
"ix_file_form_evidence_grants_expires_at",
|
||||||
|
table_name="file_form_evidence_grants",
|
||||||
|
)
|
||||||
|
op.drop_index(
|
||||||
|
"ix_file_form_evidence_grants_status",
|
||||||
|
table_name="file_form_evidence_grants",
|
||||||
|
)
|
||||||
|
op.drop_index(
|
||||||
|
"ix_file_form_evidence_grants_custodian_user_id",
|
||||||
|
table_name="file_form_evidence_grants",
|
||||||
|
)
|
||||||
|
op.drop_index(
|
||||||
|
"ix_file_form_evidence_grants_form_instance_id",
|
||||||
|
table_name="file_form_evidence_grants",
|
||||||
|
)
|
||||||
|
op.drop_index(
|
||||||
|
"ix_file_form_evidence_grants_tenant_id",
|
||||||
|
table_name="file_form_evidence_grants",
|
||||||
|
)
|
||||||
|
op.drop_table("file_form_evidence_grants")
|
||||||
@@ -0,0 +1,52 @@
|
|||||||
|
"""govern managed file retention and legal holds
|
||||||
|
|
||||||
|
Revision ID: a2b3c4d5e6f9
|
||||||
|
Revises: f1a2b3c4d5e7
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
|
||||||
|
revision = "a2b3c4d5e6f9"
|
||||||
|
down_revision = "f1a2b3c4d5e7"
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
with op.batch_alter_table("file_assets") as batch_op:
|
||||||
|
batch_op.add_column(
|
||||||
|
sa.Column("retained_until", sa.DateTime(timezone=True), nullable=True)
|
||||||
|
)
|
||||||
|
batch_op.add_column(
|
||||||
|
sa.Column(
|
||||||
|
"legal_hold", sa.Boolean(), nullable=False, server_default=sa.false()
|
||||||
|
)
|
||||||
|
)
|
||||||
|
batch_op.add_column(
|
||||||
|
sa.Column(
|
||||||
|
"lifecycle_revision", sa.Integer(), nullable=False, server_default="1"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
batch_op.add_column(
|
||||||
|
sa.Column("lifecycle_reason", sa.String(length=500), nullable=True)
|
||||||
|
)
|
||||||
|
batch_op.create_index(
|
||||||
|
op.f("ix_file_assets_retained_until"), ["retained_until"], unique=False
|
||||||
|
)
|
||||||
|
batch_op.create_index(
|
||||||
|
op.f("ix_file_assets_legal_hold"), ["legal_hold"], unique=False
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
with op.batch_alter_table("file_assets") as batch_op:
|
||||||
|
batch_op.drop_index(op.f("ix_file_assets_legal_hold"))
|
||||||
|
batch_op.drop_index(op.f("ix_file_assets_retained_until"))
|
||||||
|
batch_op.drop_column("lifecycle_reason")
|
||||||
|
batch_op.drop_column("lifecycle_revision")
|
||||||
|
batch_op.drop_column("legal_hold")
|
||||||
|
batch_op.drop_column("retained_until")
|
||||||
+34
@@ -0,0 +1,34 @@
|
|||||||
|
"""add stale-action revisions to Files integrity operations
|
||||||
|
|
||||||
|
Revision ID: f1a2b3c4d5e7
|
||||||
|
Revises: d0e1f2a3b4c6
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
|
||||||
|
revision = "f1a2b3c4d5e7"
|
||||||
|
down_revision = "d0e1f2a3b4c6"
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
with op.batch_alter_table("file_integrity_scans") as batch_op:
|
||||||
|
batch_op.add_column(
|
||||||
|
sa.Column("revision", sa.Integer(), nullable=False, server_default="1")
|
||||||
|
)
|
||||||
|
with op.batch_alter_table("file_integrity_findings") as batch_op:
|
||||||
|
batch_op.add_column(
|
||||||
|
sa.Column("revision", sa.Integer(), nullable=False, server_default="1")
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
with op.batch_alter_table("file_integrity_findings") as batch_op:
|
||||||
|
batch_op.drop_column("revision")
|
||||||
|
with op.batch_alter_table("file_integrity_scans") as batch_op:
|
||||||
|
batch_op.drop_column("revision")
|
||||||
@@ -0,0 +1,113 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from collections.abc import Sequence
|
||||||
|
from urllib.parse import quote
|
||||||
|
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from govoplan_core.core.records import (
|
||||||
|
RecordContractError,
|
||||||
|
RecordSourceLocator,
|
||||||
|
RecordSourceReference,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.db.models import FileBlob, FileVersion
|
||||||
|
from govoplan_files.backend.storage.common import FileStorageError
|
||||||
|
from govoplan_files.backend.storage.files import get_asset_for_user
|
||||||
|
|
||||||
|
|
||||||
|
CAPABILITY_RECORD_SOURCE_FILES = "records.source.files"
|
||||||
|
|
||||||
|
|
||||||
|
class FilesRecordSource:
|
||||||
|
provider_id = "files"
|
||||||
|
|
||||||
|
def resource_types(self) -> Sequence[str]:
|
||||||
|
return ("file_version",)
|
||||||
|
|
||||||
|
def resolve(
|
||||||
|
self,
|
||||||
|
session: object,
|
||||||
|
principal: object,
|
||||||
|
*,
|
||||||
|
locator: RecordSourceLocator,
|
||||||
|
purpose: str,
|
||||||
|
) -> RecordSourceReference:
|
||||||
|
if not isinstance(session, Session):
|
||||||
|
raise RecordContractError(
|
||||||
|
"Files record references require a database session."
|
||||||
|
)
|
||||||
|
tenant_id = str(getattr(principal, "tenant_id", "") or "").strip()
|
||||||
|
if not tenant_id or locator.tenant_id != tenant_id:
|
||||||
|
raise RecordContractError("Files record references cannot cross tenants.")
|
||||||
|
if locator.source_module != "files" or locator.resource_type != "file_version":
|
||||||
|
raise RecordContractError("Unsupported Files record source type.")
|
||||||
|
if not str(purpose or "").strip():
|
||||||
|
raise RecordContractError("Files record references require a purpose.")
|
||||||
|
if not hasattr(principal, "has") or not (
|
||||||
|
principal.has("files:file:read") or principal.has("files:file:admin")
|
||||||
|
):
|
||||||
|
raise RecordContractError("Current Files read permission is required.")
|
||||||
|
user = getattr(principal, "user", None)
|
||||||
|
user_id = str(
|
||||||
|
getattr(user, "id", "") or getattr(principal, "membership_id", "") or ""
|
||||||
|
).strip()
|
||||||
|
if not user_id:
|
||||||
|
raise RecordContractError(
|
||||||
|
"Files record references require a tenant user principal."
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
asset = get_asset_for_user(
|
||||||
|
session,
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
user_id=user_id,
|
||||||
|
asset_id=locator.resource_id,
|
||||||
|
is_admin=principal.has("files:file:admin"),
|
||||||
|
)
|
||||||
|
except FileStorageError as exc:
|
||||||
|
raise RecordContractError(str(exc)) from exc
|
||||||
|
version_query = session.query(FileVersion).filter(
|
||||||
|
FileVersion.tenant_id == tenant_id,
|
||||||
|
FileVersion.file_asset_id == asset.id,
|
||||||
|
)
|
||||||
|
revision = locator.source_revision.strip()
|
||||||
|
version = version_query.filter(FileVersion.id == revision).one_or_none()
|
||||||
|
if version is None:
|
||||||
|
raise RecordContractError("The exact file version does not exist.")
|
||||||
|
blob = session.get(FileBlob, version.blob_id)
|
||||||
|
if blob is None or blob.tenant_id != tenant_id:
|
||||||
|
raise RecordContractError(
|
||||||
|
"The exact file version has no managed content object."
|
||||||
|
)
|
||||||
|
if blob.quarantined_at is not None or blob.integrity_status == "failed":
|
||||||
|
raise RecordContractError(
|
||||||
|
"The exact file version failed the current integrity gate."
|
||||||
|
)
|
||||||
|
return RecordSourceReference(
|
||||||
|
locator=locator,
|
||||||
|
label=version.filename_at_upload,
|
||||||
|
authority_mode="external_authoritative",
|
||||||
|
content_sha256=version.checksum_sha256,
|
||||||
|
content_type=version.content_type,
|
||||||
|
size_bytes=version.size_bytes,
|
||||||
|
recorded_at=version.created_at,
|
||||||
|
launch_url=(
|
||||||
|
f"/files?fileId={quote(asset.id, safe='')}&versionId={quote(version.id, safe='')}"
|
||||||
|
),
|
||||||
|
metadata={
|
||||||
|
"display_path": version.display_path_at_upload,
|
||||||
|
"version_number": version.version_number,
|
||||||
|
"integrity_status": blob.integrity_status,
|
||||||
|
"protection": blob.protection_discriminator,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def create_files_record_source(_context: object) -> FilesRecordSource:
|
||||||
|
return FilesRecordSource()
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"CAPABILITY_RECORD_SOURCE_FILES",
|
||||||
|
"FilesRecordSource",
|
||||||
|
"create_files_record_source",
|
||||||
|
]
|
||||||
@@ -1039,6 +1039,12 @@ def _asset_response(
|
|||||||
created_at=asset.created_at.isoformat(),
|
created_at=asset.created_at.isoformat(),
|
||||||
updated_at=asset.updated_at.isoformat(),
|
updated_at=asset.updated_at.isoformat(),
|
||||||
deleted_at=asset.deleted_at.isoformat() if asset.deleted_at else None,
|
deleted_at=asset.deleted_at.isoformat() if asset.deleted_at else None,
|
||||||
|
retained_until=(
|
||||||
|
asset.retained_until.isoformat() if asset.retained_until else None
|
||||||
|
),
|
||||||
|
legal_hold=asset.legal_hold,
|
||||||
|
lifecycle_revision=asset.lifecycle_revision,
|
||||||
|
lifecycle_reason=asset.lifecycle_reason,
|
||||||
audit_relevant=asset_is_audit_relevant(session, asset),
|
audit_relevant=asset_is_audit_relevant(session, asset),
|
||||||
metadata=metadata,
|
metadata=metadata,
|
||||||
source_provenance=source_provenance_from_metadata(metadata),
|
source_provenance=source_provenance_from_metadata(metadata),
|
||||||
|
|||||||
@@ -11,7 +11,9 @@ from govoplan_files.backend.routes.connector_settings import (
|
|||||||
router as connector_settings_router,
|
router as connector_settings_router,
|
||||||
)
|
)
|
||||||
from govoplan_files.backend.routes.folders import router as folders_router
|
from govoplan_files.backend.routes.folders import router as folders_router
|
||||||
|
from govoplan_files.backend.routes.form_evidence import router as form_evidence_router
|
||||||
from govoplan_files.backend.routes.integrity import router as integrity_router
|
from govoplan_files.backend.routes.integrity import router as integrity_router
|
||||||
|
from govoplan_files.backend.routes.lifecycle import router as lifecycle_router
|
||||||
from govoplan_files.backend.routes.listing import router as listing_router
|
from govoplan_files.backend.routes.listing import router as listing_router
|
||||||
from govoplan_files.backend.routes.shares import router as shares_router
|
from govoplan_files.backend.routes.shares import router as shares_router
|
||||||
from govoplan_files.backend.routes.spaces import router as spaces_router
|
from govoplan_files.backend.routes.spaces import router as spaces_router
|
||||||
@@ -23,7 +25,9 @@ router = APIRouter()
|
|||||||
for workflow_router in (
|
for workflow_router in (
|
||||||
spaces_router,
|
spaces_router,
|
||||||
folders_router,
|
folders_router,
|
||||||
|
form_evidence_router,
|
||||||
integrity_router,
|
integrity_router,
|
||||||
|
lifecycle_router,
|
||||||
listing_router,
|
listing_router,
|
||||||
uploads_router,
|
uploads_router,
|
||||||
connector_settings_router,
|
connector_settings_router,
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ from govoplan_files.backend.storage.common import FileStorageError
|
|||||||
from govoplan_files.backend.storage.files import (
|
from govoplan_files.backend.storage.files import (
|
||||||
get_asset_for_user,
|
get_asset_for_user,
|
||||||
read_asset_bytes,
|
read_asset_bytes,
|
||||||
|
read_asset_version_bytes,
|
||||||
soft_delete_assets,
|
soft_delete_assets,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -85,6 +86,46 @@ def download_file(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/{file_id}/versions/{version_id}/download")
|
||||||
|
def download_file_version(
|
||||||
|
file_id: str,
|
||||||
|
version_id: str,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(require_scope("files:file:download")),
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
asset = get_asset_for_user(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
asset_id=file_id,
|
||||||
|
is_admin=_is_admin(principal),
|
||||||
|
)
|
||||||
|
data, version, blob = read_asset_version_bytes(session, asset, version_id)
|
||||||
|
_audit_connector_event(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
action="files.connector.accessed",
|
||||||
|
asset=asset,
|
||||||
|
version=version,
|
||||||
|
blob=blob,
|
||||||
|
operation="download-version",
|
||||||
|
commit=True,
|
||||||
|
)
|
||||||
|
except FileStorageError as exc:
|
||||||
|
raise _http_error(exc, not_found=True) from exc
|
||||||
|
headers = {
|
||||||
|
"Content-Disposition": _attachment_disposition(
|
||||||
|
version.filename_at_upload or asset.filename
|
||||||
|
)
|
||||||
|
}
|
||||||
|
return StreamingResponse(
|
||||||
|
BytesIO(data),
|
||||||
|
media_type=blob.content_type or "application/octet-stream",
|
||||||
|
headers=headers,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@router.delete("/{file_id}", response_model=BulkDeleteResponse)
|
@router.delete("/{file_id}", response_model=BulkDeleteResponse)
|
||||||
def delete_file(
|
def delete_file(
|
||||||
file_id: str,
|
file_id: str,
|
||||||
|
|||||||
@@ -5,11 +5,14 @@ from fastapi import APIRouter, Depends, HTTPException, status
|
|||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from govoplan_core.auth import ApiPrincipal, require_any_scope, require_scope
|
from govoplan_core.auth import ApiPrincipal, require_any_scope, require_scope
|
||||||
|
from govoplan_core.audit.logging import audit_from_principal
|
||||||
from govoplan_files.backend.schemas import (
|
from govoplan_files.backend.schemas import (
|
||||||
FileConnectorBrowseItem,
|
FileConnectorBrowseItem,
|
||||||
FileConnectorBrowseResponse,
|
FileConnectorBrowseResponse,
|
||||||
FileConnectorImportRequest,
|
FileConnectorImportRequest,
|
||||||
FileConnectorSyncResponse,
|
FileConnectorSyncResponse,
|
||||||
|
FileConnectorWriteRequest,
|
||||||
|
FileConnectorWriteResponse,
|
||||||
FileUploadResponse,
|
FileUploadResponse,
|
||||||
)
|
)
|
||||||
from govoplan_core.db.session import get_session
|
from govoplan_core.db.session import get_session
|
||||||
@@ -35,8 +38,14 @@ from govoplan_files.backend.storage.connector_policy import (
|
|||||||
)
|
)
|
||||||
from govoplan_files.backend.storage.files import (
|
from govoplan_files.backend.storage.files import (
|
||||||
create_file_asset,
|
create_file_asset,
|
||||||
|
get_asset_for_user,
|
||||||
|
read_asset_bytes,
|
||||||
sync_file_asset_from_source,
|
sync_file_asset_from_source,
|
||||||
)
|
)
|
||||||
|
from govoplan_files.backend.storage.connector_spaces import (
|
||||||
|
get_connector_space_for_user,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.storage.connector_writes import write_connector_file
|
||||||
|
|
||||||
|
|
||||||
from govoplan_files.backend.route_support import (
|
from govoplan_files.backend.route_support import (
|
||||||
@@ -55,6 +64,112 @@ from govoplan_files.backend.route_support import (
|
|||||||
router = APIRouter(prefix="/files", tags=["files"])
|
router = APIRouter(prefix="/files", tags=["files"])
|
||||||
|
|
||||||
|
|
||||||
|
@router.post(
|
||||||
|
"/connector-spaces/{space_id}/write-back",
|
||||||
|
response_model=FileConnectorWriteResponse,
|
||||||
|
)
|
||||||
|
def write_back_connector_file(
|
||||||
|
space_id: str,
|
||||||
|
payload: FileConnectorWriteRequest,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(require_scope("files:connector:write")),
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
space = get_connector_space_for_user(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
space_id=space_id,
|
||||||
|
is_admin=_is_admin(principal),
|
||||||
|
)
|
||||||
|
if space.read_only:
|
||||||
|
raise FileStorageError("This connector space is read-only")
|
||||||
|
profile = _visible_connector_profile(
|
||||||
|
session, principal, space.connector_profile_id
|
||||||
|
)
|
||||||
|
requested_path = normalize_connector_browse_path(payload.remote_path)
|
||||||
|
remote_path = normalize_connector_browse_path(
|
||||||
|
"/".join(part for part in (space.remote_path, requested_path) if part)
|
||||||
|
)
|
||||||
|
decision = connector_policy_decision(
|
||||||
|
ConnectorAccessRequest(
|
||||||
|
connector_id=profile.id,
|
||||||
|
credential_id=profile.credential_profile_id,
|
||||||
|
provider=profile.provider,
|
||||||
|
external_path=remote_path,
|
||||||
|
external_url=connector_effective_endpoint_url(
|
||||||
|
provider=profile.provider,
|
||||||
|
endpoint_url=profile.endpoint_url,
|
||||||
|
metadata=profile.metadata,
|
||||||
|
),
|
||||||
|
operation="write",
|
||||||
|
),
|
||||||
|
profile.policy_sources,
|
||||||
|
)
|
||||||
|
if not decision.allowed:
|
||||||
|
raise ConnectorPolicyDenied(decision)
|
||||||
|
asset = get_asset_for_user(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
asset_id=payload.file_id,
|
||||||
|
require_write=True,
|
||||||
|
is_admin=_is_admin(principal),
|
||||||
|
)
|
||||||
|
data, version, blob = read_asset_bytes(session, asset)
|
||||||
|
connector_library_id = space.library_id
|
||||||
|
content_type = blob.content_type
|
||||||
|
# Close the read snapshot before the independent recovery transaction
|
||||||
|
# records authority for the external effect. This avoids upgrading an
|
||||||
|
# older SQLite read snapshot after the ledger commit.
|
||||||
|
session.commit()
|
||||||
|
result = write_connector_file(
|
||||||
|
profile,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
library_id=connector_library_id,
|
||||||
|
remote_path=remote_path,
|
||||||
|
data=data,
|
||||||
|
content_type=content_type,
|
||||||
|
expected_revision=payload.expected_revision,
|
||||||
|
idempotency_key=payload.idempotency_key,
|
||||||
|
)
|
||||||
|
audit_from_principal(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
action="files.connector.written",
|
||||||
|
object_type="file",
|
||||||
|
object_id=asset.id,
|
||||||
|
details={
|
||||||
|
"file_version_id": version.id,
|
||||||
|
"file_blob_id": blob.id,
|
||||||
|
"checksum_sha256": blob.checksum_sha256,
|
||||||
|
"connector_space_id": space.id,
|
||||||
|
"connector_profile_id": profile.id,
|
||||||
|
"remote_path": remote_path,
|
||||||
|
"recovery_operation_id": result.recovery_operation_id,
|
||||||
|
"recovery_status": result.status,
|
||||||
|
"revision": result.revision,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
return FileConnectorWriteResponse(
|
||||||
|
recovery_operation_id=result.recovery_operation_id,
|
||||||
|
status=result.status,
|
||||||
|
replayed=result.replayed,
|
||||||
|
provider=result.provider,
|
||||||
|
remote_path=result.remote_path,
|
||||||
|
revision=result.revision,
|
||||||
|
checksum_sha256=result.checksum_sha256,
|
||||||
|
size_bytes=result.size_bytes,
|
||||||
|
)
|
||||||
|
except ConnectorPolicyDenied as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _connector_policy_error(exc) from exc
|
||||||
|
except (FileStorageError, ConnectorBrowseError) as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _http_error(exc) from exc
|
||||||
|
|
||||||
|
|
||||||
@router.post(
|
@router.post(
|
||||||
"/connectors/profiles/{profile_id}/import", response_model=FileUploadResponse
|
"/connectors/profiles/{profile_id}/import", response_model=FileUploadResponse
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -0,0 +1,172 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
import hashlib
|
||||||
|
|
||||||
|
from fastapi import (
|
||||||
|
APIRouter,
|
||||||
|
Depends,
|
||||||
|
File as FastAPIFile,
|
||||||
|
Header,
|
||||||
|
HTTPException,
|
||||||
|
UploadFile,
|
||||||
|
status,
|
||||||
|
)
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from govoplan_core.core.events import (
|
||||||
|
EventActorRef,
|
||||||
|
EventObjectRef,
|
||||||
|
EventTenantRef,
|
||||||
|
PlatformEvent,
|
||||||
|
emit_platform_event,
|
||||||
|
)
|
||||||
|
from govoplan_core.core.institutional import EvidenceReference
|
||||||
|
from govoplan_core.db.session import get_session
|
||||||
|
from govoplan_files.backend.db.models import FileFormEvidenceGrant
|
||||||
|
from govoplan_files.backend.route_support import _read_limited_upload
|
||||||
|
from govoplan_files.backend.storage.common import FileStorageError
|
||||||
|
from govoplan_files.backend.storage.files import create_file_asset
|
||||||
|
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/files", tags=["files"])
|
||||||
|
|
||||||
|
|
||||||
|
@router.post(
|
||||||
|
"/form-evidence/upload",
|
||||||
|
response_model=dict[str, object],
|
||||||
|
status_code=status.HTTP_201_CREATED,
|
||||||
|
)
|
||||||
|
def upload_form_evidence(
|
||||||
|
file: UploadFile = FastAPIFile(...),
|
||||||
|
x_form_evidence_token: str = Header(alias="X-Form-Evidence-Token"),
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
) -> dict[str, object]:
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
try:
|
||||||
|
grant = (
|
||||||
|
session.query(FileFormEvidenceGrant)
|
||||||
|
.filter(
|
||||||
|
FileFormEvidenceGrant.token_sha256
|
||||||
|
== _token_sha256(x_form_evidence_token)
|
||||||
|
)
|
||||||
|
.with_for_update()
|
||||||
|
.one_or_none()
|
||||||
|
)
|
||||||
|
if grant is None or grant.status != "issued" or _aware(grant.expires_at) <= now:
|
||||||
|
raise _upload_unavailable()
|
||||||
|
content_type = (
|
||||||
|
str(file.content_type or "application/octet-stream")
|
||||||
|
.split(";", 1)[0]
|
||||||
|
.strip()
|
||||||
|
.casefold()
|
||||||
|
)
|
||||||
|
if grant.allowed_content_types and content_type not in set(
|
||||||
|
grant.allowed_content_types
|
||||||
|
):
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_415_UNSUPPORTED_MEDIA_TYPE,
|
||||||
|
detail="This Form does not accept the uploaded document type.",
|
||||||
|
)
|
||||||
|
data = _read_limited_upload(file, max_bytes=grant.max_size_bytes)
|
||||||
|
stored = create_file_asset(
|
||||||
|
session,
|
||||||
|
tenant_id=grant.tenant_id,
|
||||||
|
owner_type="user",
|
||||||
|
owner_id=grant.custodian_user_id,
|
||||||
|
user_id=grant.custodian_user_id,
|
||||||
|
filename=file.filename or "form-attachment",
|
||||||
|
data=data,
|
||||||
|
folder=f"Form submissions/{grant.form_instance_id}",
|
||||||
|
content_type=content_type,
|
||||||
|
description="Managed attachment captured through Forms Runtime.",
|
||||||
|
metadata={
|
||||||
|
"form_evidence": {
|
||||||
|
"grant_id": grant.id,
|
||||||
|
"form_instance_id": grant.form_instance_id,
|
||||||
|
"form_definition_id": grant.form_definition_id,
|
||||||
|
"form_definition_revision": grant.form_definition_revision,
|
||||||
|
"purpose": grant.purpose,
|
||||||
|
},
|
||||||
|
"source_provenance": {
|
||||||
|
"source_type": "form_evidence",
|
||||||
|
"connector_id": "forms_runtime.evidence.files",
|
||||||
|
"provider": "forms_runtime",
|
||||||
|
"external_id": grant.id,
|
||||||
|
"revision": grant.form_definition_revision,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
conflict_strategy="rename",
|
||||||
|
is_admin=True,
|
||||||
|
)
|
||||||
|
grant.status = "uploaded"
|
||||||
|
grant.file_asset_id = stored.asset.id
|
||||||
|
grant.file_version_id = stored.version.id
|
||||||
|
grant.uploaded_at = now
|
||||||
|
session.add(grant)
|
||||||
|
evidence = EvidenceReference(
|
||||||
|
kind="document",
|
||||||
|
owner_module="files",
|
||||||
|
evidence_id=stored.asset.id,
|
||||||
|
tenant_id=grant.tenant_id,
|
||||||
|
version=stored.version.id,
|
||||||
|
checksum=stored.version.checksum_sha256,
|
||||||
|
source_ref=(f"form_submission:{grant.form_instance_id}:grant:{grant.id}"),
|
||||||
|
responsible_actor_ref=f"user:{grant.custodian_user_id}",
|
||||||
|
captured_at=now,
|
||||||
|
)
|
||||||
|
emit_platform_event(
|
||||||
|
session,
|
||||||
|
PlatformEvent(
|
||||||
|
type="files.form_evidence.uploaded",
|
||||||
|
module_id="files",
|
||||||
|
payload={
|
||||||
|
"grant_id": grant.id,
|
||||||
|
"form_instance_id": grant.form_instance_id,
|
||||||
|
"file_asset_id": stored.asset.id,
|
||||||
|
"file_version_id": stored.version.id,
|
||||||
|
"checksum_sha256": stored.version.checksum_sha256,
|
||||||
|
"size_bytes": stored.version.size_bytes,
|
||||||
|
},
|
||||||
|
occurred_at=now,
|
||||||
|
actor=EventActorRef(type="user", id=grant.custodian_user_id),
|
||||||
|
tenant=EventTenantRef(id=grant.tenant_id),
|
||||||
|
resource=EventObjectRef(type="file", id=stored.asset.id),
|
||||||
|
classification="confidential",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
except HTTPException:
|
||||||
|
session.rollback()
|
||||||
|
raise
|
||||||
|
except (FileStorageError, ValueError) as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
|
||||||
|
detail=str(exc),
|
||||||
|
) from exc
|
||||||
|
return {
|
||||||
|
"grant_id": grant.id,
|
||||||
|
"evidence": evidence.to_dict(include_inspection=True),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _token_sha256(token: str) -> str:
|
||||||
|
clean = str(token or "").strip()
|
||||||
|
if len(clean) < 32:
|
||||||
|
raise _upload_unavailable()
|
||||||
|
return hashlib.sha256(clean.encode("utf-8")).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def _upload_unavailable() -> HTTPException:
|
||||||
|
return HTTPException(
|
||||||
|
status_code=status.HTTP_404_NOT_FOUND,
|
||||||
|
detail="The Form evidence upload is unavailable.",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _aware(value: datetime) -> datetime:
|
||||||
|
return value if value.tzinfo is not None else value.replace(tzinfo=UTC)
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["router", "upload_form_evidence"]
|
||||||
@@ -18,6 +18,7 @@ from govoplan_files.backend.schemas import (
|
|||||||
FileIntegrityFindingResponse,
|
FileIntegrityFindingResponse,
|
||||||
FileIntegrityFindingsResponse,
|
FileIntegrityFindingsResponse,
|
||||||
FileIntegrityScanCreateRequest,
|
FileIntegrityScanCreateRequest,
|
||||||
|
FileIntegrityScanRunRequest,
|
||||||
FileIntegrityScanResponse,
|
FileIntegrityScanResponse,
|
||||||
FileIntegrityScansResponse,
|
FileIntegrityScansResponse,
|
||||||
)
|
)
|
||||||
@@ -93,10 +94,17 @@ def create_scan(
|
|||||||
@router.post("/scans/{scan_id}/run", response_model=FileIntegrityScanResponse)
|
@router.post("/scans/{scan_id}/run", response_model=FileIntegrityScanResponse)
|
||||||
def run_scan_batch(
|
def run_scan_batch(
|
||||||
scan_id: str,
|
scan_id: str,
|
||||||
|
payload: FileIntegrityScanRunRequest,
|
||||||
session: Session = Depends(get_session),
|
session: Session = Depends(get_session),
|
||||||
principal: ApiPrincipal = Depends(require_scope("files:file:admin")),
|
principal: ApiPrincipal = Depends(require_scope("files:file:admin")),
|
||||||
) -> FileIntegrityScanResponse:
|
) -> FileIntegrityScanResponse:
|
||||||
scan = _scan_for_tenant(session, scan_id, principal.tenant_id)
|
scan = _scan_for_tenant(
|
||||||
|
session,
|
||||||
|
scan_id,
|
||||||
|
principal.tenant_id,
|
||||||
|
for_update=True,
|
||||||
|
)
|
||||||
|
_assert_expected_revision(scan.revision, payload.expected_revision)
|
||||||
previous_status = scan.status
|
previous_status = scan.status
|
||||||
try:
|
try:
|
||||||
run_integrity_scan_batch(session, scan)
|
run_integrity_scan_batch(session, scan)
|
||||||
@@ -117,7 +125,12 @@ def run_scan_batch(
|
|||||||
return _scan_response(scan)
|
return _scan_response(scan)
|
||||||
except (FileStorageError, StorageBackendError) as exc:
|
except (FileStorageError, StorageBackendError) as exc:
|
||||||
session.rollback()
|
session.rollback()
|
||||||
scan = _scan_for_tenant(session, scan_id, principal.tenant_id)
|
scan = _scan_for_tenant(
|
||||||
|
session,
|
||||||
|
scan_id,
|
||||||
|
principal.tenant_id,
|
||||||
|
for_update=True,
|
||||||
|
)
|
||||||
mark_integrity_scan_failed(scan, error=exc)
|
mark_integrity_scan_failed(scan, error=exc)
|
||||||
audit_from_principal(
|
audit_from_principal(
|
||||||
session,
|
session,
|
||||||
@@ -172,7 +185,13 @@ def recheck_finding(
|
|||||||
session: Session = Depends(get_session),
|
session: Session = Depends(get_session),
|
||||||
principal: ApiPrincipal = Depends(require_scope("files:file:admin")),
|
principal: ApiPrincipal = Depends(require_scope("files:file:admin")),
|
||||||
) -> FileIntegrityActionResponse:
|
) -> FileIntegrityActionResponse:
|
||||||
finding = _finding_for_tenant(session, finding_id, principal.tenant_id)
|
finding = _finding_for_tenant(
|
||||||
|
session,
|
||||||
|
finding_id,
|
||||||
|
principal.tenant_id,
|
||||||
|
for_update=True,
|
||||||
|
)
|
||||||
|
_assert_expected_revision(finding.revision, payload.expected_revision)
|
||||||
try:
|
try:
|
||||||
result = recheck_integrity_finding(
|
result = recheck_integrity_finding(
|
||||||
session,
|
session,
|
||||||
@@ -198,7 +217,13 @@ def cleanup_finding(
|
|||||||
session: Session = Depends(get_session),
|
session: Session = Depends(get_session),
|
||||||
principal: ApiPrincipal = Depends(require_scope("files:file:admin")),
|
principal: ApiPrincipal = Depends(require_scope("files:file:admin")),
|
||||||
) -> FileIntegrityActionResponse:
|
) -> FileIntegrityActionResponse:
|
||||||
finding = _finding_for_tenant(session, finding_id, principal.tenant_id)
|
finding = _finding_for_tenant(
|
||||||
|
session,
|
||||||
|
finding_id,
|
||||||
|
principal.tenant_id,
|
||||||
|
for_update=True,
|
||||||
|
)
|
||||||
|
_assert_expected_revision(finding.revision, payload.expected_revision)
|
||||||
try:
|
try:
|
||||||
result = cleanup_orphan_finding(
|
result = cleanup_orphan_finding(
|
||||||
session,
|
session,
|
||||||
@@ -218,8 +243,13 @@ def _scan_for_tenant(
|
|||||||
session: Session,
|
session: Session,
|
||||||
scan_id: str,
|
scan_id: str,
|
||||||
tenant_id: str,
|
tenant_id: str,
|
||||||
|
*,
|
||||||
|
for_update: bool = False,
|
||||||
) -> FileIntegrityScan:
|
) -> FileIntegrityScan:
|
||||||
scan = session.get(FileIntegrityScan, scan_id)
|
query = session.query(FileIntegrityScan).filter(FileIntegrityScan.id == scan_id)
|
||||||
|
if for_update:
|
||||||
|
query = query.populate_existing().with_for_update()
|
||||||
|
scan = query.one_or_none()
|
||||||
if scan is None or scan.tenant_id != tenant_id:
|
if scan is None or scan.tenant_id != tenant_id:
|
||||||
raise HTTPException(
|
raise HTTPException(
|
||||||
status_code=status.HTTP_404_NOT_FOUND,
|
status_code=status.HTTP_404_NOT_FOUND,
|
||||||
@@ -232,8 +262,15 @@ def _finding_for_tenant(
|
|||||||
session: Session,
|
session: Session,
|
||||||
finding_id: str,
|
finding_id: str,
|
||||||
tenant_id: str,
|
tenant_id: str,
|
||||||
|
*,
|
||||||
|
for_update: bool = False,
|
||||||
) -> FileIntegrityFinding:
|
) -> FileIntegrityFinding:
|
||||||
finding = session.get(FileIntegrityFinding, finding_id)
|
query = session.query(FileIntegrityFinding).filter(
|
||||||
|
FileIntegrityFinding.id == finding_id
|
||||||
|
)
|
||||||
|
if for_update:
|
||||||
|
query = query.populate_existing().with_for_update()
|
||||||
|
finding = query.one_or_none()
|
||||||
if finding is None or finding.tenant_id != tenant_id:
|
if finding is None or finding.tenant_id != tenant_id:
|
||||||
raise HTTPException(
|
raise HTTPException(
|
||||||
status_code=status.HTTP_404_NOT_FOUND,
|
status_code=status.HTTP_404_NOT_FOUND,
|
||||||
@@ -242,6 +279,17 @@ def _finding_for_tenant(
|
|||||||
return finding
|
return finding
|
||||||
|
|
||||||
|
|
||||||
|
def _assert_expected_revision(current: int, expected: int) -> None:
|
||||||
|
if current != expected:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_409_CONFLICT,
|
||||||
|
detail=(
|
||||||
|
"The integrity record changed after it was loaded; reload before "
|
||||||
|
"performing this action."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _audit_integrity_action(session, principal, result) -> None:
|
def _audit_integrity_action(session, principal, result) -> None:
|
||||||
audit_from_principal(
|
audit_from_principal(
|
||||||
session,
|
session,
|
||||||
@@ -271,6 +319,7 @@ def _scan_response(scan: FileIntegrityScan) -> FileIntegrityScanResponse:
|
|||||||
storage_backend=scan.storage_backend,
|
storage_backend=scan.storage_backend,
|
||||||
storage_prefix=scan.storage_prefix,
|
storage_prefix=scan.storage_prefix,
|
||||||
status=scan.status,
|
status=scan.status,
|
||||||
|
revision=scan.revision,
|
||||||
phase=scan.phase,
|
phase=scan.phase,
|
||||||
verify_checksums=scan.verify_checksums,
|
verify_checksums=scan.verify_checksums,
|
||||||
batch_size=scan.batch_size,
|
batch_size=scan.batch_size,
|
||||||
@@ -299,6 +348,7 @@ def _finding_response(
|
|||||||
tenant_id=finding.tenant_id,
|
tenant_id=finding.tenant_id,
|
||||||
kind=finding.kind,
|
kind=finding.kind,
|
||||||
state=finding.state,
|
state=finding.state,
|
||||||
|
revision=finding.revision,
|
||||||
blob_id=finding.blob_id,
|
blob_id=finding.blob_id,
|
||||||
storage_key=finding.storage_key,
|
storage_key=finding.storage_key,
|
||||||
expected_size_bytes=finding.expected_size_bytes,
|
expected_size_bytes=finding.expected_size_bytes,
|
||||||
|
|||||||
@@ -0,0 +1,297 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Depends
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from govoplan_core.auth import ApiPrincipal, require_scope
|
||||||
|
from govoplan_core.audit.logging import audit_from_principal
|
||||||
|
from govoplan_core.db.session import get_session
|
||||||
|
from govoplan_files.backend.route_support import (
|
||||||
|
_asset_response,
|
||||||
|
_connector_space_response,
|
||||||
|
_http_error,
|
||||||
|
_is_admin,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.schemas import (
|
||||||
|
FileAssetResponse,
|
||||||
|
FileBlobGcRequest,
|
||||||
|
FileBlobGcResponse,
|
||||||
|
FileConnectorSpaceResponse,
|
||||||
|
FileFolderDeleteRequest,
|
||||||
|
FileFolderRestoreResponse,
|
||||||
|
FileLifecycleUpdateRequest,
|
||||||
|
FilePurgeExecuteRequest,
|
||||||
|
FilePurgePreviewItem,
|
||||||
|
FilePurgePreviewRequest,
|
||||||
|
FilePurgePreviewResponse,
|
||||||
|
FilePurgeResponse,
|
||||||
|
FileRestoreResponse,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.storage.common import FileStorageError
|
||||||
|
from govoplan_files.backend.storage.lifecycle import (
|
||||||
|
execute_asset_purge,
|
||||||
|
garbage_collect_unreferenced_blobs,
|
||||||
|
get_asset_for_lifecycle,
|
||||||
|
preview_asset_purge,
|
||||||
|
restore_asset,
|
||||||
|
restore_connector_space,
|
||||||
|
restore_folder,
|
||||||
|
set_asset_lifecycle,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/files", tags=["files-lifecycle"])
|
||||||
|
|
||||||
|
|
||||||
|
@router.patch("/{file_id}/lifecycle", response_model=FileAssetResponse)
|
||||||
|
def update_file_lifecycle(
|
||||||
|
file_id: str,
|
||||||
|
payload: FileLifecycleUpdateRequest,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(require_scope("files:file:retention")),
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
asset = get_asset_for_lifecycle(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
asset_id=file_id,
|
||||||
|
is_admin=_is_admin(principal),
|
||||||
|
)
|
||||||
|
set_asset_lifecycle(
|
||||||
|
session,
|
||||||
|
asset,
|
||||||
|
retained_until=payload.retained_until,
|
||||||
|
legal_hold=payload.legal_hold,
|
||||||
|
reason=payload.reason,
|
||||||
|
expected_revision=payload.expected_revision,
|
||||||
|
)
|
||||||
|
audit_from_principal(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
action="files.lifecycle.updated",
|
||||||
|
object_type="file",
|
||||||
|
object_id=asset.id,
|
||||||
|
details={
|
||||||
|
"retained_until": (
|
||||||
|
asset.retained_until.isoformat() if asset.retained_until else None
|
||||||
|
),
|
||||||
|
"legal_hold": asset.legal_hold,
|
||||||
|
"lifecycle_revision": asset.lifecycle_revision,
|
||||||
|
"reason": asset.lifecycle_reason,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
return _asset_response(session, asset, include_shares=True)
|
||||||
|
except FileStorageError as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _http_error(exc, not_found=True) from exc
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/assets/{file_id}/restore", response_model=FileRestoreResponse)
|
||||||
|
def restore_file(
|
||||||
|
file_id: str,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(require_scope("files:file:restore")),
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
asset = get_asset_for_lifecycle(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
asset_id=file_id,
|
||||||
|
is_admin=_is_admin(principal),
|
||||||
|
)
|
||||||
|
changed = restore_asset(session, asset)
|
||||||
|
if changed:
|
||||||
|
audit_from_principal(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
action="files.file.restored",
|
||||||
|
object_type="file",
|
||||||
|
object_id=asset.id,
|
||||||
|
details={"display_path": asset.display_path},
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
return FileRestoreResponse(restored_count=1 if changed else 0)
|
||||||
|
except FileStorageError as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _http_error(exc, not_found=True) from exc
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/folders/restore", response_model=FileFolderRestoreResponse)
|
||||||
|
def restore_file_folder(
|
||||||
|
payload: FileFolderDeleteRequest,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(require_scope("files:file:restore")),
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
restored_folders, restored_files = restore_folder(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
owner_type=payload.owner_type,
|
||||||
|
owner_id=payload.owner_id,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
path=payload.path,
|
||||||
|
recursive=payload.recursive,
|
||||||
|
is_admin=_is_admin(principal),
|
||||||
|
)
|
||||||
|
audit_from_principal(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
action="files.folder.restored",
|
||||||
|
object_type="file_folder",
|
||||||
|
object_id=payload.path,
|
||||||
|
details={
|
||||||
|
"restored_folders": restored_folders,
|
||||||
|
"restored_files": restored_files,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
return FileFolderRestoreResponse(
|
||||||
|
restored_folders=restored_folders, restored_files=restored_files
|
||||||
|
)
|
||||||
|
except FileStorageError as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _http_error(exc, not_found=True) from exc
|
||||||
|
|
||||||
|
|
||||||
|
@router.post(
|
||||||
|
"/connector-spaces/{space_id}/restore",
|
||||||
|
response_model=FileConnectorSpaceResponse,
|
||||||
|
)
|
||||||
|
def restore_file_connector_space(
|
||||||
|
space_id: str,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(require_scope("files:file:restore")),
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
space = restore_connector_space(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
space_id=space_id,
|
||||||
|
is_admin=_is_admin(principal),
|
||||||
|
)
|
||||||
|
audit_from_principal(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
action="files.connector_space.restored",
|
||||||
|
object_type="file_connector_space",
|
||||||
|
object_id=space.id,
|
||||||
|
details={"label": space.label},
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
return _connector_space_response(space)
|
||||||
|
except FileStorageError as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _http_error(exc, not_found=True) from exc
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/purge/preview", response_model=FilePurgePreviewResponse)
|
||||||
|
def preview_file_purge(
|
||||||
|
payload: FilePurgePreviewRequest,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(require_scope("files:file:purge")),
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
preview = preview_asset_purge(
|
||||||
|
session, tenant_id=principal.tenant_id, file_ids=payload.file_ids
|
||||||
|
)
|
||||||
|
return FilePurgePreviewResponse(
|
||||||
|
preview_sha256=preview.preview_sha256,
|
||||||
|
eligible=preview.eligible,
|
||||||
|
items=[
|
||||||
|
FilePurgePreviewItem(
|
||||||
|
file_id=item.file_id,
|
||||||
|
filename=item.filename,
|
||||||
|
lifecycle_revision=item.lifecycle_revision,
|
||||||
|
deleted_at=(item.deleted_at.isoformat() if item.deleted_at else None),
|
||||||
|
retained_until=(
|
||||||
|
item.retained_until.isoformat() if item.retained_until else None
|
||||||
|
),
|
||||||
|
legal_hold=item.legal_hold,
|
||||||
|
blockers=list(item.blockers),
|
||||||
|
blob_ids=list(item.blob_ids),
|
||||||
|
)
|
||||||
|
for item in preview.items
|
||||||
|
],
|
||||||
|
)
|
||||||
|
except FileStorageError as exc:
|
||||||
|
raise _http_error(exc) from exc
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/purge/execute", response_model=FilePurgeResponse)
|
||||||
|
def execute_file_purge(
|
||||||
|
payload: FilePurgeExecuteRequest,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(require_scope("files:file:purge")),
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
result = execute_asset_purge(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
file_ids=payload.file_ids,
|
||||||
|
preview_sha256=payload.preview_sha256,
|
||||||
|
idempotency_key=payload.idempotency_key,
|
||||||
|
approval_reference=payload.approval_reference,
|
||||||
|
before_commit=lambda assets: audit_from_principal(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
action="files.files.purged",
|
||||||
|
object_type="file_batch",
|
||||||
|
object_id=payload.preview_sha256,
|
||||||
|
details={
|
||||||
|
"file_ids": [asset.id for asset in assets],
|
||||||
|
"approval_reference": payload.approval_reference,
|
||||||
|
"preview_sha256": payload.preview_sha256,
|
||||||
|
},
|
||||||
|
),
|
||||||
|
)
|
||||||
|
return FilePurgeResponse(
|
||||||
|
recovery_operation_id=result.recovery_operation_id,
|
||||||
|
status=result.status,
|
||||||
|
replayed=result.replayed,
|
||||||
|
purged_files=result.purged_files,
|
||||||
|
released_blobs=result.released_blobs,
|
||||||
|
)
|
||||||
|
except FileStorageError as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _http_error(exc) from exc
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/purge/blobs", response_model=FileBlobGcResponse)
|
||||||
|
def garbage_collect_file_blobs(
|
||||||
|
payload: FileBlobGcRequest,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(require_scope("files:file:purge")),
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
result = garbage_collect_unreferenced_blobs(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
limit=payload.limit,
|
||||||
|
approval_reference=payload.approval_reference,
|
||||||
|
)
|
||||||
|
audit_from_principal(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
action="files.blobs.garbage_collected",
|
||||||
|
object_type="file_blob_batch",
|
||||||
|
object_id=principal.tenant_id,
|
||||||
|
details={
|
||||||
|
"inspected_blobs": result.inspected_blobs,
|
||||||
|
"deleted_blobs": result.deleted_blobs,
|
||||||
|
"unresolved_operation_ids": list(result.unresolved_operation_ids),
|
||||||
|
"approval_reference": payload.approval_reference,
|
||||||
|
},
|
||||||
|
commit=True,
|
||||||
|
)
|
||||||
|
return FileBlobGcResponse(
|
||||||
|
inspected_blobs=result.inspected_blobs,
|
||||||
|
deleted_blobs=result.deleted_blobs,
|
||||||
|
unresolved_operation_ids=list(result.unresolved_operation_ids),
|
||||||
|
)
|
||||||
|
except FileStorageError as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _http_error(exc) from exc
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from typing import Literal
|
from typing import Literal
|
||||||
from fastapi import APIRouter, Depends, Query
|
from fastapi import APIRouter, Depends, HTTPException, Query, status
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from govoplan_core.auth import ApiPrincipal, require_scope
|
from govoplan_core.auth import ApiPrincipal, require_scope
|
||||||
@@ -14,6 +14,7 @@ from govoplan_files.backend.storage.files import (
|
|||||||
count_assets_for_user,
|
count_assets_for_user,
|
||||||
list_assets_for_user,
|
list_assets_for_user,
|
||||||
list_assets_for_user_window,
|
list_assets_for_user_window,
|
||||||
|
list_recent_assets_for_user,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -79,6 +80,7 @@ def list_files(
|
|||||||
path_prefix: str | None = None,
|
path_prefix: str | None = None,
|
||||||
campaign_usage: Literal["linked", "unlinked"] | None = None,
|
campaign_usage: Literal["linked", "unlinked"] | None = None,
|
||||||
audit_relevant: bool | None = None,
|
audit_relevant: bool | None = None,
|
||||||
|
sort: Literal["path", "recent"] = "path",
|
||||||
page_size: int | None = Query(default=None, ge=1, le=1000),
|
page_size: int | None = Query(default=None, ge=1, le=1000),
|
||||||
cursor: str | None = None,
|
cursor: str | None = None,
|
||||||
session: Session = Depends(get_session),
|
session: Session = Depends(get_session),
|
||||||
@@ -99,6 +101,31 @@ def list_files(
|
|||||||
audit_relevant=audit_relevant,
|
audit_relevant=audit_relevant,
|
||||||
is_admin=_is_admin(principal),
|
is_admin=_is_admin(principal),
|
||||||
)
|
)
|
||||||
|
if sort == "recent":
|
||||||
|
if cursor:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
|
||||||
|
detail="Recent file projections do not accept a path-order cursor.",
|
||||||
|
)
|
||||||
|
recent_limit = page_size or 25
|
||||||
|
assets = list_recent_assets_for_user(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
limit=recent_limit,
|
||||||
|
owner_type=owner_type,
|
||||||
|
owner_id=owner_id,
|
||||||
|
campaign_id=campaign_id,
|
||||||
|
path_prefix=path_prefix,
|
||||||
|
campaign_usage=campaign_usage,
|
||||||
|
audit_relevant=audit_relevant,
|
||||||
|
is_admin=_is_admin(principal),
|
||||||
|
)
|
||||||
|
return FileListResponse(
|
||||||
|
files=_asset_list_response(session, assets, include_shares=True),
|
||||||
|
total=total,
|
||||||
|
watermark=watermark,
|
||||||
|
)
|
||||||
effective_page_size = _cursor_page_size(FILES_LIST_CURSOR_SCOPE, cursor, page_size)
|
effective_page_size = _cursor_page_size(FILES_LIST_CURSOR_SCOPE, cursor, page_size)
|
||||||
if effective_page_size is not None:
|
if effective_page_size is not None:
|
||||||
fingerprint = _files_list_fingerprint(
|
fingerprint = _files_list_fingerprint(
|
||||||
|
|||||||
@@ -27,6 +27,7 @@ from govoplan_files.backend.storage.connector_spaces import (
|
|||||||
list_connector_spaces_for_user,
|
list_connector_spaces_for_user,
|
||||||
soft_delete_connector_space,
|
soft_delete_connector_space,
|
||||||
update_connector_space,
|
update_connector_space,
|
||||||
|
validate_connector_space_write_mode,
|
||||||
)
|
)
|
||||||
from govoplan_files.backend.storage.connector_policy import (
|
from govoplan_files.backend.storage.connector_policy import (
|
||||||
ConnectorPolicyDenied,
|
ConnectorPolicyDenied,
|
||||||
@@ -145,6 +146,7 @@ def create_file_connector_space(
|
|||||||
)
|
)
|
||||||
if not decision.allowed:
|
if not decision.allowed:
|
||||||
raise ConnectorPolicyDenied(decision)
|
raise ConnectorPolicyDenied(decision)
|
||||||
|
validate_connector_space_write_mode(profile, read_only=payload.read_only)
|
||||||
space = create_connector_space(
|
space = create_connector_space(
|
||||||
session,
|
session,
|
||||||
tenant_id=principal.tenant_id,
|
tenant_id=principal.tenant_id,
|
||||||
@@ -156,6 +158,7 @@ def create_file_connector_space(
|
|||||||
library_id=payload.library_id,
|
library_id=payload.library_id,
|
||||||
remote_path=payload.remote_path,
|
remote_path=payload.remote_path,
|
||||||
sync_mode=payload.sync_mode,
|
sync_mode=payload.sync_mode,
|
||||||
|
read_only=payload.read_only,
|
||||||
metadata=payload.metadata,
|
metadata=payload.metadata,
|
||||||
is_admin=_is_admin(principal),
|
is_admin=_is_admin(principal),
|
||||||
)
|
)
|
||||||
@@ -201,7 +204,11 @@ def update_file_connector_space(
|
|||||||
except FileStorageError as exc:
|
except FileStorageError as exc:
|
||||||
raise _http_error(exc, not_found=True) from exc
|
raise _http_error(exc, not_found=True) from exc
|
||||||
try:
|
try:
|
||||||
if payload.library_id is not None or payload.remote_path is not None:
|
if (
|
||||||
|
payload.library_id is not None
|
||||||
|
or payload.remote_path is not None
|
||||||
|
or payload.read_only is not None
|
||||||
|
):
|
||||||
profile = _visible_connector_profile(
|
profile = _visible_connector_profile(
|
||||||
session, principal, space.connector_profile_id
|
session, principal, space.connector_profile_id
|
||||||
)
|
)
|
||||||
@@ -217,6 +224,14 @@ def update_file_connector_space(
|
|||||||
)
|
)
|
||||||
if not decision.allowed:
|
if not decision.allowed:
|
||||||
raise ConnectorPolicyDenied(decision)
|
raise ConnectorPolicyDenied(decision)
|
||||||
|
validate_connector_space_write_mode(
|
||||||
|
profile,
|
||||||
|
read_only=(
|
||||||
|
payload.read_only
|
||||||
|
if payload.read_only is not None
|
||||||
|
else space.read_only
|
||||||
|
),
|
||||||
|
)
|
||||||
update_connector_space(
|
update_connector_space(
|
||||||
session,
|
session,
|
||||||
space,
|
space,
|
||||||
@@ -225,6 +240,7 @@ def update_file_connector_space(
|
|||||||
library_id=payload.library_id,
|
library_id=payload.library_id,
|
||||||
remote_path=payload.remote_path,
|
remote_path=payload.remote_path,
|
||||||
sync_mode=payload.sync_mode,
|
sync_mode=payload.sync_mode,
|
||||||
|
read_only=payload.read_only,
|
||||||
is_active=payload.is_active,
|
is_active=payload.is_active,
|
||||||
metadata=payload.metadata,
|
metadata=payload.metadata,
|
||||||
is_admin=_is_admin(principal),
|
is_admin=_is_admin(principal),
|
||||||
|
|||||||
@@ -37,6 +37,7 @@ class FileConnectorSpaceCreateRequest(BaseModel):
|
|||||||
library_id: str | None = None
|
library_id: str | None = None
|
||||||
remote_path: str = ""
|
remote_path: str = ""
|
||||||
sync_mode: Literal["manual"] = "manual"
|
sync_mode: Literal["manual"] = "manual"
|
||||||
|
read_only: bool = True
|
||||||
metadata: dict[str, Any] = Field(default_factory=dict)
|
metadata: dict[str, Any] = Field(default_factory=dict)
|
||||||
|
|
||||||
|
|
||||||
@@ -45,6 +46,7 @@ class FileConnectorSpaceUpdateRequest(BaseModel):
|
|||||||
library_id: str | None = None
|
library_id: str | None = None
|
||||||
remote_path: str | None = None
|
remote_path: str | None = None
|
||||||
sync_mode: Literal["manual"] | None = None
|
sync_mode: Literal["manual"] | None = None
|
||||||
|
read_only: bool | None = None
|
||||||
is_active: bool | None = None
|
is_active: bool | None = None
|
||||||
metadata: dict[str, Any] | None = None
|
metadata: dict[str, Any] | None = None
|
||||||
|
|
||||||
@@ -101,6 +103,7 @@ class FileIntegrityScanResponse(BaseModel):
|
|||||||
storage_backend: str
|
storage_backend: str
|
||||||
storage_prefix: str
|
storage_prefix: str
|
||||||
status: str
|
status: str
|
||||||
|
revision: int
|
||||||
phase: str
|
phase: str
|
||||||
verify_checksums: bool
|
verify_checksums: bool
|
||||||
batch_size: int
|
batch_size: int
|
||||||
@@ -127,6 +130,7 @@ class FileIntegrityFindingResponse(BaseModel):
|
|||||||
tenant_id: str
|
tenant_id: str
|
||||||
kind: str
|
kind: str
|
||||||
state: str
|
state: str
|
||||||
|
revision: int
|
||||||
blob_id: str | None = None
|
blob_id: str | None = None
|
||||||
storage_key: str
|
storage_key: str
|
||||||
expected_size_bytes: int | None = None
|
expected_size_bytes: int | None = None
|
||||||
@@ -145,6 +149,11 @@ class FileIntegrityFindingsResponse(BaseModel):
|
|||||||
|
|
||||||
class FileIntegrityActionRequest(BaseModel):
|
class FileIntegrityActionRequest(BaseModel):
|
||||||
dry_run: bool = True
|
dry_run: bool = True
|
||||||
|
expected_revision: int = Field(ge=1)
|
||||||
|
|
||||||
|
|
||||||
|
class FileIntegrityScanRunRequest(BaseModel):
|
||||||
|
expected_revision: int = Field(ge=1)
|
||||||
|
|
||||||
|
|
||||||
class FileIntegrityActionResponse(BaseModel):
|
class FileIntegrityActionResponse(BaseModel):
|
||||||
@@ -185,6 +194,10 @@ class FileAssetResponse(BaseModel):
|
|||||||
created_at: str
|
created_at: str
|
||||||
updated_at: str
|
updated_at: str
|
||||||
deleted_at: str | None = None
|
deleted_at: str | None = None
|
||||||
|
retained_until: str | None = None
|
||||||
|
legal_hold: bool = False
|
||||||
|
lifecycle_revision: int = 1
|
||||||
|
lifecycle_reason: str | None = None
|
||||||
audit_relevant: bool = False
|
audit_relevant: bool = False
|
||||||
metadata: dict[str, Any] | None = None
|
metadata: dict[str, Any] | None = None
|
||||||
source_provenance: FileSourceProvenance | None = None
|
source_provenance: FileSourceProvenance | None = None
|
||||||
@@ -470,6 +483,7 @@ class FileConnectorProviderResponse(BaseModel):
|
|||||||
installed: bool
|
installed: bool
|
||||||
browse_supported: bool
|
browse_supported: bool
|
||||||
import_supported: bool
|
import_supported: bool
|
||||||
|
write_supported: bool = False
|
||||||
optional_dependency: str | None = None
|
optional_dependency: str | None = None
|
||||||
permission_model: str
|
permission_model: str
|
||||||
sync_strategy: str
|
sync_strategy: str
|
||||||
@@ -528,6 +542,24 @@ class FileConnectorSyncResponse(BaseModel):
|
|||||||
current_version_id: str
|
current_version_id: str
|
||||||
|
|
||||||
|
|
||||||
|
class FileConnectorWriteRequest(BaseModel):
|
||||||
|
file_id: str
|
||||||
|
remote_path: str
|
||||||
|
idempotency_key: str = Field(min_length=8, max_length=160)
|
||||||
|
expected_revision: str | None = Field(default=None, max_length=500)
|
||||||
|
|
||||||
|
|
||||||
|
class FileConnectorWriteResponse(BaseModel):
|
||||||
|
recovery_operation_id: str
|
||||||
|
status: str
|
||||||
|
replayed: bool = False
|
||||||
|
provider: str
|
||||||
|
remote_path: str
|
||||||
|
revision: str | None = None
|
||||||
|
checksum_sha256: str
|
||||||
|
size_bytes: int
|
||||||
|
|
||||||
|
|
||||||
class BulkDeleteRequest(BaseModel):
|
class BulkDeleteRequest(BaseModel):
|
||||||
file_ids: list[str]
|
file_ids: list[str]
|
||||||
|
|
||||||
@@ -536,6 +568,69 @@ class BulkDeleteResponse(BaseModel):
|
|||||||
deleted_count: int
|
deleted_count: int
|
||||||
|
|
||||||
|
|
||||||
|
class FileLifecycleUpdateRequest(BaseModel):
|
||||||
|
retained_until: datetime | None = None
|
||||||
|
legal_hold: bool = False
|
||||||
|
reason: str = Field(min_length=1, max_length=500)
|
||||||
|
expected_revision: int = Field(ge=1)
|
||||||
|
|
||||||
|
|
||||||
|
class FileRestoreResponse(BaseModel):
|
||||||
|
restored_count: int
|
||||||
|
|
||||||
|
|
||||||
|
class FileFolderRestoreResponse(BaseModel):
|
||||||
|
restored_folders: int
|
||||||
|
restored_files: int
|
||||||
|
|
||||||
|
|
||||||
|
class FilePurgePreviewRequest(BaseModel):
|
||||||
|
file_ids: list[str] = Field(min_length=1, max_length=100)
|
||||||
|
|
||||||
|
|
||||||
|
class FilePurgePreviewItem(BaseModel):
|
||||||
|
file_id: str
|
||||||
|
filename: str
|
||||||
|
lifecycle_revision: int
|
||||||
|
deleted_at: str | None = None
|
||||||
|
retained_until: str | None = None
|
||||||
|
legal_hold: bool = False
|
||||||
|
blockers: list[str] = Field(default_factory=list)
|
||||||
|
blob_ids: list[str] = Field(default_factory=list)
|
||||||
|
|
||||||
|
|
||||||
|
class FilePurgePreviewResponse(BaseModel):
|
||||||
|
preview_sha256: str
|
||||||
|
eligible: bool
|
||||||
|
items: list[FilePurgePreviewItem]
|
||||||
|
|
||||||
|
|
||||||
|
class FilePurgeExecuteRequest(FilePurgePreviewRequest):
|
||||||
|
preview_sha256: str = Field(min_length=64, max_length=64)
|
||||||
|
idempotency_key: str = Field(min_length=8, max_length=160)
|
||||||
|
approval_reference: str = Field(min_length=3, max_length=1000)
|
||||||
|
confirmation: Literal["PURGE"]
|
||||||
|
|
||||||
|
|
||||||
|
class FilePurgeResponse(BaseModel):
|
||||||
|
recovery_operation_id: str
|
||||||
|
status: str
|
||||||
|
replayed: bool = False
|
||||||
|
purged_files: int = 0
|
||||||
|
released_blobs: int = 0
|
||||||
|
|
||||||
|
|
||||||
|
class FileBlobGcRequest(BaseModel):
|
||||||
|
limit: int = Field(default=25, ge=1, le=100)
|
||||||
|
approval_reference: str = Field(min_length=3, max_length=1000)
|
||||||
|
|
||||||
|
|
||||||
|
class FileBlobGcResponse(BaseModel):
|
||||||
|
inspected_blobs: int
|
||||||
|
deleted_blobs: int
|
||||||
|
unresolved_operation_ids: list[str] = Field(default_factory=list)
|
||||||
|
|
||||||
|
|
||||||
class ConflictResolutionRequest(BaseModel):
|
class ConflictResolutionRequest(BaseModel):
|
||||||
target_path: str
|
target_path: str
|
||||||
action: Literal["overwrite", "rename", "skip"]
|
action: Literal["overwrite", "rename", "skip"]
|
||||||
|
|||||||
@@ -0,0 +1,329 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from collections.abc import Mapping, Sequence
|
||||||
|
from urllib.parse import quote
|
||||||
|
|
||||||
|
from sqlalchemy import func, or_, select
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from govoplan_core.auth import ApiPrincipal
|
||||||
|
from govoplan_core.core.events import PlatformEvent
|
||||||
|
from govoplan_core.core.modules import ModuleContext
|
||||||
|
from govoplan_core.core.search import (
|
||||||
|
SearchAuthorizationRequest,
|
||||||
|
SearchBackfillPage,
|
||||||
|
SearchBackfillRequest,
|
||||||
|
SearchDocument,
|
||||||
|
SearchIndexChange,
|
||||||
|
SearchResourceReference,
|
||||||
|
SearchResourceType,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.db.models import FileAsset, FileFolder, FileShare
|
||||||
|
from govoplan_files.backend.storage.share_state import effective_file_share_clause
|
||||||
|
|
||||||
|
|
||||||
|
PROVIDER_ID = "files.objects"
|
||||||
|
RESOURCE_MODELS = {
|
||||||
|
"file": FileAsset,
|
||||||
|
"folder": FileFolder,
|
||||||
|
}
|
||||||
|
READ_SCOPE = "files:file:read"
|
||||||
|
ADMIN_SCOPE = "files:file:admin"
|
||||||
|
|
||||||
|
|
||||||
|
class FilesSearchSource:
|
||||||
|
def resource_types(self) -> Sequence[SearchResourceType]:
|
||||||
|
return (
|
||||||
|
SearchResourceType(
|
||||||
|
provider_id=PROVIDER_ID,
|
||||||
|
module_id="files",
|
||||||
|
resource_type="file",
|
||||||
|
label="Files",
|
||||||
|
requires_authorization_recheck=True,
|
||||||
|
),
|
||||||
|
SearchResourceType(
|
||||||
|
provider_id=PROVIDER_ID,
|
||||||
|
module_id="files",
|
||||||
|
resource_type="folder",
|
||||||
|
label="File folders",
|
||||||
|
requires_authorization_recheck=True,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
def backfill(
|
||||||
|
self,
|
||||||
|
session: object,
|
||||||
|
*,
|
||||||
|
request: SearchBackfillRequest,
|
||||||
|
) -> SearchBackfillPage:
|
||||||
|
db = _session(session)
|
||||||
|
model = _model(request.provider_id, request.resource_type)
|
||||||
|
statement = select(model).where(
|
||||||
|
model.tenant_id == request.tenant_id,
|
||||||
|
model.deleted_at.is_(None),
|
||||||
|
)
|
||||||
|
if request.cursor:
|
||||||
|
statement = statement.where(model.id > request.cursor)
|
||||||
|
rows = list(
|
||||||
|
db.scalars(
|
||||||
|
statement.order_by(model.id).limit(request.limit + 1)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
has_more = len(rows) > request.limit
|
||||||
|
selected = rows[: request.limit]
|
||||||
|
shares = (
|
||||||
|
_shares_by_asset(db, selected)
|
||||||
|
if request.resource_type == "file"
|
||||||
|
else {}
|
||||||
|
)
|
||||||
|
high_watermark = db.scalar(
|
||||||
|
select(func.max(model.updated_at)).where(
|
||||||
|
model.tenant_id == request.tenant_id,
|
||||||
|
model.deleted_at.is_(None),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return SearchBackfillPage(
|
||||||
|
documents=tuple(
|
||||||
|
_document(
|
||||||
|
row,
|
||||||
|
resource_type=request.resource_type,
|
||||||
|
shares=shares.get(row.id, ()),
|
||||||
|
)
|
||||||
|
for row in selected
|
||||||
|
),
|
||||||
|
next_cursor=selected[-1].id if has_more and selected else None,
|
||||||
|
complete=not has_more,
|
||||||
|
high_watermark=(
|
||||||
|
high_watermark.isoformat()
|
||||||
|
if high_watermark is not None
|
||||||
|
else None
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
def authorize(
|
||||||
|
self,
|
||||||
|
session: object,
|
||||||
|
principal: object,
|
||||||
|
*,
|
||||||
|
requests: Sequence[SearchAuthorizationRequest],
|
||||||
|
) -> Mapping[str, bool]:
|
||||||
|
decisions = {item.reference.key: False for item in requests}
|
||||||
|
if not isinstance(principal, ApiPrincipal) or not (
|
||||||
|
principal.has(READ_SCOPE) or principal.has(ADMIN_SCOPE)
|
||||||
|
):
|
||||||
|
return decisions
|
||||||
|
db = _session(session)
|
||||||
|
for request in requests:
|
||||||
|
reference = request.reference
|
||||||
|
if (
|
||||||
|
reference.tenant_id != principal.tenant_id
|
||||||
|
or reference.module_id != "files"
|
||||||
|
or reference.resource_type not in RESOURCE_MODELS
|
||||||
|
):
|
||||||
|
continue
|
||||||
|
decisions[reference.key] = _can_read(
|
||||||
|
db,
|
||||||
|
principal,
|
||||||
|
reference=reference,
|
||||||
|
)
|
||||||
|
return decisions
|
||||||
|
|
||||||
|
def index_changes_for_event(
|
||||||
|
self,
|
||||||
|
session: object,
|
||||||
|
*,
|
||||||
|
event: PlatformEvent,
|
||||||
|
delivery_key: str,
|
||||||
|
) -> Sequence[SearchIndexChange]:
|
||||||
|
if (
|
||||||
|
event.module_id != "files"
|
||||||
|
or event.tenant is None
|
||||||
|
or event.resource is None
|
||||||
|
or event.resource.id is None
|
||||||
|
or event.resource.type not in RESOURCE_MODELS
|
||||||
|
):
|
||||||
|
return ()
|
||||||
|
db = _session(session)
|
||||||
|
reference = SearchResourceReference(
|
||||||
|
tenant_id=event.tenant.id,
|
||||||
|
module_id="files",
|
||||||
|
resource_type=event.resource.type,
|
||||||
|
resource_id=event.resource.id,
|
||||||
|
)
|
||||||
|
model = RESOURCE_MODELS[event.resource.type]
|
||||||
|
row = db.get(model, event.resource.id)
|
||||||
|
deleted = row is None or row.tenant_id != event.tenant.id or row.deleted_at is not None
|
||||||
|
cursor = event.event_id
|
||||||
|
document = None
|
||||||
|
if not deleted:
|
||||||
|
shares = (
|
||||||
|
tuple(_active_shares(db, row.id))
|
||||||
|
if event.resource.type == "file"
|
||||||
|
else ()
|
||||||
|
)
|
||||||
|
document = _document(
|
||||||
|
row,
|
||||||
|
resource_type=event.resource.type,
|
||||||
|
shares=shares,
|
||||||
|
change_cursor=cursor,
|
||||||
|
)
|
||||||
|
return (
|
||||||
|
SearchIndexChange(
|
||||||
|
change_id=f"{delivery_key}:{PROVIDER_ID}:{event.resource.type}",
|
||||||
|
provider_id=PROVIDER_ID,
|
||||||
|
kind="delete" if deleted else "upsert",
|
||||||
|
reference=reference,
|
||||||
|
source_revision=(
|
||||||
|
document.source_revision if document is not None else cursor
|
||||||
|
),
|
||||||
|
cursor=cursor,
|
||||||
|
document=document,
|
||||||
|
occurred_at=event.occurred_at,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def create_files_search_source(_context: ModuleContext) -> FilesSearchSource:
|
||||||
|
return FilesSearchSource()
|
||||||
|
|
||||||
|
|
||||||
|
def _model(provider_id: str, resource_type: str):
|
||||||
|
if provider_id != PROVIDER_ID or resource_type not in RESOURCE_MODELS:
|
||||||
|
raise ValueError("Unsupported Files search source.")
|
||||||
|
return RESOURCE_MODELS[resource_type]
|
||||||
|
|
||||||
|
|
||||||
|
def _document(
|
||||||
|
row: FileAsset | FileFolder,
|
||||||
|
*,
|
||||||
|
resource_type: str,
|
||||||
|
shares: Sequence[FileShare] = (),
|
||||||
|
change_cursor: str | None = None,
|
||||||
|
) -> SearchDocument:
|
||||||
|
is_file = isinstance(row, FileAsset)
|
||||||
|
title = row.filename if is_file else (row.path.rsplit("/", 1)[-1] or row.path)
|
||||||
|
path = row.display_path if is_file else row.path
|
||||||
|
owner_id = row.owner_user_id if row.owner_type == "user" else row.owner_group_id
|
||||||
|
tokens = [f"scope:{READ_SCOPE}", f"scope:{ADMIN_SCOPE}"]
|
||||||
|
if owner_id:
|
||||||
|
tokens.append(
|
||||||
|
f"membership:{owner_id}"
|
||||||
|
if row.owner_type == "user"
|
||||||
|
else f"group:{owner_id}"
|
||||||
|
)
|
||||||
|
for share in shares:
|
||||||
|
prefix = "membership" if share.target_type == "user" else share.target_type
|
||||||
|
if prefix in {"membership", "group", "tenant"}:
|
||||||
|
tokens.append(f"{prefix}:{share.target_id}")
|
||||||
|
updated_at = row.updated_at or row.created_at
|
||||||
|
revision = (
|
||||||
|
f"{row.current_version_id or 'none'}:{updated_at.isoformat()}"
|
||||||
|
if is_file
|
||||||
|
else updated_at.isoformat()
|
||||||
|
)
|
||||||
|
return SearchDocument(
|
||||||
|
tenant_id=row.tenant_id,
|
||||||
|
module_id="files",
|
||||||
|
provider_id=PROVIDER_ID,
|
||||||
|
resource_type=resource_type,
|
||||||
|
resource_id=row.id,
|
||||||
|
title=title,
|
||||||
|
url=f"/files?{resource_type}Id={quote(row.id, safe='')}",
|
||||||
|
summary=((row.description or "") if is_file else path)[:4000] or None,
|
||||||
|
body=" ".join(
|
||||||
|
value for value in (path, row.description if is_file else None) if value
|
||||||
|
)[:200_000],
|
||||||
|
keywords=(path[:200], row.owner_type[:200]),
|
||||||
|
visibility="restricted",
|
||||||
|
acl_tokens=tuple(dict.fromkeys(tokens)),
|
||||||
|
metadata={
|
||||||
|
"path": path,
|
||||||
|
"owner_type": row.owner_type,
|
||||||
|
"current_version_id": row.current_version_id if is_file else None,
|
||||||
|
},
|
||||||
|
source_revision=revision,
|
||||||
|
change_cursor=change_cursor,
|
||||||
|
source_updated_at=updated_at,
|
||||||
|
requires_authorization_recheck=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _can_read(
|
||||||
|
session: Session,
|
||||||
|
principal: ApiPrincipal,
|
||||||
|
*,
|
||||||
|
reference: SearchResourceReference,
|
||||||
|
) -> bool:
|
||||||
|
model = RESOURCE_MODELS[reference.resource_type]
|
||||||
|
row = session.get(model, reference.resource_id)
|
||||||
|
if row is None or row.tenant_id != principal.tenant_id or row.deleted_at is not None:
|
||||||
|
return False
|
||||||
|
if principal.has(ADMIN_SCOPE):
|
||||||
|
return True
|
||||||
|
user_id = str(getattr(principal.user, "id", "") or principal.membership_id or "")
|
||||||
|
if row.owner_type == "user" and row.owner_user_id == user_id:
|
||||||
|
return True
|
||||||
|
if row.owner_type == "group" and row.owner_group_id in principal.group_ids:
|
||||||
|
return True
|
||||||
|
if reference.resource_type != "file":
|
||||||
|
return False
|
||||||
|
target_clauses = [
|
||||||
|
(FileShare.target_type == "user") & (FileShare.target_id == user_id),
|
||||||
|
(FileShare.target_type == "tenant")
|
||||||
|
& (FileShare.target_id == principal.tenant_id),
|
||||||
|
]
|
||||||
|
if principal.group_ids:
|
||||||
|
target_clauses.append(
|
||||||
|
(FileShare.target_type == "group")
|
||||||
|
& (FileShare.target_id.in_(tuple(principal.group_ids)))
|
||||||
|
)
|
||||||
|
return session.scalar(
|
||||||
|
select(FileShare.id).where(
|
||||||
|
FileShare.tenant_id == principal.tenant_id,
|
||||||
|
FileShare.file_asset_id == row.id,
|
||||||
|
effective_file_share_clause(),
|
||||||
|
or_(*target_clauses),
|
||||||
|
).limit(1)
|
||||||
|
) is not None
|
||||||
|
|
||||||
|
|
||||||
|
def _active_shares(session: Session, asset_id: str) -> Sequence[FileShare]:
|
||||||
|
return tuple(
|
||||||
|
session.scalars(
|
||||||
|
select(FileShare).where(
|
||||||
|
FileShare.file_asset_id == asset_id,
|
||||||
|
effective_file_share_clause(),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _shares_by_asset(
|
||||||
|
session: Session,
|
||||||
|
rows: Sequence[FileAsset | FileFolder],
|
||||||
|
) -> dict[str, tuple[FileShare, ...]]:
|
||||||
|
asset_ids = [row.id for row in rows if isinstance(row, FileAsset)]
|
||||||
|
grouped: dict[str, list[FileShare]] = {asset_id: [] for asset_id in asset_ids}
|
||||||
|
if not asset_ids:
|
||||||
|
return {}
|
||||||
|
for share in session.scalars(
|
||||||
|
select(FileShare).where(
|
||||||
|
FileShare.file_asset_id.in_(asset_ids),
|
||||||
|
effective_file_share_clause(),
|
||||||
|
)
|
||||||
|
):
|
||||||
|
grouped.setdefault(share.file_asset_id, []).append(share)
|
||||||
|
return {key: tuple(value) for key, value in grouped.items()}
|
||||||
|
|
||||||
|
|
||||||
|
def _session(value: object) -> Session:
|
||||||
|
if not isinstance(value, Session):
|
||||||
|
raise TypeError("Files search requires a SQLAlchemy session.")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"FilesSearchSource",
|
||||||
|
"PROVIDER_ID",
|
||||||
|
"create_files_search_source",
|
||||||
|
]
|
||||||
@@ -15,8 +15,8 @@ from defusedxml import ElementTree as SafeElementTree
|
|||||||
|
|
||||||
from govoplan_core.security.outbound_http import (
|
from govoplan_core.security.outbound_http import (
|
||||||
OutboundHttpError,
|
OutboundHttpError,
|
||||||
validate_unpinned_sdk_host,
|
validate_outbound_host,
|
||||||
validate_unpinned_sdk_http_url,
|
validate_outbound_http_url,
|
||||||
)
|
)
|
||||||
|
|
||||||
from govoplan_files.backend.storage.http_client import ConnectorHttpError, request_connector_bytes
|
from govoplan_files.backend.storage.http_client import ConnectorHttpError, request_connector_bytes
|
||||||
@@ -27,6 +27,12 @@ from govoplan_files.backend.storage.connector_deployment import (
|
|||||||
validate_connector_tls_metadata,
|
validate_connector_tls_metadata,
|
||||||
)
|
)
|
||||||
from govoplan_files.backend.storage.connector_profiles import ConnectorProfile
|
from govoplan_files.backend.storage.connector_profiles import ConnectorProfile
|
||||||
|
from govoplan_files.backend.storage.sdk_peer_pinning import (
|
||||||
|
SdkPeerPinningError,
|
||||||
|
create_pinned_s3_client,
|
||||||
|
install_pinned_smb_transport,
|
||||||
|
pinned_smb_connection_cache,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class ConnectorBrowseError(RuntimeError):
|
class ConnectorBrowseError(RuntimeError):
|
||||||
@@ -246,6 +252,28 @@ def _browse_smb(profile: ConnectorProfile, *, path: str) -> list[ConnectorBrowse
|
|||||||
|
|
||||||
def _browse_s3(profile: ConnectorProfile, *, path: str, library_id: str | None, continuation_token: str | None) -> list[ConnectorBrowseItem]:
|
def _browse_s3(profile: ConnectorProfile, *, path: str, library_id: str | None, continuation_token: str | None) -> list[ConnectorBrowseItem]:
|
||||||
client = _s3_client(profile)
|
client = _s3_client(profile)
|
||||||
|
try:
|
||||||
|
return _browse_s3_with_client(
|
||||||
|
client,
|
||||||
|
profile=profile,
|
||||||
|
path=path,
|
||||||
|
library_id=library_id,
|
||||||
|
continuation_token=continuation_token,
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
close = getattr(client, "close", None)
|
||||||
|
if callable(close):
|
||||||
|
close()
|
||||||
|
|
||||||
|
|
||||||
|
def _browse_s3_with_client(
|
||||||
|
client: Any,
|
||||||
|
*,
|
||||||
|
profile: ConnectorProfile,
|
||||||
|
path: str,
|
||||||
|
library_id: str | None,
|
||||||
|
continuation_token: str | None,
|
||||||
|
) -> list[ConnectorBrowseItem]:
|
||||||
bucket = _s3_bucket(profile, library_id)
|
bucket = _s3_bucket(profile, library_id)
|
||||||
if not bucket:
|
if not bucket:
|
||||||
try:
|
try:
|
||||||
@@ -313,23 +341,22 @@ def _s3_client(profile: ConnectorProfile) -> Any:
|
|||||||
raise ConnectorBrowseError("Secret-ref S3 credentials need a runtime secret resolver before live browsing")
|
raise ConnectorBrowseError("Secret-ref S3 credentials need a runtime secret resolver before live browsing")
|
||||||
if profile.endpoint_url:
|
if profile.endpoint_url:
|
||||||
try:
|
try:
|
||||||
endpoint_url = validate_unpinned_sdk_http_url(
|
endpoint_url = validate_outbound_http_url(
|
||||||
profile.endpoint_url,
|
profile.endpoint_url,
|
||||||
label="S3 connector endpoint",
|
label="S3 connector endpoint",
|
||||||
)
|
)
|
||||||
except OutboundHttpError as exc:
|
except OutboundHttpError as exc:
|
||||||
raise ConnectorBrowseError(str(exc)) from exc
|
raise ConnectorBrowseError(str(exc)) from exc
|
||||||
else:
|
else:
|
||||||
raise ConnectorBrowseError(
|
endpoint_url = None
|
||||||
"S3 connector endpoint discovery uses an SDK transport that cannot guarantee connection-time DNS/IP "
|
|
||||||
"pinning; live S3 access is disabled until that transport supports pinning"
|
|
||||||
)
|
|
||||||
try:
|
try:
|
||||||
boto3 = import_module("boto3")
|
|
||||||
config_module = import_module("botocore.config")
|
config_module = import_module("botocore.config")
|
||||||
|
unsigned = import_module("botocore").UNSIGNED
|
||||||
except ImportError as exc:
|
except ImportError as exc:
|
||||||
raise ConnectorBrowseUnsupported("S3 connector browsing requires the optional boto3 dependency") from exc
|
raise ConnectorBrowseUnsupported("S3 connector browsing requires the optional boto3 dependency") from exc
|
||||||
kwargs: dict[str, object] = {"endpoint_url": endpoint_url}
|
kwargs: dict[str, object] = {}
|
||||||
|
if endpoint_url:
|
||||||
|
kwargs["endpoint_url"] = endpoint_url
|
||||||
region = _metadata_string(profile, "region") or _metadata_string(profile, "aws_region")
|
region = _metadata_string(profile, "region") or _metadata_string(profile, "aws_region")
|
||||||
if region:
|
if region:
|
||||||
kwargs["region_name"] = region
|
kwargs["region_name"] = region
|
||||||
@@ -346,10 +373,21 @@ def _s3_client(profile: ConnectorProfile) -> Any:
|
|||||||
if verify is not None:
|
if verify is not None:
|
||||||
kwargs["verify"] = verify
|
kwargs["verify"] = verify
|
||||||
addressing_style = _s3_addressing_style(profile)
|
addressing_style = _s3_addressing_style(profile)
|
||||||
|
config_values: dict[str, object] = {
|
||||||
|
"proxies": {},
|
||||||
|
"retries": {"mode": "standard", "max_attempts": 4},
|
||||||
|
}
|
||||||
if addressing_style:
|
if addressing_style:
|
||||||
kwargs["config"] = config_module.Config(s3={"addressing_style": addressing_style})
|
config_values["s3"] = {"addressing_style": addressing_style}
|
||||||
|
if bool(access_key) != bool(secret_key):
|
||||||
|
raise ConnectorBrowseError("S3 connectors require both an access key and a secret key")
|
||||||
|
if not access_key:
|
||||||
|
config_values["signature_version"] = unsigned
|
||||||
|
kwargs["config"] = config_module.Config(**config_values)
|
||||||
try:
|
try:
|
||||||
return boto3.client("s3", **kwargs)
|
return create_pinned_s3_client(**kwargs)
|
||||||
|
except SdkPeerPinningError as exc:
|
||||||
|
raise ConnectorBrowseError(str(exc)) from exc
|
||||||
except Exception as exc: # pragma: no cover - concrete exception types are dependency-version specific
|
except Exception as exc: # pragma: no cover - concrete exception types are dependency-version specific
|
||||||
raise ConnectorBrowseError(f"S3 connector could not be initialized: {exc}") from exc
|
raise ConnectorBrowseError(f"S3 connector could not be initialized: {exc}") from exc
|
||||||
|
|
||||||
@@ -783,7 +821,7 @@ def _smb_location(profile: ConnectorProfile) -> _SmbLocation:
|
|||||||
raise ConnectorBrowseError("SMB connector endpoint_url must include a server")
|
raise ConnectorBrowseError("SMB connector endpoint_url must include a server")
|
||||||
port = parsed.port or _int(profile.metadata.get("port")) or 445
|
port = parsed.port or _int(profile.metadata.get("port")) or 445
|
||||||
try:
|
try:
|
||||||
validate_unpinned_sdk_host(server, port=port, label="SMB connector endpoint")
|
validate_outbound_host(server, port=port, label="SMB connector endpoint")
|
||||||
except OutboundHttpError as exc:
|
except OutboundHttpError as exc:
|
||||||
raise ConnectorBrowseError(str(exc)) from exc
|
raise ConnectorBrowseError(str(exc)) from exc
|
||||||
path_parts = [part for part in unquote(parsed.path or "").strip("/").split("/") if part]
|
path_parts = [part for part in unquote(parsed.path or "").strip("/").split("/") if part]
|
||||||
@@ -810,6 +848,7 @@ def _smb_unc_path(location: _SmbLocation, path: str) -> str:
|
|||||||
def _smb_client_kwargs(profile: ConnectorProfile, location: _SmbLocation) -> dict[str, object]:
|
def _smb_client_kwargs(profile: ConnectorProfile, location: _SmbLocation) -> dict[str, object]:
|
||||||
kwargs: dict[str, object] = {
|
kwargs: dict[str, object] = {
|
||||||
"port": location.port,
|
"port": location.port,
|
||||||
|
"connection_cache": pinned_smb_connection_cache(),
|
||||||
"require_signing": _metadata_bool(profile, "require_signing", default=True),
|
"require_signing": _metadata_bool(profile, "require_signing", default=True),
|
||||||
"auth_protocol": _metadata_string(profile, "auth_protocol") or "ntlm",
|
"auth_protocol": _metadata_string(profile, "auth_protocol") or "ntlm",
|
||||||
}
|
}
|
||||||
@@ -845,9 +884,11 @@ def _profile_token(profile: ConnectorProfile) -> str | None:
|
|||||||
|
|
||||||
def _smbclient_module() -> Any:
|
def _smbclient_module() -> Any:
|
||||||
try:
|
try:
|
||||||
return import_module("smbclient")
|
return install_pinned_smb_transport(import_module("smbclient"))
|
||||||
except ImportError as exc:
|
except ImportError as exc:
|
||||||
raise ConnectorBrowseUnsupported("SMB connector browsing requires the optional smbprotocol dependency") from exc
|
raise ConnectorBrowseUnsupported("SMB connector browsing requires the optional smbprotocol dependency") from exc
|
||||||
|
except SdkPeerPinningError as exc:
|
||||||
|
raise ConnectorBrowseUnsupported(str(exc)) from exc
|
||||||
|
|
||||||
|
|
||||||
def _smb_entry_stat(entry: object) -> object | None:
|
def _smb_entry_stat(entry: object) -> object | None:
|
||||||
|
|||||||
@@ -314,6 +314,7 @@ def _read_s3_file(profile: ConnectorProfile, *, library_id: str, path: str, max_
|
|||||||
if not key:
|
if not key:
|
||||||
raise ConnectorImportError("S3 import requires an object key")
|
raise ConnectorImportError("S3 import requires an object key")
|
||||||
client = _s3_import_client(profile)
|
client = _s3_import_client(profile)
|
||||||
|
try:
|
||||||
detail = _s3_object_detail(client, bucket=bucket, key=key, max_bytes=max_bytes)
|
detail = _s3_object_detail(client, bucket=bucket, key=key, max_bytes=max_bytes)
|
||||||
version_id = _clean(detail.get("VersionId"))
|
version_id = _clean(detail.get("VersionId"))
|
||||||
response, data = _download_s3_object(
|
response, data = _download_s3_object(
|
||||||
@@ -323,6 +324,10 @@ def _read_s3_file(profile: ConnectorProfile, *, library_id: str, path: str, max_
|
|||||||
version_id=version_id,
|
version_id=version_id,
|
||||||
max_bytes=max_bytes,
|
max_bytes=max_bytes,
|
||||||
)
|
)
|
||||||
|
finally:
|
||||||
|
close = getattr(client, "close", None)
|
||||||
|
if callable(close):
|
||||||
|
close()
|
||||||
content_type = _clean(response.get("ContentType") if isinstance(response, dict) else None) or _clean(detail.get("ContentType")) or mimetypes.guess_type(key)[0]
|
content_type = _clean(response.get("ContentType") if isinstance(response, dict) else None) or _clean(detail.get("ContentType")) or mimetypes.guess_type(key)[0]
|
||||||
etag = _clean(response.get("ETag") if isinstance(response, dict) else None) or _clean(detail.get("ETag"))
|
etag = _clean(response.get("ETag") if isinstance(response, dict) else None) or _clean(detail.get("ETag"))
|
||||||
filename = filename_from_path(key)
|
filename = filename_from_path(key)
|
||||||
|
|||||||
@@ -32,6 +32,7 @@ class ConnectorProviderDescriptor:
|
|||||||
"installed": self.installed,
|
"installed": self.installed,
|
||||||
"browse_supported": self.browse_supported,
|
"browse_supported": self.browse_supported,
|
||||||
"import_supported": self.import_supported,
|
"import_supported": self.import_supported,
|
||||||
|
"write_supported": self.provider == "s3",
|
||||||
"optional_dependency": self.optional_dependency,
|
"optional_dependency": self.optional_dependency,
|
||||||
"permission_model": self.permission_model,
|
"permission_model": self.permission_model,
|
||||||
"sync_strategy": self.sync_strategy,
|
"sync_strategy": self.sync_strategy,
|
||||||
@@ -110,7 +111,7 @@ def connector_provider_descriptors() -> tuple[ConnectorProviderDescriptor, ...]:
|
|||||||
conflict_strategy="Managed import/sync uses existing files conflict_strategy handling after download.",
|
conflict_strategy="Managed import/sync uses existing files conflict_strategy handling after download.",
|
||||||
preview_strategy="Previews are generated from the frozen managed file after import or sync, not directly from the share.",
|
preview_strategy="Previews are generated from the frozen managed file after import or sync, not directly from the share.",
|
||||||
audit_events=("files.connector.imported", "files.connector.synced", "files.connector.accessed"),
|
audit_events=("files.connector.imported", "files.connector.synced", "files.connector.accessed"),
|
||||||
notes="Browse/import logic is implemented, but live smbprotocol access fails closed until the SDK supports connection-time DNS/IP pinning for initial connections and DFS referrals.",
|
notes="Initial sessions, reconnects, aliases, and DFS referral targets use the Files-owned pinned smbprotocol transport and the deployment-wide private-network policy.",
|
||||||
),
|
),
|
||||||
ConnectorProviderDescriptor(
|
ConnectorProviderDescriptor(
|
||||||
provider="s3",
|
provider="s3",
|
||||||
@@ -125,7 +126,7 @@ def connector_provider_descriptors() -> tuple[ConnectorProviderDescriptor, ...]:
|
|||||||
conflict_strategy="Managed import/sync uses existing files conflict_strategy handling after object download.",
|
conflict_strategy="Managed import/sync uses existing files conflict_strategy handling after object download.",
|
||||||
preview_strategy="Previews are generated from the frozen managed file after import or sync, not directly from the bucket.",
|
preview_strategy="Previews are generated from the frozen managed file after import or sync, not directly from the bucket.",
|
||||||
audit_events=("files.connector.imported", "files.connector.synced", "files.connector.accessed"),
|
audit_events=("files.connector.imported", "files.connector.synced", "files.connector.accessed"),
|
||||||
notes="Browse/import logic is implemented, but live boto3 access fails closed until its HTTP transport supports connection-time DNS/IP pinning and redirect revalidation.",
|
notes="Every botocore connection pool uses the Files-owned pinned transport, including retries, redirects, endpoint discovery, and virtual-host aliases; outbound proxies and ambient credential discovery are disabled.",
|
||||||
),
|
),
|
||||||
ConnectorProviderDescriptor(
|
ConnectorProviderDescriptor(
|
||||||
provider="sharepoint",
|
provider="sharepoint",
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ from govoplan_files.backend.storage.connector_profiles import ConnectorProfile
|
|||||||
|
|
||||||
|
|
||||||
SYNC_MODES = {"manual"}
|
SYNC_MODES = {"manual"}
|
||||||
|
WRITABLE_PROVIDERS = {"s3"}
|
||||||
|
|
||||||
|
|
||||||
def connector_space_owner_id(space: FileConnectorSpace) -> str:
|
def connector_space_owner_id(space: FileConnectorSpace) -> str:
|
||||||
@@ -40,6 +41,7 @@ def create_connector_space(
|
|||||||
ensure_owner_access(session, tenant_id=tenant_id, owner_type=owner_type, owner_id=owner_id, user_id=user_id, is_admin=is_admin)
|
ensure_owner_access(session, tenant_id=tenant_id, owner_type=owner_type, owner_id=owner_id, user_id=user_id, is_admin=is_admin)
|
||||||
label = _normalize_label(label)
|
label = _normalize_label(label)
|
||||||
sync_mode = _normalize_sync_mode(sync_mode)
|
sync_mode = _normalize_sync_mode(sync_mode)
|
||||||
|
validate_connector_space_write_mode(profile, read_only=read_only)
|
||||||
remote_path = normalize_connector_browse_path(remote_path)
|
remote_path = normalize_connector_browse_path(remote_path)
|
||||||
library_id = _clean_optional(library_id)
|
library_id = _clean_optional(library_id)
|
||||||
|
|
||||||
@@ -145,6 +147,7 @@ def update_connector_space(
|
|||||||
library_id: str | None = None,
|
library_id: str | None = None,
|
||||||
remote_path: str | None = None,
|
remote_path: str | None = None,
|
||||||
sync_mode: str | None = None,
|
sync_mode: str | None = None,
|
||||||
|
read_only: bool | None = None,
|
||||||
is_active: bool | None = None,
|
is_active: bool | None = None,
|
||||||
metadata: Mapping[str, Any] | None = None,
|
metadata: Mapping[str, Any] | None = None,
|
||||||
is_admin: bool = False,
|
is_admin: bool = False,
|
||||||
@@ -178,6 +181,8 @@ def update_connector_space(
|
|||||||
space.remote_path = normalize_connector_browse_path(remote_path)
|
space.remote_path = normalize_connector_browse_path(remote_path)
|
||||||
if sync_mode is not None:
|
if sync_mode is not None:
|
||||||
space.sync_mode = _normalize_sync_mode(sync_mode)
|
space.sync_mode = _normalize_sync_mode(sync_mode)
|
||||||
|
if read_only is not None:
|
||||||
|
space.read_only = bool(read_only)
|
||||||
if is_active is not None:
|
if is_active is not None:
|
||||||
space.is_active = bool(is_active)
|
space.is_active = bool(is_active)
|
||||||
if metadata is not None:
|
if metadata is not None:
|
||||||
@@ -187,6 +192,17 @@ def update_connector_space(
|
|||||||
return space
|
return space
|
||||||
|
|
||||||
|
|
||||||
|
def validate_connector_space_write_mode(
|
||||||
|
profile: ConnectorProfile, *, read_only: bool
|
||||||
|
) -> None:
|
||||||
|
if read_only:
|
||||||
|
return
|
||||||
|
if profile.provider not in WRITABLE_PROVIDERS or "write" not in profile.capabilities:
|
||||||
|
raise FileStorageError(
|
||||||
|
"Two-way mode requires an S3 connection with the write capability enabled"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def soft_delete_connector_space(
|
def soft_delete_connector_space(
|
||||||
session: Session,
|
session: Session,
|
||||||
space: FileConnectorSpace,
|
space: FileConnectorSpace,
|
||||||
|
|||||||
@@ -136,11 +136,6 @@ def connector_profile_usable_for_import(profile: ConnectorProfile) -> bool:
|
|||||||
and descriptor.import_supported
|
and descriptor.import_supported
|
||||||
):
|
):
|
||||||
return False
|
return False
|
||||||
# These SDK paths intentionally fail closed until every connection peer and
|
|
||||||
# SDK-managed redirect/referral can be pinned and revalidated.
|
|
||||||
if profile.provider in {"s3", "smb"}:
|
|
||||||
return False
|
|
||||||
|
|
||||||
# Prove that the initial root browse performed by the current Files UI is
|
# Prove that the initial root browse performed by the current Files UI is
|
||||||
# policy-allowed. A later selected remote path/item is checked again.
|
# policy-allowed. A later selected remote path/item is checked again.
|
||||||
return connector_policy_decision(
|
return connector_policy_decision(
|
||||||
|
|||||||
@@ -0,0 +1,438 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
import hashlib
|
||||||
|
from typing import Any, Mapping
|
||||||
|
|
||||||
|
from govoplan_core.core.recovery import (
|
||||||
|
RecoveryGuaranteeError,
|
||||||
|
RecoveryMode,
|
||||||
|
RecoveryPlan,
|
||||||
|
RecoveryStatus,
|
||||||
|
)
|
||||||
|
from govoplan_core.core.recovery_runtime import (
|
||||||
|
RecoveryOperationBusy,
|
||||||
|
RecoveryOperationStateConflict,
|
||||||
|
begin_durable_recovery_operation,
|
||||||
|
)
|
||||||
|
from govoplan_core.core.runtime_coordination import process_runtime_identity
|
||||||
|
from govoplan_core.db.session import get_database
|
||||||
|
from govoplan_files.backend.storage.common import FileStorageError
|
||||||
|
from govoplan_files.backend.storage.connector_browse import (
|
||||||
|
ConnectorBrowseError,
|
||||||
|
ConnectorBrowseUnsupported,
|
||||||
|
_clean,
|
||||||
|
_s3_bucket,
|
||||||
|
_s3_client,
|
||||||
|
_s3_object_key,
|
||||||
|
normalize_connector_browse_path,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.storage.connector_profiles import ConnectorProfile
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class ConnectorWriteResult:
|
||||||
|
recovery_operation_id: str
|
||||||
|
status: str
|
||||||
|
replayed: bool
|
||||||
|
provider: str
|
||||||
|
remote_path: str
|
||||||
|
revision: str | None
|
||||||
|
checksum_sha256: str
|
||||||
|
size_bytes: int
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class _HeadProbe:
|
||||||
|
observed: Mapping[str, Any] | None
|
||||||
|
verified: bool
|
||||||
|
exception_type: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def write_connector_file(
|
||||||
|
profile: ConnectorProfile,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
library_id: str | None,
|
||||||
|
remote_path: str,
|
||||||
|
data: bytes,
|
||||||
|
content_type: str | None,
|
||||||
|
expected_revision: str | None,
|
||||||
|
idempotency_key: str,
|
||||||
|
) -> ConnectorWriteResult:
|
||||||
|
if profile.provider != "s3" or "write" not in profile.capabilities:
|
||||||
|
raise FileStorageError(
|
||||||
|
"Remote writes require an S3 connection with the write capability enabled"
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
normalized_path = normalize_connector_browse_path(remote_path)
|
||||||
|
except ConnectorBrowseError as exc:
|
||||||
|
raise FileStorageError(str(exc)) from exc
|
||||||
|
if not normalized_path:
|
||||||
|
raise FileStorageError("Remote write requires an object path")
|
||||||
|
try:
|
||||||
|
bucket = _s3_bucket(profile, library_id)
|
||||||
|
key = _s3_object_key(profile, normalized_path)
|
||||||
|
except ConnectorBrowseError as exc:
|
||||||
|
raise FileStorageError(str(exc)) from exc
|
||||||
|
if not bucket or not key:
|
||||||
|
raise FileStorageError("S3 remote write requires a bucket and object path")
|
||||||
|
checksum = hashlib.sha256(data).hexdigest()
|
||||||
|
target_digest = hashlib.sha256(f"{bucket}:{key}".encode("utf-8")).hexdigest()
|
||||||
|
request = {
|
||||||
|
"tenant_id": tenant_id,
|
||||||
|
"connector_profile_id": profile.id,
|
||||||
|
"provider": profile.provider,
|
||||||
|
"remote_target_sha256": target_digest,
|
||||||
|
"content_sha256": checksum,
|
||||||
|
"size_bytes": len(data),
|
||||||
|
"expected_revision": expected_revision,
|
||||||
|
}
|
||||||
|
try:
|
||||||
|
started = begin_durable_recovery_operation(
|
||||||
|
get_database().SessionLocal,
|
||||||
|
identity=process_runtime_identity(),
|
||||||
|
module_id="files",
|
||||||
|
operation_type="connector-s3-write",
|
||||||
|
idempotency_key=f"files-connector-write:{idempotency_key}",
|
||||||
|
request=request,
|
||||||
|
recovery_plan=RecoveryPlan(
|
||||||
|
mode=RecoveryMode.FORWARD_RECOVERY,
|
||||||
|
preconditions=(
|
||||||
|
"the connector space is explicitly configured for two-way writes",
|
||||||
|
"the connection and inherited policy allow the exact remote path",
|
||||||
|
"a conditional create or expected remote revision prevents blind overwrite",
|
||||||
|
),
|
||||||
|
forward_recovery_steps=(
|
||||||
|
"inspect provider metadata for the request and content digests",
|
||||||
|
"resolve success only for an exact digest match or retry after verified absence",
|
||||||
|
),
|
||||||
|
verification_steps=(
|
||||||
|
"read S3 object metadata after the write",
|
||||||
|
"match the recorded request and content digests without downloading content",
|
||||||
|
),
|
||||||
|
),
|
||||||
|
precondition_evidence={
|
||||||
|
"remote_target_sha256": target_digest,
|
||||||
|
"content_sha256": checksum,
|
||||||
|
"size_bytes": len(data),
|
||||||
|
"conditional_write": True,
|
||||||
|
},
|
||||||
|
lease_resource_key=f"files:connector:{tenant_id}:{profile.id}:{target_digest[:40]}",
|
||||||
|
lease_ttl_seconds=15 * 60,
|
||||||
|
resource_type="file_connector_object",
|
||||||
|
resource_id=target_digest,
|
||||||
|
metadata={
|
||||||
|
"resources": ["postgresql", "s3-connector"],
|
||||||
|
"provider": "s3",
|
||||||
|
"connector_profile_id": profile.id,
|
||||||
|
},
|
||||||
|
block_unresolved_resource=True,
|
||||||
|
)
|
||||||
|
except (RecoveryOperationBusy, RecoveryOperationStateConflict) as exc:
|
||||||
|
raise FileStorageError(
|
||||||
|
"This remote object is owned by another write or unresolved recovery operation"
|
||||||
|
) from exc
|
||||||
|
except (RecoveryGuaranteeError, RuntimeError, ValueError) as exc:
|
||||||
|
raise FileStorageError(
|
||||||
|
"The Files recovery ledger is unavailable; the remote object was not changed"
|
||||||
|
) from exc
|
||||||
|
if started.replayed or started.operation is None:
|
||||||
|
return ConnectorWriteResult(
|
||||||
|
recovery_operation_id=started.operation_id,
|
||||||
|
status=started.status,
|
||||||
|
replayed=True,
|
||||||
|
provider="s3",
|
||||||
|
remote_path=normalized_path,
|
||||||
|
revision=None,
|
||||||
|
checksum_sha256=checksum,
|
||||||
|
size_bytes=len(data),
|
||||||
|
)
|
||||||
|
operation = started.operation
|
||||||
|
try:
|
||||||
|
client = _s3_client(profile)
|
||||||
|
except (ConnectorBrowseError, ConnectorBrowseUnsupported) as exc:
|
||||||
|
operation.reject(
|
||||||
|
summary="The S3 client was unavailable before the remote effect",
|
||||||
|
evidence=_verified(
|
||||||
|
{"external_effect_started": False}, exception_type=type(exc).__name__
|
||||||
|
),
|
||||||
|
)
|
||||||
|
raise FileStorageError(str(exc)) from exc
|
||||||
|
try:
|
||||||
|
try:
|
||||||
|
before = _head_object(client, bucket=bucket, key=key)
|
||||||
|
except Exception as exc:
|
||||||
|
operation.reject(
|
||||||
|
summary="Remote preconditions could not be inspected before the effect",
|
||||||
|
evidence=_verified(
|
||||||
|
{"external_effect_started": False},
|
||||||
|
exception_type=type(exc).__name__,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
raise FileStorageError("S3 remote revision lookup failed") from exc
|
||||||
|
expected = _normalize_revision(expected_revision)
|
||||||
|
if before is not None and expected is None:
|
||||||
|
operation.reject(
|
||||||
|
summary="A blind remote overwrite was rejected",
|
||||||
|
evidence=_verified(
|
||||||
|
{"remote_object_exists": True, "expected_revision_supplied": False}
|
||||||
|
),
|
||||||
|
)
|
||||||
|
raise FileStorageError(
|
||||||
|
"The remote object already exists; reload it and supply its expected revision"
|
||||||
|
)
|
||||||
|
if expected is not None and (
|
||||||
|
before is None or expected not in _observed_revisions(before)
|
||||||
|
):
|
||||||
|
operation.reject(
|
||||||
|
summary="The expected remote revision did not match",
|
||||||
|
evidence=_verified(
|
||||||
|
{
|
||||||
|
"remote_object_exists": before is not None,
|
||||||
|
"expected_revision_matches": False,
|
||||||
|
}
|
||||||
|
),
|
||||||
|
)
|
||||||
|
raise FileStorageError("The remote object changed; reload before writing")
|
||||||
|
params: dict[str, object] = {
|
||||||
|
"Bucket": bucket,
|
||||||
|
"Key": key,
|
||||||
|
"Body": data,
|
||||||
|
"Metadata": {
|
||||||
|
"govoplan-sha256": checksum,
|
||||||
|
"govoplan-request-id": started.operation_id,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
if content_type:
|
||||||
|
params["ContentType"] = content_type
|
||||||
|
if before is None:
|
||||||
|
params["IfNoneMatch"] = "*"
|
||||||
|
else:
|
||||||
|
params["IfMatch"] = str(before.get("ETag") or expected_revision or "")
|
||||||
|
try:
|
||||||
|
client.put_object(**params)
|
||||||
|
except Exception as exc:
|
||||||
|
probe = _probe_head_object(client, bucket=bucket, key=key)
|
||||||
|
if probe.verified and _matches_effect(
|
||||||
|
probe.observed,
|
||||||
|
checksum=checksum,
|
||||||
|
operation_id=started.operation_id,
|
||||||
|
):
|
||||||
|
operation.succeed(
|
||||||
|
evidence=_success_evidence(
|
||||||
|
probe.observed,
|
||||||
|
checksum=checksum,
|
||||||
|
operation_id=started.operation_id,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return _result(
|
||||||
|
started.operation_id,
|
||||||
|
normalized_path,
|
||||||
|
checksum,
|
||||||
|
len(data),
|
||||||
|
probe.observed,
|
||||||
|
)
|
||||||
|
operation.unresolved(
|
||||||
|
status=RecoveryStatus.OUTCOME_UNKNOWN,
|
||||||
|
summary="The S3 write outcome requires reconciliation",
|
||||||
|
evidence={
|
||||||
|
"remote_target_sha256": target_digest,
|
||||||
|
"observed": _public_observation(probe),
|
||||||
|
"exception_type": type(exc).__name__,
|
||||||
|
},
|
||||||
|
failure_summary="Inspect the remote object metadata before retrying this path",
|
||||||
|
)
|
||||||
|
return ConnectorWriteResult(
|
||||||
|
recovery_operation_id=started.operation_id,
|
||||||
|
status=RecoveryStatus.OUTCOME_UNKNOWN.value,
|
||||||
|
replayed=False,
|
||||||
|
provider="s3",
|
||||||
|
remote_path=normalized_path,
|
||||||
|
revision=_revision(probe.observed),
|
||||||
|
checksum_sha256=checksum,
|
||||||
|
size_bytes=len(data),
|
||||||
|
)
|
||||||
|
probe = _probe_head_object(client, bucket=bucket, key=key)
|
||||||
|
if not probe.verified:
|
||||||
|
operation.unresolved(
|
||||||
|
status=RecoveryStatus.OUTCOME_UNKNOWN,
|
||||||
|
summary="The S3 write returned but provider evidence could not be queried",
|
||||||
|
evidence={
|
||||||
|
"remote_target_sha256": target_digest,
|
||||||
|
"observed": _public_observation(probe),
|
||||||
|
},
|
||||||
|
failure_summary="Inspect the remote object metadata before retrying this path",
|
||||||
|
)
|
||||||
|
return ConnectorWriteResult(
|
||||||
|
recovery_operation_id=started.operation_id,
|
||||||
|
status=RecoveryStatus.OUTCOME_UNKNOWN.value,
|
||||||
|
replayed=False,
|
||||||
|
provider="s3",
|
||||||
|
remote_path=normalized_path,
|
||||||
|
revision=None,
|
||||||
|
checksum_sha256=checksum,
|
||||||
|
size_bytes=len(data),
|
||||||
|
)
|
||||||
|
observed = probe.observed
|
||||||
|
if not _matches_effect(
|
||||||
|
observed, checksum=checksum, operation_id=started.operation_id
|
||||||
|
):
|
||||||
|
operation.unresolved(
|
||||||
|
status=RecoveryStatus.RECOVERY_REQUIRED,
|
||||||
|
summary="The S3 write returned but exact provider evidence did not match",
|
||||||
|
evidence={
|
||||||
|
"remote_target_sha256": target_digest,
|
||||||
|
"observed": _public_observation(probe),
|
||||||
|
},
|
||||||
|
failure_summary="Reconcile the remote object before another write",
|
||||||
|
)
|
||||||
|
return ConnectorWriteResult(
|
||||||
|
recovery_operation_id=started.operation_id,
|
||||||
|
status=RecoveryStatus.RECOVERY_REQUIRED.value,
|
||||||
|
replayed=False,
|
||||||
|
provider="s3",
|
||||||
|
remote_path=normalized_path,
|
||||||
|
revision=_revision(observed),
|
||||||
|
checksum_sha256=checksum,
|
||||||
|
size_bytes=len(data),
|
||||||
|
)
|
||||||
|
operation.succeed(
|
||||||
|
evidence=_success_evidence(
|
||||||
|
observed, checksum=checksum, operation_id=started.operation_id
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return _result(
|
||||||
|
started.operation_id,
|
||||||
|
normalized_path,
|
||||||
|
checksum,
|
||||||
|
len(data),
|
||||||
|
observed,
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
close = getattr(client, "close", None)
|
||||||
|
if callable(close):
|
||||||
|
close()
|
||||||
|
|
||||||
|
|
||||||
|
def _head_object(client: Any, *, bucket: str, key: str) -> Mapping[str, Any] | None:
|
||||||
|
try:
|
||||||
|
response = client.head_object(Bucket=bucket, Key=key)
|
||||||
|
except Exception as exc:
|
||||||
|
if _is_not_found(exc):
|
||||||
|
return None
|
||||||
|
raise
|
||||||
|
if not isinstance(response, Mapping):
|
||||||
|
raise FileStorageError("S3 connector returned invalid object metadata")
|
||||||
|
return response
|
||||||
|
|
||||||
|
|
||||||
|
def _probe_head_object(client: Any, *, bucket: str, key: str) -> _HeadProbe:
|
||||||
|
try:
|
||||||
|
return _HeadProbe(
|
||||||
|
observed=_head_object(client, bucket=bucket, key=key), verified=True
|
||||||
|
)
|
||||||
|
except Exception as exc:
|
||||||
|
return _HeadProbe(
|
||||||
|
observed=None,
|
||||||
|
verified=False,
|
||||||
|
exception_type=type(exc).__name__,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _is_not_found(exc: Exception) -> bool:
|
||||||
|
response = getattr(exc, "response", None)
|
||||||
|
if not isinstance(response, Mapping):
|
||||||
|
return False
|
||||||
|
error = response.get("Error")
|
||||||
|
code = error.get("Code") if isinstance(error, Mapping) else None
|
||||||
|
status = response.get("ResponseMetadata")
|
||||||
|
http_status = status.get("HTTPStatusCode") if isinstance(status, Mapping) else None
|
||||||
|
return str(code).casefold() in {"404", "nosuchkey", "notfound"} or http_status == 404
|
||||||
|
|
||||||
|
|
||||||
|
def _metadata(observed: Mapping[str, Any] | None) -> Mapping[str, Any]:
|
||||||
|
value = observed.get("Metadata") if observed else None
|
||||||
|
return value if isinstance(value, Mapping) else {}
|
||||||
|
|
||||||
|
|
||||||
|
def _matches_effect(
|
||||||
|
observed: Mapping[str, Any] | None, *, checksum: str, operation_id: str
|
||||||
|
) -> bool:
|
||||||
|
metadata = _metadata(observed)
|
||||||
|
return bool(
|
||||||
|
observed is not None
|
||||||
|
and _clean(metadata.get("govoplan-sha256")) == checksum
|
||||||
|
and _clean(metadata.get("govoplan-request-id")) == operation_id
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _revision(observed: Mapping[str, Any] | None) -> str | None:
|
||||||
|
if observed is None:
|
||||||
|
return None
|
||||||
|
return _clean(observed.get("VersionId") or observed.get("ETag"))
|
||||||
|
|
||||||
|
|
||||||
|
def _normalize_revision(value: str | None) -> str | None:
|
||||||
|
cleaned = _clean(value)
|
||||||
|
return cleaned.strip('"') if cleaned else None
|
||||||
|
|
||||||
|
|
||||||
|
def _observed_revisions(observed: Mapping[str, Any]) -> set[str]:
|
||||||
|
return {
|
||||||
|
normalized
|
||||||
|
for value in (observed.get("VersionId"), observed.get("ETag"))
|
||||||
|
if (normalized := _normalize_revision(_clean(value))) is not None
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _success_evidence(
|
||||||
|
observed: Mapping[str, Any] | None, *, checksum: str, operation_id: str
|
||||||
|
) -> dict[str, object]:
|
||||||
|
return _verified(
|
||||||
|
{
|
||||||
|
"remote_object_present": observed is not None,
|
||||||
|
"content_digest_matches": _clean(_metadata(observed).get("govoplan-sha256")) == checksum,
|
||||||
|
"request_marker_matches": _clean(_metadata(observed).get("govoplan-request-id")) == operation_id,
|
||||||
|
},
|
||||||
|
revision=_revision(observed),
|
||||||
|
content_sha256=checksum,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _public_observation(probe: _HeadProbe) -> dict[str, object]:
|
||||||
|
observed = probe.observed
|
||||||
|
return {
|
||||||
|
"probe_verified": probe.verified,
|
||||||
|
"probe_exception_type": probe.exception_type,
|
||||||
|
"present": observed is not None,
|
||||||
|
"revision": _revision(observed),
|
||||||
|
"has_content_digest": bool(_clean(_metadata(observed).get("govoplan-sha256"))),
|
||||||
|
"has_request_marker": bool(_clean(_metadata(observed).get("govoplan-request-id"))),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _verified(checks: dict[str, object], **details: object) -> dict[str, object]:
|
||||||
|
return {"verified": True, "checks": checks, **details}
|
||||||
|
|
||||||
|
|
||||||
|
def _result(
|
||||||
|
operation_id: str,
|
||||||
|
remote_path: str,
|
||||||
|
checksum: str,
|
||||||
|
size_bytes: int,
|
||||||
|
observed: Mapping[str, Any] | None,
|
||||||
|
) -> ConnectorWriteResult:
|
||||||
|
return ConnectorWriteResult(
|
||||||
|
recovery_operation_id=operation_id,
|
||||||
|
status=RecoveryStatus.SUCCEEDED.value,
|
||||||
|
replayed=False,
|
||||||
|
provider="s3",
|
||||||
|
remote_path=remote_path,
|
||||||
|
revision=_revision(observed),
|
||||||
|
checksum_sha256=checksum,
|
||||||
|
size_bytes=size_bytes,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["ConnectorWriteResult", "write_connector_file"]
|
||||||
@@ -84,6 +84,7 @@ def _get_or_create_blob(
|
|||||||
blob = (
|
blob = (
|
||||||
session.query(FileBlob)
|
session.query(FileBlob)
|
||||||
.filter(FileBlob.tenant_id == tenant_id, FileBlob.checksum_sha256 == checksum, FileBlob.size_bytes == size, FileBlob.protection_discriminator == protection_discriminator)
|
.filter(FileBlob.tenant_id == tenant_id, FileBlob.checksum_sha256 == checksum, FileBlob.size_bytes == size, FileBlob.protection_discriminator == protection_discriminator)
|
||||||
|
.with_for_update()
|
||||||
.one_or_none()
|
.one_or_none()
|
||||||
)
|
)
|
||||||
if blob:
|
if blob:
|
||||||
@@ -529,6 +530,41 @@ def list_assets_for_user(
|
|||||||
return query.order_by(FileAsset.display_path.asc(), FileAsset.updated_at.desc(), FileAsset.id.asc()).all()
|
return query.order_by(FileAsset.display_path.asc(), FileAsset.updated_at.desc(), FileAsset.id.asc()).all()
|
||||||
|
|
||||||
|
|
||||||
|
def list_recent_assets_for_user(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
user_id: str,
|
||||||
|
limit: int,
|
||||||
|
owner_type: str | None = None,
|
||||||
|
owner_id: str | None = None,
|
||||||
|
campaign_id: str | None = None,
|
||||||
|
path_prefix: str | None = None,
|
||||||
|
campaign_usage: str | None = None,
|
||||||
|
audit_relevant: bool | None = None,
|
||||||
|
is_admin: bool = False,
|
||||||
|
) -> list[FileAsset]:
|
||||||
|
"""Return a bounded recent projection through the normal access query."""
|
||||||
|
|
||||||
|
query = _asset_visibility_query_for_user(
|
||||||
|
session,
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
user_id=user_id,
|
||||||
|
owner_type=owner_type,
|
||||||
|
owner_id=owner_id,
|
||||||
|
campaign_id=campaign_id,
|
||||||
|
path_prefix=path_prefix,
|
||||||
|
campaign_usage=campaign_usage,
|
||||||
|
audit_relevant=audit_relevant,
|
||||||
|
is_admin=is_admin,
|
||||||
|
)
|
||||||
|
return (
|
||||||
|
query.order_by(FileAsset.updated_at.desc(), FileAsset.id.asc())
|
||||||
|
.limit(max(1, limit))
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def list_assets_for_user_window(
|
def list_assets_for_user_window(
|
||||||
session: Session,
|
session: Session,
|
||||||
*,
|
*,
|
||||||
@@ -596,21 +632,31 @@ def _asset_visibility_query_for_user(
|
|||||||
if owner_type == "group" and owner_id:
|
if owner_type == "group" and owner_id:
|
||||||
query = query.filter(FileAsset.owner_group_id == owner_id)
|
query = query.filter(FileAsset.owner_group_id == owner_id)
|
||||||
if campaign_id:
|
if campaign_id:
|
||||||
query = query.join(FileShare, FileShare.file_asset_id == FileAsset.id).filter(
|
campaign_share = exists().where(
|
||||||
FileShare.tenant_id == tenant_id,
|
FileShare.tenant_id == tenant_id,
|
||||||
|
FileShare.file_asset_id == FileAsset.id,
|
||||||
FileShare.target_type == "campaign",
|
FileShare.target_type == "campaign",
|
||||||
FileShare.target_id == campaign_id,
|
FileShare.target_id == campaign_id,
|
||||||
effective_file_share_clause(),
|
effective_file_share_clause(),
|
||||||
)
|
)
|
||||||
|
query = query.filter(campaign_share)
|
||||||
elif not is_admin and not owner_type:
|
elif not is_admin and not owner_type:
|
||||||
group_ids = user_group_ids(session, tenant_id=tenant_id, user_id=user_id)
|
group_ids = user_group_ids(session, tenant_id=tenant_id, user_id=user_id)
|
||||||
query = query.outerjoin(FileShare, FileShare.file_asset_id == FileAsset.id).filter(
|
active_share = exists().where(
|
||||||
|
FileShare.tenant_id == tenant_id,
|
||||||
|
FileShare.file_asset_id == FileAsset.id,
|
||||||
|
effective_file_share_clause(),
|
||||||
|
or_(
|
||||||
|
(FileShare.target_type == "user") & (FileShare.target_id == user_id),
|
||||||
|
(FileShare.target_type == "group") & (FileShare.target_id.in_(group_ids)),
|
||||||
|
(FileShare.target_type == "tenant") & (FileShare.target_id == tenant_id),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
query = query.filter(
|
||||||
or_(
|
or_(
|
||||||
(FileAsset.owner_type == "user") & (FileAsset.owner_user_id == user_id),
|
(FileAsset.owner_type == "user") & (FileAsset.owner_user_id == user_id),
|
||||||
(FileAsset.owner_type == "group") & (FileAsset.owner_group_id.in_(group_ids)),
|
(FileAsset.owner_type == "group") & (FileAsset.owner_group_id.in_(group_ids)),
|
||||||
effective_file_share_clause() & (FileShare.target_type == "user") & (FileShare.target_id == user_id),
|
active_share,
|
||||||
effective_file_share_clause() & (FileShare.target_type == "group") & (FileShare.target_id.in_(group_ids)),
|
|
||||||
effective_file_share_clause() & (FileShare.target_type == "tenant") & (FileShare.target_id == tenant_id),
|
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
if path_prefix:
|
if path_prefix:
|
||||||
@@ -639,7 +685,7 @@ def _asset_visibility_query_for_user(
|
|||||||
CampaignAttachmentUse.use_stage == "sent",
|
CampaignAttachmentUse.use_stage == "sent",
|
||||||
)
|
)
|
||||||
query = query.filter(sent_use_exists if audit_relevant else ~sent_use_exists)
|
query = query.filter(sent_use_exists if audit_relevant else ~sent_use_exists)
|
||||||
return query.distinct()
|
return query
|
||||||
|
|
||||||
|
|
||||||
def count_assets_for_user(
|
def count_assets_for_user(
|
||||||
@@ -721,6 +767,21 @@ def read_asset_bytes(session: Session, asset: FileAsset) -> tuple[bytes, FileVer
|
|||||||
return read_verified_blob_bytes(blob, backend=backend), version, blob
|
return read_verified_blob_bytes(blob, backend=backend), version, blob
|
||||||
|
|
||||||
|
|
||||||
|
def read_asset_version_bytes(
|
||||||
|
session: Session,
|
||||||
|
asset: FileAsset,
|
||||||
|
version_id: str,
|
||||||
|
) -> tuple[bytes, FileVersion, FileBlob]:
|
||||||
|
version = session.get(FileVersion, version_id)
|
||||||
|
if version is None or version.file_asset_id != asset.id:
|
||||||
|
raise FileStorageError("File version not found")
|
||||||
|
blob = session.get(FileBlob, version.blob_id)
|
||||||
|
if blob is None or blob.tenant_id != asset.tenant_id:
|
||||||
|
raise FileStorageError("File blob not found")
|
||||||
|
backend = get_storage_backend()
|
||||||
|
return read_verified_blob_bytes(blob, backend=backend), version, blob
|
||||||
|
|
||||||
|
|
||||||
def share_file(
|
def share_file(
|
||||||
session: Session,
|
session: Session,
|
||||||
*,
|
*,
|
||||||
|
|||||||
@@ -99,6 +99,7 @@ def run_integrity_scan_batch(
|
|||||||
scan.phase = "completed"
|
scan.phase = "completed"
|
||||||
scan.status = "completed"
|
scan.status = "completed"
|
||||||
scan.completed_at = utcnow()
|
scan.completed_at = utcnow()
|
||||||
|
scan.revision += 1
|
||||||
session.add(scan)
|
session.add(scan)
|
||||||
return scan
|
return scan
|
||||||
|
|
||||||
@@ -110,6 +111,7 @@ def mark_integrity_scan_failed(
|
|||||||
) -> None:
|
) -> None:
|
||||||
scan.status = "failed"
|
scan.status = "failed"
|
||||||
scan.last_error = type(error).__name__[:255]
|
scan.last_error = type(error).__name__[:255]
|
||||||
|
scan.revision += 1
|
||||||
|
|
||||||
|
|
||||||
def inspect_blob(
|
def inspect_blob(
|
||||||
@@ -269,6 +271,7 @@ def recheck_integrity_finding(
|
|||||||
finding.resolved_at = utcnow()
|
finding.resolved_at = utcnow()
|
||||||
finding.resolved_by_user_id = user_id
|
finding.resolved_by_user_id = user_id
|
||||||
session.add(finding)
|
session.add(finding)
|
||||||
|
finding.revision += 1
|
||||||
changed = previous != (
|
changed = previous != (
|
||||||
blob.integrity_status,
|
blob.integrity_status,
|
||||||
blob.quarantined_at,
|
blob.quarantined_at,
|
||||||
@@ -312,6 +315,7 @@ def cleanup_orphan_finding(
|
|||||||
finding.state = "resolved"
|
finding.state = "resolved"
|
||||||
finding.resolved_at = utcnow()
|
finding.resolved_at = utcnow()
|
||||||
finding.resolved_by_user_id = user_id
|
finding.resolved_by_user_id = user_id
|
||||||
|
finding.revision += 1
|
||||||
session.add(finding)
|
session.add(finding)
|
||||||
return IntegrityActionResult(
|
return IntegrityActionResult(
|
||||||
action="retained_referenced",
|
action="retained_referenced",
|
||||||
@@ -360,6 +364,7 @@ def cleanup_orphan_finding(
|
|||||||
finding.state = "deleted"
|
finding.state = "deleted"
|
||||||
finding.resolved_at = utcnow()
|
finding.resolved_at = utcnow()
|
||||||
finding.resolved_by_user_id = user_id
|
finding.resolved_by_user_id = user_id
|
||||||
|
finding.revision += 1
|
||||||
session.add(finding)
|
session.add(finding)
|
||||||
return IntegrityActionResult(
|
return IntegrityActionResult(
|
||||||
action=action,
|
action=action,
|
||||||
@@ -465,6 +470,8 @@ def _record_blob_finding(
|
|||||||
expected_size_bytes=blob.storage_size_bytes if blob.storage_size_bytes is not None else blob.size_bytes,
|
expected_size_bytes=blob.storage_size_bytes if blob.storage_size_bytes is not None else blob.size_bytes,
|
||||||
expected_checksum_sha256=blob.storage_checksum_sha256 if blob.storage_checksum_sha256 is not None else blob.checksum_sha256,
|
expected_checksum_sha256=blob.storage_checksum_sha256 if blob.storage_checksum_sha256 is not None else blob.checksum_sha256,
|
||||||
)
|
)
|
||||||
|
else:
|
||||||
|
finding.revision += 1
|
||||||
_update_finding_from_inspection(finding, inspection)
|
_update_finding_from_inspection(finding, inspection)
|
||||||
session.add(finding)
|
session.add(finding)
|
||||||
return finding
|
return finding
|
||||||
@@ -514,6 +521,7 @@ def _resolve_scan_blob_findings(
|
|||||||
):
|
):
|
||||||
finding.state = "resolved"
|
finding.state = "resolved"
|
||||||
finding.resolved_at = utcnow()
|
finding.resolved_at = utcnow()
|
||||||
|
finding.revision += 1
|
||||||
session.add(finding)
|
session.add(finding)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,796 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from typing import Callable, Iterable
|
||||||
|
|
||||||
|
from sqlalchemy import func, inspect, or_
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from govoplan_core.core.recovery import (
|
||||||
|
RecoveryGuaranteeError,
|
||||||
|
RecoveryMode,
|
||||||
|
RecoveryPlan,
|
||||||
|
RecoveryStatus,
|
||||||
|
)
|
||||||
|
from govoplan_core.core.recovery_runtime import (
|
||||||
|
RecoveryOperationBusy,
|
||||||
|
RecoveryOperationStateConflict,
|
||||||
|
begin_durable_recovery_operation,
|
||||||
|
)
|
||||||
|
from govoplan_core.core.runtime_coordination import process_runtime_identity
|
||||||
|
from govoplan_core.db.session import get_database
|
||||||
|
from govoplan_files.backend.db.models import (
|
||||||
|
CampaignAttachmentUse,
|
||||||
|
FileAsset,
|
||||||
|
FileBlob,
|
||||||
|
FileConnectorSpace,
|
||||||
|
FileFolder,
|
||||||
|
FileFormEvidenceGrant,
|
||||||
|
FileShare,
|
||||||
|
FileVersion,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.storage.access import ensure_owner_access
|
||||||
|
from govoplan_files.backend.storage.backends import (
|
||||||
|
StorageBackendError,
|
||||||
|
get_storage_backend,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.storage.common import FileStorageError, utcnow
|
||||||
|
from govoplan_files.backend.storage.connector_spaces import connector_space_owner_id
|
||||||
|
from govoplan_files.backend.storage.paths import normalize_folder
|
||||||
|
from govoplan_files.backend.storage.share_state import effective_file_share_clause
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class PurgePreviewItem:
|
||||||
|
file_id: str
|
||||||
|
filename: str
|
||||||
|
lifecycle_revision: int
|
||||||
|
deleted_at: datetime | None
|
||||||
|
retained_until: datetime | None
|
||||||
|
legal_hold: bool
|
||||||
|
blockers: tuple[str, ...]
|
||||||
|
blob_ids: tuple[str, ...]
|
||||||
|
|
||||||
|
def digest_payload(self) -> dict[str, object]:
|
||||||
|
return {
|
||||||
|
"file_id": self.file_id,
|
||||||
|
"filename": self.filename,
|
||||||
|
"lifecycle_revision": self.lifecycle_revision,
|
||||||
|
"deleted_at": _iso(self.deleted_at),
|
||||||
|
"retained_until": _iso(self.retained_until),
|
||||||
|
"legal_hold": self.legal_hold,
|
||||||
|
"blockers": list(self.blockers),
|
||||||
|
"blob_ids": list(self.blob_ids),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class PurgePreview:
|
||||||
|
preview_sha256: str
|
||||||
|
items: tuple[PurgePreviewItem, ...]
|
||||||
|
|
||||||
|
@property
|
||||||
|
def eligible(self) -> bool:
|
||||||
|
return all(not item.blockers for item in self.items)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class PurgeResult:
|
||||||
|
recovery_operation_id: str
|
||||||
|
status: str
|
||||||
|
replayed: bool
|
||||||
|
purged_files: int
|
||||||
|
released_blobs: int
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class BlobGcResult:
|
||||||
|
inspected_blobs: int
|
||||||
|
deleted_blobs: int
|
||||||
|
unresolved_operation_ids: tuple[str, ...]
|
||||||
|
|
||||||
|
|
||||||
|
def set_asset_lifecycle(
|
||||||
|
session: Session,
|
||||||
|
asset: FileAsset,
|
||||||
|
*,
|
||||||
|
retained_until: datetime | None,
|
||||||
|
legal_hold: bool,
|
||||||
|
reason: str,
|
||||||
|
expected_revision: int,
|
||||||
|
) -> FileAsset:
|
||||||
|
if asset.lifecycle_revision != expected_revision:
|
||||||
|
raise FileStorageError("File lifecycle settings changed; reload before saving")
|
||||||
|
asset.retained_until = retained_until
|
||||||
|
asset.legal_hold = bool(legal_hold)
|
||||||
|
asset.lifecycle_reason = reason.strip()
|
||||||
|
asset.lifecycle_revision += 1
|
||||||
|
session.add(asset)
|
||||||
|
session.flush()
|
||||||
|
return asset
|
||||||
|
|
||||||
|
|
||||||
|
def get_asset_for_lifecycle(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
user_id: str,
|
||||||
|
asset_id: str,
|
||||||
|
is_admin: bool,
|
||||||
|
) -> FileAsset:
|
||||||
|
asset = session.get(FileAsset, asset_id)
|
||||||
|
if asset is None or asset.tenant_id != tenant_id:
|
||||||
|
raise FileStorageError("File not found")
|
||||||
|
ensure_owner_access(
|
||||||
|
session,
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
owner_type=asset.owner_type,
|
||||||
|
owner_id=_asset_owner_id(asset),
|
||||||
|
user_id=user_id,
|
||||||
|
is_admin=is_admin,
|
||||||
|
)
|
||||||
|
return asset
|
||||||
|
|
||||||
|
|
||||||
|
def restore_asset(session: Session, asset: FileAsset) -> bool:
|
||||||
|
if asset.deleted_at is None:
|
||||||
|
return False
|
||||||
|
collision = _asset_owner_query(session, asset).filter(
|
||||||
|
FileAsset.id != asset.id,
|
||||||
|
FileAsset.display_path == asset.display_path,
|
||||||
|
FileAsset.deleted_at.is_(None),
|
||||||
|
).first()
|
||||||
|
if collision is not None:
|
||||||
|
raise FileStorageError(
|
||||||
|
"The file cannot be restored because its path is already in use"
|
||||||
|
)
|
||||||
|
asset.deleted_at = None
|
||||||
|
asset.lifecycle_revision += 1
|
||||||
|
session.add(asset)
|
||||||
|
session.flush()
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def restore_folder(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
owner_type: str,
|
||||||
|
owner_id: str,
|
||||||
|
user_id: str,
|
||||||
|
path: str,
|
||||||
|
recursive: bool,
|
||||||
|
is_admin: bool,
|
||||||
|
) -> tuple[int, int]:
|
||||||
|
ensure_owner_access(
|
||||||
|
session,
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
owner_type=owner_type,
|
||||||
|
owner_id=owner_id,
|
||||||
|
user_id=user_id,
|
||||||
|
is_admin=is_admin,
|
||||||
|
)
|
||||||
|
normalized = normalize_folder(path)
|
||||||
|
if not normalized:
|
||||||
|
raise FileStorageError("Folder path is required")
|
||||||
|
prefix = f"{normalized}/"
|
||||||
|
folders = _folder_owner_query(
|
||||||
|
session, tenant_id=tenant_id, owner_type=owner_type, owner_id=owner_id
|
||||||
|
).filter(FileFolder.deleted_at.is_not(None))
|
||||||
|
assets = _asset_owner_query_by_values(
|
||||||
|
session, tenant_id=tenant_id, owner_type=owner_type, owner_id=owner_id
|
||||||
|
).filter(FileAsset.deleted_at.is_not(None))
|
||||||
|
if recursive:
|
||||||
|
folders = folders.filter(
|
||||||
|
(FileFolder.path == normalized) | FileFolder.path.like(f"{prefix}%")
|
||||||
|
)
|
||||||
|
assets = assets.filter(FileAsset.display_path.like(f"{prefix}%"))
|
||||||
|
else:
|
||||||
|
folders = folders.filter(FileFolder.path == normalized)
|
||||||
|
assets = assets.filter(False)
|
||||||
|
folder_rows = folders.order_by(FileFolder.path.asc()).all()
|
||||||
|
asset_rows = assets.order_by(FileAsset.display_path.asc()).all()
|
||||||
|
if not folder_rows:
|
||||||
|
raise FileStorageError("Deleted folder not found")
|
||||||
|
for folder in folder_rows:
|
||||||
|
collision = _folder_owner_query(
|
||||||
|
session,
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
owner_type=owner_type,
|
||||||
|
owner_id=owner_id,
|
||||||
|
).filter(
|
||||||
|
FileFolder.id != folder.id,
|
||||||
|
FileFolder.path == folder.path,
|
||||||
|
FileFolder.deleted_at.is_(None),
|
||||||
|
).first()
|
||||||
|
if collision is not None:
|
||||||
|
raise FileStorageError(
|
||||||
|
f"The folder cannot be restored because its path is in use: {folder.path}"
|
||||||
|
)
|
||||||
|
for asset in asset_rows:
|
||||||
|
collision = _asset_owner_query(session, asset).filter(
|
||||||
|
FileAsset.id != asset.id,
|
||||||
|
FileAsset.display_path == asset.display_path,
|
||||||
|
FileAsset.deleted_at.is_(None),
|
||||||
|
).first()
|
||||||
|
if collision is not None:
|
||||||
|
raise FileStorageError(
|
||||||
|
f"A file cannot be restored because its path is in use: {asset.display_path}"
|
||||||
|
)
|
||||||
|
for folder in folder_rows:
|
||||||
|
folder.deleted_at = None
|
||||||
|
session.add(folder)
|
||||||
|
for asset in asset_rows:
|
||||||
|
asset.deleted_at = None
|
||||||
|
asset.lifecycle_revision += 1
|
||||||
|
session.add(asset)
|
||||||
|
session.flush()
|
||||||
|
return len(folder_rows), len(asset_rows)
|
||||||
|
|
||||||
|
|
||||||
|
def restore_connector_space(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
user_id: str,
|
||||||
|
space_id: str,
|
||||||
|
is_admin: bool,
|
||||||
|
) -> FileConnectorSpace:
|
||||||
|
space = session.get(FileConnectorSpace, space_id)
|
||||||
|
if space is None or space.tenant_id != tenant_id or space.deleted_at is None:
|
||||||
|
raise FileStorageError("Deleted connector space not found")
|
||||||
|
ensure_owner_access(
|
||||||
|
session,
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
owner_type=space.owner_type,
|
||||||
|
owner_id=connector_space_owner_id(space),
|
||||||
|
user_id=user_id,
|
||||||
|
is_admin=is_admin,
|
||||||
|
)
|
||||||
|
collision = _connector_space_owner_query(session, space).filter(
|
||||||
|
FileConnectorSpace.id != space.id,
|
||||||
|
FileConnectorSpace.label == space.label,
|
||||||
|
FileConnectorSpace.deleted_at.is_(None),
|
||||||
|
).first()
|
||||||
|
if collision is not None:
|
||||||
|
raise FileStorageError(
|
||||||
|
"The connector space cannot be restored because its label is in use"
|
||||||
|
)
|
||||||
|
space.deleted_at = None
|
||||||
|
space.is_active = True
|
||||||
|
session.add(space)
|
||||||
|
session.flush()
|
||||||
|
return space
|
||||||
|
|
||||||
|
|
||||||
|
def preview_asset_purge(
|
||||||
|
session: Session, *, tenant_id: str, file_ids: Iterable[str]
|
||||||
|
) -> PurgePreview:
|
||||||
|
normalized_ids = sorted(set(str(value).strip() for value in file_ids if str(value).strip()))
|
||||||
|
if not normalized_ids or len(normalized_ids) > 100:
|
||||||
|
raise FileStorageError("Purge preview must contain between 1 and 100 files")
|
||||||
|
assets = (
|
||||||
|
session.query(FileAsset)
|
||||||
|
.filter(FileAsset.tenant_id == tenant_id, FileAsset.id.in_(normalized_ids))
|
||||||
|
.order_by(FileAsset.id.asc())
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
by_id = {asset.id: asset for asset in assets}
|
||||||
|
items: list[PurgePreviewItem] = []
|
||||||
|
for file_id in normalized_ids:
|
||||||
|
asset = by_id.get(file_id)
|
||||||
|
if asset is None:
|
||||||
|
items.append(
|
||||||
|
PurgePreviewItem(
|
||||||
|
file_id=file_id,
|
||||||
|
filename="",
|
||||||
|
lifecycle_revision=0,
|
||||||
|
deleted_at=None,
|
||||||
|
retained_until=None,
|
||||||
|
legal_hold=False,
|
||||||
|
blockers=("not_found",),
|
||||||
|
blob_ids=(),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
blockers = _asset_purge_blockers(session, asset)
|
||||||
|
blob_ids = tuple(
|
||||||
|
sorted(
|
||||||
|
row[0]
|
||||||
|
for row in session.query(FileVersion.blob_id)
|
||||||
|
.filter(FileVersion.file_asset_id == asset.id)
|
||||||
|
.distinct()
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
)
|
||||||
|
items.append(
|
||||||
|
PurgePreviewItem(
|
||||||
|
file_id=asset.id,
|
||||||
|
filename=asset.filename,
|
||||||
|
lifecycle_revision=asset.lifecycle_revision,
|
||||||
|
deleted_at=asset.deleted_at,
|
||||||
|
retained_until=asset.retained_until,
|
||||||
|
legal_hold=asset.legal_hold,
|
||||||
|
blockers=tuple(blockers),
|
||||||
|
blob_ids=blob_ids,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
payload = [item.digest_payload() for item in items]
|
||||||
|
digest = hashlib.sha256(
|
||||||
|
json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||||
|
).hexdigest()
|
||||||
|
return PurgePreview(preview_sha256=digest, items=tuple(items))
|
||||||
|
|
||||||
|
|
||||||
|
def execute_asset_purge(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
file_ids: Iterable[str],
|
||||||
|
preview_sha256: str,
|
||||||
|
idempotency_key: str,
|
||||||
|
approval_reference: str,
|
||||||
|
before_commit: Callable[[list[FileAsset]], None] | None = None,
|
||||||
|
) -> PurgeResult:
|
||||||
|
normalized_ids = sorted(set(str(value).strip() for value in file_ids if str(value).strip()))
|
||||||
|
if not normalized_ids or len(normalized_ids) > 100:
|
||||||
|
raise FileStorageError("Purge must contain between 1 and 100 files")
|
||||||
|
request = {
|
||||||
|
"tenant_id": tenant_id,
|
||||||
|
"file_ids": normalized_ids,
|
||||||
|
"preview_sha256": preview_sha256,
|
||||||
|
"approval_reference": approval_reference,
|
||||||
|
}
|
||||||
|
try:
|
||||||
|
started = begin_durable_recovery_operation(
|
||||||
|
get_database().SessionLocal,
|
||||||
|
identity=process_runtime_identity(),
|
||||||
|
module_id="files",
|
||||||
|
operation_type="asset-hard-purge",
|
||||||
|
idempotency_key=f"files-asset-purge:{idempotency_key}",
|
||||||
|
request=request,
|
||||||
|
recovery_plan=RecoveryPlan(
|
||||||
|
mode=RecoveryMode.IRREVERSIBLE,
|
||||||
|
preconditions=(
|
||||||
|
"the actor holds the dedicated Files purge permission",
|
||||||
|
"the signed-off preview still matches current lifecycle state",
|
||||||
|
"every target is soft-deleted and free of retention or legal-hold blockers",
|
||||||
|
),
|
||||||
|
verification_steps=(
|
||||||
|
"verify every target FileAsset and FileVersion row is absent",
|
||||||
|
"recalculate retained FileBlob reference counts before later garbage collection",
|
||||||
|
),
|
||||||
|
approval_reference=approval_reference,
|
||||||
|
),
|
||||||
|
precondition_evidence={
|
||||||
|
"preview_sha256": preview_sha256,
|
||||||
|
"target_count": len(normalized_ids),
|
||||||
|
},
|
||||||
|
lease_resource_key=f"files:purge:{tenant_id}",
|
||||||
|
lease_ttl_seconds=15 * 60,
|
||||||
|
resource_type="file_asset_batch",
|
||||||
|
resource_id=preview_sha256,
|
||||||
|
metadata={"resources": ["postgresql"], "bounded_target_count": len(normalized_ids)},
|
||||||
|
block_unresolved_resource=True,
|
||||||
|
)
|
||||||
|
except (RecoveryOperationBusy, RecoveryOperationStateConflict) as exc:
|
||||||
|
raise FileStorageError(
|
||||||
|
"Another purge or unresolved retry owns the tenant purge fence"
|
||||||
|
) from exc
|
||||||
|
except (RecoveryGuaranteeError, RuntimeError, ValueError) as exc:
|
||||||
|
raise FileStorageError("The Files recovery ledger is unavailable; nothing was purged") from exc
|
||||||
|
if started.replayed or started.operation is None:
|
||||||
|
return PurgeResult(
|
||||||
|
recovery_operation_id=started.operation_id,
|
||||||
|
status=started.status,
|
||||||
|
replayed=True,
|
||||||
|
purged_files=0,
|
||||||
|
released_blobs=0,
|
||||||
|
)
|
||||||
|
operation = started.operation
|
||||||
|
try:
|
||||||
|
preview = preview_asset_purge(session, tenant_id=tenant_id, file_ids=normalized_ids)
|
||||||
|
if preview.preview_sha256 != preview_sha256:
|
||||||
|
operation.reject(
|
||||||
|
summary="The purge preview became stale before execution",
|
||||||
|
evidence=_verified_evidence(
|
||||||
|
{"preview_matches": False},
|
||||||
|
expected_preview_sha256=preview_sha256,
|
||||||
|
observed_preview_sha256=preview.preview_sha256,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
raise FileStorageError("The purge preview is stale; preview again")
|
||||||
|
if not preview.eligible:
|
||||||
|
operation.reject(
|
||||||
|
summary="The purge was blocked by current lifecycle policy",
|
||||||
|
evidence=_verified_evidence(
|
||||||
|
{"lifecycle_policy_allows": False},
|
||||||
|
blocked_files=[
|
||||||
|
{"file_id": item.file_id, "blockers": list(item.blockers)}
|
||||||
|
for item in preview.items
|
||||||
|
if item.blockers
|
||||||
|
]
|
||||||
|
),
|
||||||
|
)
|
||||||
|
raise FileStorageError("One or more files are not eligible for purge")
|
||||||
|
assets = (
|
||||||
|
session.query(FileAsset)
|
||||||
|
.filter(FileAsset.tenant_id == tenant_id, FileAsset.id.in_(normalized_ids))
|
||||||
|
.with_for_update()
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
blob_ids = sorted({blob_id for item in preview.items for blob_id in item.blob_ids})
|
||||||
|
if before_commit is not None:
|
||||||
|
before_commit(assets)
|
||||||
|
session.query(FileShare).filter(
|
||||||
|
FileShare.file_asset_id.in_(normalized_ids)
|
||||||
|
).delete(synchronize_session=False)
|
||||||
|
session.query(FileVersion).filter(
|
||||||
|
FileVersion.file_asset_id.in_(normalized_ids)
|
||||||
|
).delete(synchronize_session=False)
|
||||||
|
for asset in assets:
|
||||||
|
session.delete(asset)
|
||||||
|
session.flush()
|
||||||
|
released_blobs = 0
|
||||||
|
for blob_id in blob_ids:
|
||||||
|
references = int(
|
||||||
|
session.query(func.count(FileVersion.id))
|
||||||
|
.filter(FileVersion.blob_id == blob_id)
|
||||||
|
.scalar()
|
||||||
|
or 0
|
||||||
|
)
|
||||||
|
blob = session.get(FileBlob, blob_id)
|
||||||
|
if blob is not None:
|
||||||
|
blob.ref_count = references
|
||||||
|
session.add(blob)
|
||||||
|
if references == 0:
|
||||||
|
released_blobs += 1
|
||||||
|
operation.commit_verified_success(
|
||||||
|
session,
|
||||||
|
evidence=_verified_evidence(
|
||||||
|
{
|
||||||
|
"assets_absent": True,
|
||||||
|
"versions_absent": True,
|
||||||
|
"blob_reference_counts_recalculated": True,
|
||||||
|
},
|
||||||
|
purged_file_ids=normalized_ids,
|
||||||
|
purged_files=len(assets),
|
||||||
|
released_blobs=released_blobs,
|
||||||
|
preview_sha256=preview_sha256,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
return PurgeResult(
|
||||||
|
recovery_operation_id=started.operation_id,
|
||||||
|
status=RecoveryStatus.SUCCEEDED.value,
|
||||||
|
replayed=False,
|
||||||
|
purged_files=len(assets),
|
||||||
|
released_blobs=released_blobs,
|
||||||
|
)
|
||||||
|
except FileStorageError:
|
||||||
|
session.rollback()
|
||||||
|
if not operation.closed:
|
||||||
|
operation.release_unresolved()
|
||||||
|
raise
|
||||||
|
except Exception as exc:
|
||||||
|
session.rollback()
|
||||||
|
if not operation.closed:
|
||||||
|
operation.reject(
|
||||||
|
summary="The purge failed before its database transaction committed",
|
||||||
|
evidence=_verified_evidence(
|
||||||
|
{"database_transaction_committed": False},
|
||||||
|
exception_type=type(exc).__name__,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def garbage_collect_unreferenced_blobs(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
limit: int,
|
||||||
|
approval_reference: str,
|
||||||
|
) -> BlobGcResult:
|
||||||
|
candidate_rows = (
|
||||||
|
session.query(FileBlob)
|
||||||
|
.filter(FileBlob.tenant_id == tenant_id)
|
||||||
|
.filter(
|
||||||
|
or_(FileBlob.retained_until.is_(None), FileBlob.retained_until <= utcnow())
|
||||||
|
)
|
||||||
|
.filter(
|
||||||
|
~session.query(FileVersion.id)
|
||||||
|
.filter(FileVersion.blob_id == FileBlob.id)
|
||||||
|
.exists()
|
||||||
|
)
|
||||||
|
.order_by(FileBlob.created_at.asc(), FileBlob.id.asc())
|
||||||
|
.limit(limit)
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
candidates = tuple(
|
||||||
|
(row.id, row.storage_key) for row in candidate_rows
|
||||||
|
)
|
||||||
|
# Close the read snapshot before the independent recovery transaction takes
|
||||||
|
# its durable SQLite/PostgreSQL write lock.
|
||||||
|
session.commit()
|
||||||
|
deleted = 0
|
||||||
|
unresolved: list[str] = []
|
||||||
|
backend = get_storage_backend()
|
||||||
|
for candidate_id, storage_key in candidates:
|
||||||
|
key_digest = hashlib.sha256(storage_key.encode("utf-8")).hexdigest()
|
||||||
|
request = {
|
||||||
|
"tenant_id": tenant_id,
|
||||||
|
"blob_id": candidate_id,
|
||||||
|
"storage_key_sha256": key_digest,
|
||||||
|
"approval_reference": approval_reference,
|
||||||
|
}
|
||||||
|
try:
|
||||||
|
started = begin_durable_recovery_operation(
|
||||||
|
get_database().SessionLocal,
|
||||||
|
identity=process_runtime_identity(),
|
||||||
|
module_id="files",
|
||||||
|
operation_type="blob-garbage-collection",
|
||||||
|
idempotency_key=(
|
||||||
|
f"files-blob-gc:{candidate_id}:{key_digest[:12]}:"
|
||||||
|
f"{hashlib.sha256(approval_reference.encode('utf-8')).hexdigest()[:12]}"
|
||||||
|
),
|
||||||
|
request=request,
|
||||||
|
recovery_plan=RecoveryPlan(
|
||||||
|
mode=RecoveryMode.FORWARD_RECOVERY,
|
||||||
|
preconditions=(
|
||||||
|
"the caller holds dedicated Files purge authority",
|
||||||
|
"a fresh database check shows no FileVersion reference",
|
||||||
|
),
|
||||||
|
forward_recovery_steps=(
|
||||||
|
"verify the object is absent",
|
||||||
|
"delete the unreferenced FileBlob row after absence is proven",
|
||||||
|
),
|
||||||
|
verification_steps=(
|
||||||
|
"recheck FileVersion references while holding the blob fence",
|
||||||
|
"probe the exact managed storage key after deletion",
|
||||||
|
),
|
||||||
|
approval_reference=approval_reference,
|
||||||
|
),
|
||||||
|
precondition_evidence={
|
||||||
|
"blob_id": candidate_id,
|
||||||
|
"storage_key_sha256": key_digest,
|
||||||
|
"observed_reference_count": 0,
|
||||||
|
},
|
||||||
|
lease_resource_key=f"files:blob:{tenant_id}:{candidate_id}",
|
||||||
|
lease_ttl_seconds=15 * 60,
|
||||||
|
resource_type="file_blob",
|
||||||
|
resource_id=candidate_id,
|
||||||
|
metadata={"resources": ["postgresql", "object-storage"], "storage_backend": backend.name},
|
||||||
|
block_unresolved_resource=True,
|
||||||
|
)
|
||||||
|
except (RecoveryOperationBusy, RecoveryOperationStateConflict):
|
||||||
|
continue
|
||||||
|
except (RecoveryGuaranteeError, RuntimeError, ValueError) as exc:
|
||||||
|
raise FileStorageError(
|
||||||
|
"The Files recovery ledger is unavailable; no blob bytes were deleted"
|
||||||
|
) from exc
|
||||||
|
if started.replayed or started.operation is None:
|
||||||
|
continue
|
||||||
|
operation = started.operation
|
||||||
|
try:
|
||||||
|
blob = (
|
||||||
|
session.query(FileBlob)
|
||||||
|
.filter(FileBlob.id == candidate_id, FileBlob.tenant_id == tenant_id)
|
||||||
|
.with_for_update()
|
||||||
|
.one_or_none()
|
||||||
|
)
|
||||||
|
references = int(
|
||||||
|
session.query(func.count(FileVersion.id))
|
||||||
|
.filter(FileVersion.blob_id == candidate_id)
|
||||||
|
.scalar()
|
||||||
|
or 0
|
||||||
|
)
|
||||||
|
blob_retained = bool(
|
||||||
|
blob is not None
|
||||||
|
and blob.retained_until is not None
|
||||||
|
and _as_utc(blob.retained_until) > utcnow()
|
||||||
|
)
|
||||||
|
except Exception as exc:
|
||||||
|
session.rollback()
|
||||||
|
if not operation.closed:
|
||||||
|
operation.reject(
|
||||||
|
summary="Blob eligibility could not be rechecked before deletion",
|
||||||
|
evidence=_verified_evidence(
|
||||||
|
{"external_effect_started": False},
|
||||||
|
exception_type=type(exc).__name__,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
raise FileStorageError(
|
||||||
|
"Blob eligibility could not be rechecked; no bytes were deleted"
|
||||||
|
) from exc
|
||||||
|
if blob is None or references or blob_retained:
|
||||||
|
operation.reject(
|
||||||
|
summary="Blob garbage collection was no longer eligible",
|
||||||
|
evidence=_verified_evidence(
|
||||||
|
{
|
||||||
|
"blob_present": blob is not None,
|
||||||
|
"reference_count_zero": references == 0,
|
||||||
|
"blob_retention_expired": not blob_retained,
|
||||||
|
},
|
||||||
|
reference_count=references,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
session.rollback()
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
backend.delete(blob.storage_key)
|
||||||
|
object_present = backend.exists(blob.storage_key)
|
||||||
|
except (StorageBackendError, OSError) as exc:
|
||||||
|
try:
|
||||||
|
object_present = backend.exists(blob.storage_key)
|
||||||
|
except (StorageBackendError, OSError):
|
||||||
|
object_present = None
|
||||||
|
session.rollback()
|
||||||
|
if object_present is True:
|
||||||
|
operation.fail(
|
||||||
|
summary="The unreferenced blob object was retained",
|
||||||
|
evidence={"object_present": True, "exception_type": type(exc).__name__},
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
operation.unresolved(
|
||||||
|
status=RecoveryStatus.OUTCOME_UNKNOWN,
|
||||||
|
summary="The blob deletion outcome could not be verified",
|
||||||
|
evidence={"object_present": object_present, "exception_type": type(exc).__name__},
|
||||||
|
failure_summary="Reconcile the exact blob key before retrying garbage collection",
|
||||||
|
)
|
||||||
|
unresolved.append(started.operation_id)
|
||||||
|
continue
|
||||||
|
if object_present:
|
||||||
|
session.rollback()
|
||||||
|
operation.fail(
|
||||||
|
summary="The object store did not delete the unreferenced blob",
|
||||||
|
evidence={"object_present": True},
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
session.delete(blob)
|
||||||
|
session.flush()
|
||||||
|
operation.commit_verified_success(
|
||||||
|
session,
|
||||||
|
evidence=_verified_evidence(
|
||||||
|
{
|
||||||
|
"object_absent": True,
|
||||||
|
"database_blob_absent": True,
|
||||||
|
"reference_count_zero": True,
|
||||||
|
},
|
||||||
|
blob_id=candidate_id,
|
||||||
|
storage_key_sha256=key_digest,
|
||||||
|
object_present=False,
|
||||||
|
database_blob_present=False,
|
||||||
|
reference_count=0,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
deleted += 1
|
||||||
|
except Exception as exc:
|
||||||
|
session.rollback()
|
||||||
|
if not operation.closed:
|
||||||
|
operation.unresolved(
|
||||||
|
status=RecoveryStatus.RECOVERY_REQUIRED,
|
||||||
|
summary="The object is absent but blob metadata still requires forward recovery",
|
||||||
|
evidence={"object_present": False, "exception_type": type(exc).__name__},
|
||||||
|
failure_summary="Delete the FileBlob row only after rechecking all FileVersion references",
|
||||||
|
)
|
||||||
|
unresolved.append(started.operation_id)
|
||||||
|
return BlobGcResult(
|
||||||
|
inspected_blobs=len(candidates),
|
||||||
|
deleted_blobs=deleted,
|
||||||
|
unresolved_operation_ids=tuple(unresolved),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _asset_purge_blockers(session: Session, asset: FileAsset) -> list[str]:
|
||||||
|
blockers: list[str] = []
|
||||||
|
if asset.deleted_at is None:
|
||||||
|
blockers.append("not_soft_deleted")
|
||||||
|
if asset.legal_hold:
|
||||||
|
blockers.append("legal_hold")
|
||||||
|
if asset.retained_until is not None and _as_utc(asset.retained_until) > utcnow():
|
||||||
|
blockers.append("retention_active")
|
||||||
|
if _table_exists(session, CampaignAttachmentUse.__tablename__) and session.query(
|
||||||
|
CampaignAttachmentUse.id
|
||||||
|
).filter(CampaignAttachmentUse.file_asset_id == asset.id).first() is not None:
|
||||||
|
blockers.append("campaign_evidence")
|
||||||
|
if _table_exists(session, FileFormEvidenceGrant.__tablename__) and session.query(
|
||||||
|
FileFormEvidenceGrant.id
|
||||||
|
).filter(FileFormEvidenceGrant.file_asset_id == asset.id).first() is not None:
|
||||||
|
blockers.append("form_evidence")
|
||||||
|
if session.query(FileShare.id).filter(
|
||||||
|
FileShare.file_asset_id == asset.id,
|
||||||
|
effective_file_share_clause(),
|
||||||
|
).first() is not None:
|
||||||
|
blockers.append("active_share")
|
||||||
|
return blockers
|
||||||
|
|
||||||
|
|
||||||
|
def _asset_owner_id(asset: FileAsset) -> str:
|
||||||
|
owner_id = asset.owner_user_id if asset.owner_type == "user" else asset.owner_group_id
|
||||||
|
if not owner_id:
|
||||||
|
raise FileStorageError("File has no valid owner")
|
||||||
|
return owner_id
|
||||||
|
|
||||||
|
|
||||||
|
def _asset_owner_query(session: Session, asset: FileAsset):
|
||||||
|
return _asset_owner_query_by_values(
|
||||||
|
session,
|
||||||
|
tenant_id=asset.tenant_id,
|
||||||
|
owner_type=asset.owner_type,
|
||||||
|
owner_id=_asset_owner_id(asset),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _asset_owner_query_by_values(
|
||||||
|
session: Session, *, tenant_id: str, owner_type: str, owner_id: str
|
||||||
|
):
|
||||||
|
query = session.query(FileAsset).filter(
|
||||||
|
FileAsset.tenant_id == tenant_id, FileAsset.owner_type == owner_type
|
||||||
|
)
|
||||||
|
if owner_type == "user":
|
||||||
|
return query.filter(FileAsset.owner_user_id == owner_id)
|
||||||
|
if owner_type == "group":
|
||||||
|
return query.filter(FileAsset.owner_group_id == owner_id)
|
||||||
|
raise FileStorageError("Files must be owned by a user or group")
|
||||||
|
|
||||||
|
|
||||||
|
def _folder_owner_query(
|
||||||
|
session: Session, *, tenant_id: str, owner_type: str, owner_id: str
|
||||||
|
):
|
||||||
|
query = session.query(FileFolder).filter(
|
||||||
|
FileFolder.tenant_id == tenant_id, FileFolder.owner_type == owner_type
|
||||||
|
)
|
||||||
|
if owner_type == "user":
|
||||||
|
return query.filter(FileFolder.owner_user_id == owner_id)
|
||||||
|
if owner_type == "group":
|
||||||
|
return query.filter(FileFolder.owner_group_id == owner_id)
|
||||||
|
raise FileStorageError("Folders must be owned by a user or group")
|
||||||
|
|
||||||
|
|
||||||
|
def _connector_space_owner_query(session: Session, space: FileConnectorSpace):
|
||||||
|
query = session.query(FileConnectorSpace).filter(
|
||||||
|
FileConnectorSpace.tenant_id == space.tenant_id,
|
||||||
|
FileConnectorSpace.owner_type == space.owner_type,
|
||||||
|
)
|
||||||
|
owner_id = connector_space_owner_id(space)
|
||||||
|
if space.owner_type == "user":
|
||||||
|
return query.filter(FileConnectorSpace.owner_user_id == owner_id)
|
||||||
|
return query.filter(FileConnectorSpace.owner_group_id == owner_id)
|
||||||
|
|
||||||
|
|
||||||
|
def _as_utc(value: datetime) -> datetime:
|
||||||
|
return value.replace(tzinfo=UTC) if value.tzinfo is None else value.astimezone(UTC)
|
||||||
|
|
||||||
|
|
||||||
|
def _iso(value: datetime | None) -> str | None:
|
||||||
|
return _as_utc(value).isoformat() if value is not None else None
|
||||||
|
|
||||||
|
|
||||||
|
def _table_exists(session: Session, table_name: str) -> bool:
|
||||||
|
return bool(session.bind is not None and inspect(session.bind).has_table(table_name))
|
||||||
|
|
||||||
|
|
||||||
|
def _verified_evidence(
|
||||||
|
checks: dict[str, object], **details: object
|
||||||
|
) -> dict[str, object]:
|
||||||
|
return {"verified": True, "checks": checks, **details}
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"BlobGcResult",
|
||||||
|
"PurgePreview",
|
||||||
|
"PurgePreviewItem",
|
||||||
|
"PurgeResult",
|
||||||
|
"execute_asset_purge",
|
||||||
|
"garbage_collect_unreferenced_blobs",
|
||||||
|
"get_asset_for_lifecycle",
|
||||||
|
"preview_asset_purge",
|
||||||
|
"restore_asset",
|
||||||
|
"restore_connector_space",
|
||||||
|
"restore_folder",
|
||||||
|
"set_asset_lifecycle",
|
||||||
|
]
|
||||||
@@ -1,26 +1,39 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from collections.abc import Callable
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from datetime import UTC, datetime
|
from datetime import UTC, datetime
|
||||||
import hashlib
|
import hashlib
|
||||||
|
from threading import Lock
|
||||||
from typing import Protocol
|
from typing import Protocol
|
||||||
|
|
||||||
from sqlalchemy import event
|
from sqlalchemy import event, select
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from govoplan_core.core.recovery import (
|
from govoplan_core.core.recovery import (
|
||||||
RecoveryGuaranteeError,
|
RecoveryGuaranteeError,
|
||||||
RecoveryMode,
|
RecoveryMode,
|
||||||
|
RecoveryOperation,
|
||||||
RecoveryPlan,
|
RecoveryPlan,
|
||||||
RecoveryStatus,
|
RecoveryStatus,
|
||||||
|
plan_recovery_operation,
|
||||||
|
prepare_recovery_operation,
|
||||||
|
start_recovery_operation,
|
||||||
|
verify_recovery_evidence_chain,
|
||||||
)
|
)
|
||||||
from govoplan_core.core.recovery_runtime import (
|
from govoplan_core.core.recovery_runtime import (
|
||||||
DurableRecoveryOperation,
|
DurableRecoveryOperation,
|
||||||
|
DurableRecoveryStart,
|
||||||
RecoveryOperationBusy,
|
RecoveryOperationBusy,
|
||||||
RecoveryOperationStateConflict,
|
RecoveryOperationStateConflict,
|
||||||
begin_durable_recovery_operation,
|
begin_durable_recovery_operation,
|
||||||
)
|
)
|
||||||
from govoplan_core.core.runtime_coordination import process_runtime_identity
|
from govoplan_core.core.runtime_coordination import (
|
||||||
|
RuntimeIdentity,
|
||||||
|
acquire_lease,
|
||||||
|
process_runtime_identity,
|
||||||
|
release_lease,
|
||||||
|
)
|
||||||
from govoplan_core.db.session import get_database
|
from govoplan_core.db.session import get_database
|
||||||
from govoplan_files.backend.db.models import (
|
from govoplan_files.backend.db.models import (
|
||||||
FileBlob,
|
FileBlob,
|
||||||
@@ -38,6 +51,8 @@ from govoplan_files.backend.storage.common import FileStorageError
|
|||||||
_PENDING_EFFECTS_KEY = "govoplan_files_pending_recovery_effects"
|
_PENDING_EFFECTS_KEY = "govoplan_files_pending_recovery_effects"
|
||||||
_HOOKS_INSTALLED_KEY = "govoplan_files_recovery_hooks_installed"
|
_HOOKS_INSTALLED_KEY = "govoplan_files_recovery_hooks_installed"
|
||||||
_ROLLBACK_ERRORS_KEY = "govoplan_files_recovery_rollback_errors"
|
_ROLLBACK_ERRORS_KEY = "govoplan_files_recovery_rollback_errors"
|
||||||
|
_SQLITE_FENCES: set[str] = set()
|
||||||
|
_SQLITE_FENCES_LOCK = Lock()
|
||||||
|
|
||||||
|
|
||||||
class _PendingEffect(Protocol):
|
class _PendingEffect(Protocol):
|
||||||
@@ -58,6 +73,8 @@ class PendingBlobWrite:
|
|||||||
expected_storage_checksum_sha256: str | None = None
|
expected_storage_checksum_sha256: str | None = None
|
||||||
expected_storage_size_bytes: int | None = None
|
expected_storage_size_bytes: int | None = None
|
||||||
expected_envelope_id: str | None = None
|
expected_envelope_id: str | None = None
|
||||||
|
sqlite_fence_key: str | None = None
|
||||||
|
restart_after_rollback: Callable[[], DurableRecoveryOperation] | None = None
|
||||||
|
|
||||||
def prepare_stored_bytes(
|
def prepare_stored_bytes(
|
||||||
self,
|
self,
|
||||||
@@ -72,6 +89,19 @@ class PendingBlobWrite:
|
|||||||
self.expected_envelope_id = envelope_id
|
self.expected_envelope_id = envelope_id
|
||||||
|
|
||||||
def settle(self, *, committed: bool) -> None:
|
def settle(self, *, committed: bool) -> None:
|
||||||
|
if not committed and self.restart_after_rollback is not None:
|
||||||
|
try:
|
||||||
|
self.operation = self.restart_after_rollback()
|
||||||
|
except Exception:
|
||||||
|
# SQLite cannot make the pre-effect ledger row independent of
|
||||||
|
# the caller transaction. If reconstructing the evidence row
|
||||||
|
# also fails, still avoid leaving a known new orphan behind.
|
||||||
|
if self.created_new:
|
||||||
|
try:
|
||||||
|
self.backend.delete(self.storage_key)
|
||||||
|
except (StorageBackendError, OSError):
|
||||||
|
pass
|
||||||
|
raise
|
||||||
evidence = _blob_write_evidence(self)
|
evidence = _blob_write_evidence(self)
|
||||||
if _blob_write_complete(evidence):
|
if _blob_write_complete(evidence):
|
||||||
self.operation.succeed(evidence=evidence)
|
self.operation.succeed(evidence=evidence)
|
||||||
@@ -219,12 +249,12 @@ def begin_blob_write_recovery(
|
|||||||
f"files-blob-{disposition}:{blob_id}:{state_token[:48]}"
|
f"files-blob-{disposition}:{blob_id}:{state_token[:48]}"
|
||||||
)
|
)
|
||||||
try:
|
try:
|
||||||
started = begin_durable_recovery_operation(
|
identity = process_runtime_identity()
|
||||||
get_database().SessionLocal,
|
except RuntimeError as exc:
|
||||||
identity=process_runtime_identity(),
|
raise FileStorageError(
|
||||||
module_id="files",
|
"The Files recovery ledger is unavailable; no object was written"
|
||||||
operation_type=f"blob-{disposition}",
|
) from exc
|
||||||
idempotency_key=idempotency_key,
|
lease_resource_key = f"files:blob:{tenant_id}:{blob_id}"
|
||||||
request = {
|
request = {
|
||||||
"tenant_id": tenant_id,
|
"tenant_id": tenant_id,
|
||||||
"blob_id": blob_id,
|
"blob_id": blob_id,
|
||||||
@@ -234,7 +264,7 @@ def begin_blob_write_recovery(
|
|||||||
"semantic_size_bytes": semantic_size_bytes,
|
"semantic_size_bytes": semantic_size_bytes,
|
||||||
"protection_discriminator": protection_discriminator,
|
"protection_discriminator": protection_discriminator,
|
||||||
"disposition": disposition,
|
"disposition": disposition,
|
||||||
},
|
}
|
||||||
recovery_plan = RecoveryPlan(
|
recovery_plan = RecoveryPlan(
|
||||||
mode=(
|
mode=(
|
||||||
RecoveryMode.COMPENSATION
|
RecoveryMode.COMPENSATION
|
||||||
@@ -262,34 +292,102 @@ def begin_blob_write_recovery(
|
|||||||
"reload FileBlob metadata through an independent session",
|
"reload FileBlob metadata through an independent session",
|
||||||
"stream and hash the managed object independently",
|
"stream and hash the managed object independently",
|
||||||
),
|
),
|
||||||
),
|
)
|
||||||
precondition_evidence = {
|
precondition_evidence = {
|
||||||
"blob_id": blob_id,
|
"blob_id": blob_id,
|
||||||
"storage_key_sha256": key_digest,
|
"storage_key_sha256": key_digest,
|
||||||
"semantic_checksum_sha256": semantic_checksum_sha256,
|
"semantic_checksum_sha256": semantic_checksum_sha256,
|
||||||
"semantic_size_bytes": semantic_size_bytes,
|
"semantic_size_bytes": semantic_size_bytes,
|
||||||
"created_new": created_new,
|
"created_new": created_new,
|
||||||
},
|
}
|
||||||
lease_resource_key=(
|
metadata = {
|
||||||
f"files:blob:{tenant_id}:{blob_id}"
|
"resources": ["postgresql", "object-storage"],
|
||||||
|
"storage_backend": backend.name,
|
||||||
|
"durability_mode": "independent_transaction",
|
||||||
|
}
|
||||||
|
session_factory = get_database().SessionLocal
|
||||||
|
sqlite_mode = session.get_bind().dialect.name == "sqlite"
|
||||||
|
sqlite_fence_key: str | None = None
|
||||||
|
|
||||||
|
def start_independent(*, reconstructed_after_rollback: bool = False) -> DurableRecoveryStart:
|
||||||
|
return begin_durable_recovery_operation(
|
||||||
|
session_factory,
|
||||||
|
identity=identity,
|
||||||
|
module_id="files",
|
||||||
|
operation_type=f"blob-{disposition}",
|
||||||
|
idempotency_key=idempotency_key,
|
||||||
|
request=request,
|
||||||
|
recovery_plan=recovery_plan,
|
||||||
|
precondition_evidence={
|
||||||
|
**precondition_evidence,
|
||||||
|
**(
|
||||||
|
{
|
||||||
|
"sqlite_caller_transaction_rolled_back": True,
|
||||||
|
"effect_may_have_preceded_durable_intent": True,
|
||||||
|
}
|
||||||
|
if reconstructed_after_rollback
|
||||||
|
else {}
|
||||||
),
|
),
|
||||||
|
},
|
||||||
|
lease_resource_key=lease_resource_key,
|
||||||
lease_ttl_seconds=15 * 60,
|
lease_ttl_seconds=15 * 60,
|
||||||
resource_type="file_blob",
|
resource_type="file_blob",
|
||||||
resource_id=blob_id,
|
resource_id=blob_id,
|
||||||
metadata={
|
metadata={
|
||||||
"resources": ["postgresql", "object-storage"],
|
**metadata,
|
||||||
"storage_backend": backend.name,
|
"durability_mode": (
|
||||||
|
"sqlite_post_rollback_reconstruction"
|
||||||
|
if reconstructed_after_rollback
|
||||||
|
else "independent_transaction"
|
||||||
|
),
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
if sqlite_mode:
|
||||||
|
_reserve_sqlite_fence(lease_resource_key)
|
||||||
|
sqlite_fence_key = lease_resource_key
|
||||||
|
started = _begin_caller_transaction_recovery_operation(
|
||||||
|
session,
|
||||||
|
session_factory=session_factory,
|
||||||
|
identity=identity,
|
||||||
|
module_id="files",
|
||||||
|
operation_type=f"blob-{disposition}",
|
||||||
|
idempotency_key=idempotency_key,
|
||||||
|
request=request,
|
||||||
|
recovery_plan=recovery_plan,
|
||||||
|
precondition_evidence={
|
||||||
|
**precondition_evidence,
|
||||||
|
"sqlite_caller_transaction": True,
|
||||||
|
"reduced_crash_durability": True,
|
||||||
|
},
|
||||||
|
lease_resource_key=lease_resource_key,
|
||||||
|
lease_ttl_seconds=15 * 60,
|
||||||
|
resource_type="file_blob",
|
||||||
|
resource_id=blob_id,
|
||||||
|
metadata={
|
||||||
|
**metadata,
|
||||||
|
"resources": ["sqlite", "object-storage"],
|
||||||
|
"durability_mode": "sqlite_caller_transaction",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
started = start_independent()
|
||||||
except (RecoveryOperationBusy, RecoveryOperationStateConflict) as exc:
|
except (RecoveryOperationBusy, RecoveryOperationStateConflict) as exc:
|
||||||
|
if sqlite_fence_key is not None:
|
||||||
|
_release_sqlite_fence(sqlite_fence_key)
|
||||||
raise FileStorageError(
|
raise FileStorageError(
|
||||||
"This managed blob is already owned by another recovery operation"
|
"This managed blob is already owned by another recovery operation"
|
||||||
) from exc
|
) from exc
|
||||||
except (RecoveryGuaranteeError, RuntimeError) as exc:
|
except (RecoveryGuaranteeError, RuntimeError) as exc:
|
||||||
|
if sqlite_fence_key is not None:
|
||||||
|
_release_sqlite_fence(sqlite_fence_key)
|
||||||
raise FileStorageError(
|
raise FileStorageError(
|
||||||
"The Files recovery ledger is unavailable; no object was written"
|
"The Files recovery ledger is unavailable; no object was written"
|
||||||
) from exc
|
) from exc
|
||||||
if started.replayed or started.operation is None:
|
if started.replayed or started.operation is None:
|
||||||
|
if sqlite_fence_key is not None:
|
||||||
|
_release_sqlite_fence(sqlite_fence_key)
|
||||||
raise FileStorageError(
|
raise FileStorageError(
|
||||||
"The matching Files blob operation was already completed; reload before retrying"
|
"The matching Files blob operation was already completed; reload before retrying"
|
||||||
)
|
)
|
||||||
@@ -303,6 +401,14 @@ def begin_blob_write_recovery(
|
|||||||
semantic_size_bytes=semantic_size_bytes,
|
semantic_size_bytes=semantic_size_bytes,
|
||||||
protection_discriminator=protection_discriminator,
|
protection_discriminator=protection_discriminator,
|
||||||
created_new=created_new,
|
created_new=created_new,
|
||||||
|
sqlite_fence_key=sqlite_fence_key,
|
||||||
|
restart_after_rollback=(
|
||||||
|
lambda: _require_started_operation(
|
||||||
|
start_independent(reconstructed_after_rollback=True)
|
||||||
|
)
|
||||||
|
if sqlite_mode
|
||||||
|
else None
|
||||||
|
),
|
||||||
)
|
)
|
||||||
_register_pending_effect(session, pending)
|
_register_pending_effect(session, pending)
|
||||||
return pending
|
return pending
|
||||||
@@ -398,10 +504,17 @@ def _register_pending_effect(session: Session, effect: _PendingEffect) -> None:
|
|||||||
|
|
||||||
|
|
||||||
def _after_session_commit(session: Session) -> None:
|
def _after_session_commit(session: Session) -> None:
|
||||||
|
# SQLAlchemy also emits after_commit for a released SAVEPOINT. A Files
|
||||||
|
# batch may open one while creating a later recovery row; settle only when
|
||||||
|
# the outer business transaction has actually become visible.
|
||||||
|
if session.in_nested_transaction():
|
||||||
|
return
|
||||||
_settle_pending_effects(session, committed=True)
|
_settle_pending_effects(session, committed=True)
|
||||||
|
|
||||||
|
|
||||||
def _after_session_rollback(session: Session) -> None:
|
def _after_session_rollback(session: Session) -> None:
|
||||||
|
if session.in_nested_transaction():
|
||||||
|
return
|
||||||
try:
|
try:
|
||||||
_settle_pending_effects(session, committed=False)
|
_settle_pending_effects(session, committed=False)
|
||||||
except RecoveryGuaranteeError as exc:
|
except RecoveryGuaranteeError as exc:
|
||||||
@@ -422,12 +535,139 @@ def _settle_pending_effects(session: Session, *, committed: bool) -> None:
|
|||||||
operation.release_unresolved()
|
operation.release_unresolved()
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
finally:
|
||||||
|
sqlite_fence_key = getattr(effect, "sqlite_fence_key", None)
|
||||||
|
if sqlite_fence_key:
|
||||||
|
_release_sqlite_fence(sqlite_fence_key)
|
||||||
if failures:
|
if failures:
|
||||||
raise RecoveryGuaranteeError(
|
raise RecoveryGuaranteeError(
|
||||||
f"{len(failures)} Files recovery operation(s) could not be finalized"
|
f"{len(failures)} Files recovery operation(s) could not be finalized"
|
||||||
) from failures[0]
|
) from failures[0]
|
||||||
|
|
||||||
|
|
||||||
|
def _reserve_sqlite_fence(resource_key: str) -> None:
|
||||||
|
with _SQLITE_FENCES_LOCK:
|
||||||
|
if resource_key in _SQLITE_FENCES:
|
||||||
|
raise RecoveryOperationBusy(
|
||||||
|
f"Another local SQLite transaction owns {resource_key}"
|
||||||
|
)
|
||||||
|
_SQLITE_FENCES.add(resource_key)
|
||||||
|
|
||||||
|
|
||||||
|
def _release_sqlite_fence(resource_key: str) -> None:
|
||||||
|
with _SQLITE_FENCES_LOCK:
|
||||||
|
_SQLITE_FENCES.discard(resource_key)
|
||||||
|
|
||||||
|
|
||||||
|
def _require_started_operation(started: DurableRecoveryStart) -> DurableRecoveryOperation:
|
||||||
|
if started.replayed or started.operation is None:
|
||||||
|
raise RecoveryOperationStateConflict(started.operation_id, started.status)
|
||||||
|
return started.operation
|
||||||
|
|
||||||
|
|
||||||
|
def _begin_caller_transaction_recovery_operation(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
session_factory: Callable[[], Session],
|
||||||
|
identity: RuntimeIdentity,
|
||||||
|
module_id: str,
|
||||||
|
operation_type: str,
|
||||||
|
idempotency_key: str,
|
||||||
|
request: dict[str, object],
|
||||||
|
recovery_plan: RecoveryPlan,
|
||||||
|
precondition_evidence: dict[str, object],
|
||||||
|
lease_resource_key: str,
|
||||||
|
lease_ttl_seconds: int,
|
||||||
|
resource_type: str,
|
||||||
|
resource_id: str,
|
||||||
|
metadata: dict[str, object],
|
||||||
|
) -> DurableRecoveryStart:
|
||||||
|
"""Start SQLite recovery evidence inside the caller transaction.
|
||||||
|
|
||||||
|
SQLite permits only one writer, so opening the normal independent ledger
|
||||||
|
transaction after an earlier member of a batch has written will deadlock
|
||||||
|
until the busy timeout. The caller-transaction mode preserves fencing,
|
||||||
|
request hashing, checkpoints, commit verification, and rollback
|
||||||
|
compensation, but cannot make the intent survive a hard process loss
|
||||||
|
before the caller commits. PostgreSQL never uses this reduced mode.
|
||||||
|
"""
|
||||||
|
|
||||||
|
claim = acquire_lease(
|
||||||
|
session,
|
||||||
|
installation_id=identity.installation_id,
|
||||||
|
resource_key=lease_resource_key,
|
||||||
|
holder_node_id=identity.node_id,
|
||||||
|
holder_incarnation=identity.incarnation,
|
||||||
|
ttl_seconds=lease_ttl_seconds,
|
||||||
|
metadata={
|
||||||
|
"module_id": module_id,
|
||||||
|
"operation_type": operation_type,
|
||||||
|
"durability_mode": "sqlite_caller_transaction",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
if claim is None:
|
||||||
|
raise RecoveryOperationBusy(
|
||||||
|
f"Another runtime owns the recovery fence for {lease_resource_key}"
|
||||||
|
)
|
||||||
|
existing = session.execute(
|
||||||
|
select(RecoveryOperation).where(
|
||||||
|
RecoveryOperation.installation_id == identity.installation_id,
|
||||||
|
RecoveryOperation.module_id == module_id,
|
||||||
|
RecoveryOperation.idempotency_key == idempotency_key,
|
||||||
|
)
|
||||||
|
).scalar_one_or_none()
|
||||||
|
operation = plan_recovery_operation(
|
||||||
|
session,
|
||||||
|
installation_id=identity.installation_id,
|
||||||
|
module_id=module_id,
|
||||||
|
operation_type=operation_type,
|
||||||
|
idempotency_key=idempotency_key,
|
||||||
|
request=request,
|
||||||
|
recovery_plan=recovery_plan,
|
||||||
|
resource_type=resource_type,
|
||||||
|
resource_id=resource_id,
|
||||||
|
lease_claim=claim,
|
||||||
|
metadata=metadata,
|
||||||
|
)
|
||||||
|
if existing is not None:
|
||||||
|
if operation.status == RecoveryStatus.SUCCEEDED.value:
|
||||||
|
release_lease(session, claim)
|
||||||
|
return DurableRecoveryStart(
|
||||||
|
operation_id=operation.id,
|
||||||
|
status=operation.status,
|
||||||
|
replayed=True,
|
||||||
|
operation=None,
|
||||||
|
)
|
||||||
|
raise RecoveryOperationStateConflict(operation.id, operation.status)
|
||||||
|
prepare_recovery_operation(
|
||||||
|
session,
|
||||||
|
operation,
|
||||||
|
evidence=precondition_evidence,
|
||||||
|
lease_claim=claim,
|
||||||
|
)
|
||||||
|
start_recovery_operation(
|
||||||
|
session,
|
||||||
|
operation,
|
||||||
|
evidence={"lease_resource_key": lease_resource_key},
|
||||||
|
lease_claim=claim,
|
||||||
|
)
|
||||||
|
if not verify_recovery_evidence_chain(session, operation.id):
|
||||||
|
raise RecoveryGuaranteeError(
|
||||||
|
"Recovery checkpoint chain verification failed before side effects"
|
||||||
|
)
|
||||||
|
return DurableRecoveryStart(
|
||||||
|
operation_id=operation.id,
|
||||||
|
status=operation.status,
|
||||||
|
replayed=False,
|
||||||
|
operation=DurableRecoveryOperation(
|
||||||
|
session_factory=session_factory,
|
||||||
|
operation_id=operation.id,
|
||||||
|
lease_claim=claim,
|
||||||
|
lease_ttl_seconds=lease_ttl_seconds,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _blob_write_evidence(effect: PendingBlobWrite) -> dict[str, object]:
|
def _blob_write_evidence(effect: PendingBlobWrite) -> dict[str, object]:
|
||||||
database_blob_present: bool | None
|
database_blob_present: bool | None
|
||||||
database_matches: bool | None
|
database_matches: bool | None
|
||||||
|
|||||||
@@ -0,0 +1,331 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import inspect
|
||||||
|
import socket
|
||||||
|
import threading
|
||||||
|
from functools import lru_cache
|
||||||
|
from importlib import import_module
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from govoplan_core.security.outbound_http import (
|
||||||
|
OutboundHttpError,
|
||||||
|
create_outbound_connection,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class SdkPeerPinningError(RuntimeError):
|
||||||
|
"""Raised when an optional SDK cannot be bound to the pinned transport."""
|
||||||
|
|
||||||
|
|
||||||
|
_BOTOCORE_CLIENT_CREATION_LOCK = threading.Lock()
|
||||||
|
|
||||||
|
|
||||||
|
def create_pinned_s3_client(**kwargs: Any) -> Any:
|
||||||
|
"""Construct an S3 client whose first and every later socket is pinned.
|
||||||
|
|
||||||
|
Botocore does not expose the HTTP-session class through boto3's public
|
||||||
|
client API. Its endpoint creator does expose that seam, so this function
|
||||||
|
replaces the creator only for the bounded client-construction operation.
|
||||||
|
The lock avoids an unsafe interleaving with another GovOPlaN client
|
||||||
|
construction. Any unrelated botocore client created during the short
|
||||||
|
replacement window also receives the stricter transport.
|
||||||
|
"""
|
||||||
|
|
||||||
|
try:
|
||||||
|
boto3 = import_module("boto3")
|
||||||
|
botocore_args = import_module("botocore.args")
|
||||||
|
except ImportError as exc:
|
||||||
|
raise SdkPeerPinningError(
|
||||||
|
"S3 connector browsing requires the optional boto3 dependency"
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
pinned_session_cls = _botocore_transport_types()[0]
|
||||||
|
original_creator = getattr(botocore_args, "EndpointCreator", None)
|
||||||
|
if original_creator is None or not hasattr(original_creator, "create_endpoint"):
|
||||||
|
raise SdkPeerPinningError(
|
||||||
|
"The installed botocore release does not expose the endpoint transport seam required for peer pinning"
|
||||||
|
)
|
||||||
|
|
||||||
|
class PinnedEndpointCreator(original_creator): # type: ignore[misc, valid-type]
|
||||||
|
def create_endpoint(self, *args: Any, **endpoint_kwargs: Any) -> Any:
|
||||||
|
endpoint_kwargs["http_session_cls"] = pinned_session_cls
|
||||||
|
return super().create_endpoint(*args, **endpoint_kwargs)
|
||||||
|
|
||||||
|
with _BOTOCORE_CLIENT_CREATION_LOCK:
|
||||||
|
current_creator = getattr(botocore_args, "EndpointCreator", None)
|
||||||
|
if current_creator is not original_creator:
|
||||||
|
raise SdkPeerPinningError(
|
||||||
|
"Botocore endpoint construction changed concurrently; refusing to create an unproven S3 client"
|
||||||
|
)
|
||||||
|
botocore_args.EndpointCreator = PinnedEndpointCreator
|
||||||
|
try:
|
||||||
|
session_type = getattr(getattr(boto3, "session", None), "Session", None)
|
||||||
|
if session_type is None:
|
||||||
|
raise SdkPeerPinningError(
|
||||||
|
"The installed boto3 release does not expose its isolated session constructor"
|
||||||
|
)
|
||||||
|
client = session_type().client("s3", **kwargs)
|
||||||
|
finally:
|
||||||
|
if getattr(botocore_args, "EndpointCreator", None) is PinnedEndpointCreator:
|
||||||
|
botocore_args.EndpointCreator = original_creator
|
||||||
|
|
||||||
|
http_session = getattr(getattr(client, "_endpoint", None), "http_session", None)
|
||||||
|
if http_session is None or not isinstance(http_session, pinned_session_cls):
|
||||||
|
close = getattr(client, "close", None)
|
||||||
|
if callable(close):
|
||||||
|
close()
|
||||||
|
raise SdkPeerPinningError(
|
||||||
|
"Botocore did not install the required pinned HTTP transport; the S3 client was discarded"
|
||||||
|
)
|
||||||
|
return client
|
||||||
|
|
||||||
|
|
||||||
|
@lru_cache(maxsize=1)
|
||||||
|
def _botocore_transport_types() -> tuple[type[Any], type[Any], type[Any]]:
|
||||||
|
try:
|
||||||
|
awsrequest = import_module("botocore.awsrequest")
|
||||||
|
httpsession = import_module("botocore.httpsession")
|
||||||
|
urllib3_exceptions = import_module("urllib3.exceptions")
|
||||||
|
except ImportError as exc:
|
||||||
|
raise SdkPeerPinningError(
|
||||||
|
"S3 connector browsing requires a compatible botocore HTTP transport"
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
required = (
|
||||||
|
"AWSHTTPConnection",
|
||||||
|
"AWSHTTPSConnection",
|
||||||
|
"AWSHTTPConnectionPool",
|
||||||
|
"AWSHTTPSConnectionPool",
|
||||||
|
)
|
||||||
|
if any(not hasattr(awsrequest, name) for name in required) or not hasattr(
|
||||||
|
httpsession, "URLLib3Session"
|
||||||
|
):
|
||||||
|
raise SdkPeerPinningError(
|
||||||
|
"The installed botocore release is missing the connection classes required for peer pinning"
|
||||||
|
)
|
||||||
|
|
||||||
|
def pinned_new_connection(connection: Any) -> socket.socket:
|
||||||
|
hostname = str(getattr(connection, "_dns_host", "") or "").strip()
|
||||||
|
port = int(getattr(connection, "port", 0) or 0)
|
||||||
|
if not hostname or not port:
|
||||||
|
raise urllib3_exceptions.NewConnectionError(
|
||||||
|
connection, "Pinned S3 connection is missing its target authority"
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
return create_outbound_connection(
|
||||||
|
hostname,
|
||||||
|
port,
|
||||||
|
timeout=getattr(connection, "timeout", None),
|
||||||
|
source_address=getattr(connection, "source_address", None),
|
||||||
|
socket_options=getattr(connection, "socket_options", None),
|
||||||
|
label="S3 connector peer",
|
||||||
|
)
|
||||||
|
except socket.timeout as exc:
|
||||||
|
raise urllib3_exceptions.ConnectTimeoutError(
|
||||||
|
connection,
|
||||||
|
f"Connection to {hostname} timed out while selecting an approved peer",
|
||||||
|
) from exc
|
||||||
|
except (OSError, OutboundHttpError, ValueError) as exc:
|
||||||
|
raise urllib3_exceptions.NewConnectionError(
|
||||||
|
connection,
|
||||||
|
f"S3 connector peer was rejected: {exc}",
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
class PinnedAWSHTTPConnection(awsrequest.AWSHTTPConnection):
|
||||||
|
_new_conn = pinned_new_connection
|
||||||
|
|
||||||
|
class PinnedAWSHTTPSConnection(awsrequest.AWSHTTPSConnection):
|
||||||
|
_new_conn = pinned_new_connection
|
||||||
|
|
||||||
|
class PinnedAWSHTTPConnectionPool(awsrequest.AWSHTTPConnectionPool):
|
||||||
|
ConnectionCls = PinnedAWSHTTPConnection
|
||||||
|
|
||||||
|
class PinnedAWSHTTPSConnectionPool(awsrequest.AWSHTTPSConnectionPool):
|
||||||
|
ConnectionCls = PinnedAWSHTTPSConnection
|
||||||
|
|
||||||
|
class PinnedURLLib3Session(httpsession.URLLib3Session):
|
||||||
|
def __init__(self, *args: Any, **kwargs: Any) -> None:
|
||||||
|
# A proxy would select the final target outside this process. Until
|
||||||
|
# proxy peer delegation is modeled, connector traffic is direct.
|
||||||
|
kwargs["proxies"] = {}
|
||||||
|
super().__init__(*args, **kwargs)
|
||||||
|
self._pool_classes_by_scheme = {
|
||||||
|
"http": PinnedAWSHTTPConnectionPool,
|
||||||
|
"https": PinnedAWSHTTPSConnectionPool,
|
||||||
|
}
|
||||||
|
manager = getattr(self, "_manager", None)
|
||||||
|
if manager is None or not hasattr(manager, "pool_classes_by_scheme"):
|
||||||
|
raise SdkPeerPinningError(
|
||||||
|
"The installed botocore pool manager cannot enforce pinned connection classes"
|
||||||
|
)
|
||||||
|
manager.pool_classes_by_scheme = self._pool_classes_by_scheme
|
||||||
|
|
||||||
|
return PinnedURLLib3Session, PinnedAWSHTTPConnection, PinnedAWSHTTPSConnection
|
||||||
|
|
||||||
|
|
||||||
|
@lru_cache(maxsize=1)
|
||||||
|
def pinned_smb_connection_cache() -> dict[str, Any]:
|
||||||
|
"""Return the Files-owned cache; unproven process-global sessions are never reused."""
|
||||||
|
|
||||||
|
return {}
|
||||||
|
|
||||||
|
|
||||||
|
def install_pinned_smb_transport(smbclient: Any) -> Any:
|
||||||
|
"""Install a process-wide fail-closed smbclient session factory.
|
||||||
|
|
||||||
|
smbclient routes initial connections, reconnects, and DFS targets through
|
||||||
|
``smbclient._pool.register_session``. Replacing that single factory with a
|
||||||
|
behavior-compatible implementation ensures every target uses a socket
|
||||||
|
selected by Core's deployment-wide outbound policy.
|
||||||
|
"""
|
||||||
|
|
||||||
|
try:
|
||||||
|
pool = import_module("smbclient._pool")
|
||||||
|
connection_module = import_module("smbprotocol.connection")
|
||||||
|
session_module = import_module("smbprotocol.session")
|
||||||
|
transport_module = import_module("smbprotocol.transport")
|
||||||
|
except ImportError as exc:
|
||||||
|
raise SdkPeerPinningError(
|
||||||
|
"SMB connector browsing requires the optional smbprotocol dependency"
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
current = getattr(pool, "register_session", None)
|
||||||
|
if getattr(current, "__govoplan_peer_pinned__", False):
|
||||||
|
expected_tcp = getattr(current, "__govoplan_pinned_tcp__", None)
|
||||||
|
if expected_tcp is None or getattr(connection_module, "Tcp", None) is not expected_tcp:
|
||||||
|
raise SdkPeerPinningError(
|
||||||
|
"The installed SMB transport changed after peer pinning; refusing to reuse the session factory"
|
||||||
|
)
|
||||||
|
return smbclient
|
||||||
|
required_parameters = {
|
||||||
|
"server",
|
||||||
|
"username",
|
||||||
|
"password",
|
||||||
|
"port",
|
||||||
|
"encrypt",
|
||||||
|
"connection_timeout",
|
||||||
|
"connection_cache",
|
||||||
|
"auth_protocol",
|
||||||
|
"require_signing",
|
||||||
|
}
|
||||||
|
if current is None or not required_parameters.issubset(
|
||||||
|
inspect.signature(current).parameters
|
||||||
|
):
|
||||||
|
raise SdkPeerPinningError(
|
||||||
|
"The installed smbprotocol release does not expose the session seam required for peer pinning"
|
||||||
|
)
|
||||||
|
|
||||||
|
class PinnedTcp(transport_module.Tcp):
|
||||||
|
def connect(self) -> None:
|
||||||
|
with self._sock_lock:
|
||||||
|
if self.connected:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
self._sock = create_outbound_connection(
|
||||||
|
self.server,
|
||||||
|
int(self.port),
|
||||||
|
timeout=self.timeout,
|
||||||
|
label="SMB connector peer",
|
||||||
|
)
|
||||||
|
except (OSError, OutboundHttpError, ValueError) as exc:
|
||||||
|
raise ValueError(
|
||||||
|
f"SMB connector peer '{self.server}:{self.port}' was rejected: {exc}"
|
||||||
|
) from exc
|
||||||
|
self._sock.settimeout(None)
|
||||||
|
self.connected = True
|
||||||
|
|
||||||
|
# Connection.connect() instantiates the module-level Tcp symbol on every
|
||||||
|
# reconnect. Replacing it process-wide is deliberate: an SMB connection
|
||||||
|
# created by another code path must become stricter, never bypass Files'
|
||||||
|
# peer boundary.
|
||||||
|
if not hasattr(connection_module, "Tcp"):
|
||||||
|
raise SdkPeerPinningError(
|
||||||
|
"The installed smbprotocol release cannot install the pinned TCP transport"
|
||||||
|
)
|
||||||
|
connection_module.Tcp = PinnedTcp
|
||||||
|
|
||||||
|
def pinned_register_session(
|
||||||
|
server: str,
|
||||||
|
username: str | None = None,
|
||||||
|
password: str | None = None,
|
||||||
|
port: int = 445,
|
||||||
|
encrypt: bool | None = None,
|
||||||
|
connection_timeout: float = 60,
|
||||||
|
connection_cache: dict[str, Any] | None = None,
|
||||||
|
auth_protocol: str = "negotiate",
|
||||||
|
require_signing: bool = True,
|
||||||
|
) -> Any:
|
||||||
|
cache = pinned_smb_connection_cache() if connection_cache is None else connection_cache
|
||||||
|
connection_key = f"{server.lower()}:{port}"
|
||||||
|
connection = cache.get(connection_key)
|
||||||
|
transport = getattr(connection, "transport", None)
|
||||||
|
if connection is not None and not isinstance(transport, PinnedTcp):
|
||||||
|
disconnect = getattr(connection, "disconnect", None)
|
||||||
|
if callable(disconnect):
|
||||||
|
try:
|
||||||
|
disconnect(close=True)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
cache.pop(connection_key, None)
|
||||||
|
connection = None
|
||||||
|
if connection is None or not getattr(connection.transport, "connected", False):
|
||||||
|
connection = connection_module.Connection(
|
||||||
|
pool.ClientConfig().client_guid,
|
||||||
|
server,
|
||||||
|
port,
|
||||||
|
require_signing=require_signing,
|
||||||
|
)
|
||||||
|
connection.transport = PinnedTcp(server, port)
|
||||||
|
connection.connect(timeout=connection_timeout)
|
||||||
|
if not isinstance(connection.transport, PinnedTcp):
|
||||||
|
disconnect = getattr(connection, "disconnect", None)
|
||||||
|
if callable(disconnect):
|
||||||
|
try:
|
||||||
|
disconnect(close=True)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
raise SdkPeerPinningError(
|
||||||
|
"smbprotocol replaced the required pinned TCP transport during connection setup"
|
||||||
|
)
|
||||||
|
cache[connection_key] = connection
|
||||||
|
|
||||||
|
session = next(
|
||||||
|
(
|
||||||
|
item
|
||||||
|
for item in connection.session_table.values()
|
||||||
|
if username is None or item.username == username
|
||||||
|
),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
if session is None:
|
||||||
|
session = session_module.Session(
|
||||||
|
connection,
|
||||||
|
username=username,
|
||||||
|
password=password,
|
||||||
|
require_encryption=(encrypt is True),
|
||||||
|
auth_protocol=auth_protocol,
|
||||||
|
)
|
||||||
|
session.connect()
|
||||||
|
elif encrypt is not None:
|
||||||
|
if session.encrypt_data and not encrypt:
|
||||||
|
raise ValueError(
|
||||||
|
"Cannot disable encryption on an already negotiated session."
|
||||||
|
)
|
||||||
|
if not session.encrypt_data and encrypt:
|
||||||
|
session.encrypt = True
|
||||||
|
return session
|
||||||
|
|
||||||
|
pinned_register_session.__govoplan_peer_pinned__ = True # type: ignore[attr-defined]
|
||||||
|
pinned_register_session.__govoplan_pinned_tcp__ = PinnedTcp # type: ignore[attr-defined]
|
||||||
|
pool.register_session = pinned_register_session
|
||||||
|
if hasattr(smbclient, "register_session"):
|
||||||
|
smbclient.register_session = pinned_register_session
|
||||||
|
return smbclient
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"SdkPeerPinningError",
|
||||||
|
"create_pinned_s3_client",
|
||||||
|
"install_pinned_smb_transport",
|
||||||
|
"pinned_smb_connection_cache",
|
||||||
|
]
|
||||||
@@ -2,6 +2,7 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import unittest
|
import unittest
|
||||||
from datetime import datetime, timedelta, timezone
|
from datetime import datetime, timedelta, timezone
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
from sqlalchemy import create_engine, text
|
from sqlalchemy import create_engine, text
|
||||||
from sqlalchemy.orm import sessionmaker
|
from sqlalchemy.orm import sessionmaker
|
||||||
@@ -12,7 +13,11 @@ from govoplan_core.core.change_sequence import ChangeSequenceEntry
|
|||||||
from govoplan_core.db.base import Base
|
from govoplan_core.db.base import Base
|
||||||
from govoplan_files.backend.capabilities import FilesAccessService, virtual_folder_resource_id
|
from govoplan_files.backend.capabilities import FilesAccessService, virtual_folder_resource_id
|
||||||
from govoplan_files.backend.db.models import FileAsset, FileFolder, FileShare
|
from govoplan_files.backend.db.models import FileAsset, FileFolder, FileShare
|
||||||
from govoplan_files.backend.storage.files import count_assets_for_user, list_assets_for_user
|
from govoplan_files.backend.storage.files import (
|
||||||
|
count_assets_for_user,
|
||||||
|
list_assets_for_user,
|
||||||
|
list_recent_assets_for_user,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
TENANT_ID = "tenant-1"
|
TENANT_ID = "tenant-1"
|
||||||
@@ -22,6 +27,54 @@ GROUP_ID = "group-1"
|
|||||||
|
|
||||||
|
|
||||||
class FilesAccessProviderTests(unittest.TestCase):
|
class FilesAccessProviderTests(unittest.TestCase):
|
||||||
|
def test_recent_projection_is_bounded_ordered_and_access_filtered(self) -> None:
|
||||||
|
session = _session()
|
||||||
|
self.addCleanup(_close_session, session)
|
||||||
|
_seed_access_subjects(session)
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
session.add_all(
|
||||||
|
[
|
||||||
|
FileAsset(
|
||||||
|
id="owned-older",
|
||||||
|
tenant_id=TENANT_ID,
|
||||||
|
owner_type="user",
|
||||||
|
owner_user_id=USER_ID,
|
||||||
|
display_path="older.pdf",
|
||||||
|
filename="older.pdf",
|
||||||
|
updated_at=now - timedelta(hours=2),
|
||||||
|
),
|
||||||
|
FileAsset(
|
||||||
|
id="owned-newer",
|
||||||
|
tenant_id=TENANT_ID,
|
||||||
|
owner_type="user",
|
||||||
|
owner_user_id=USER_ID,
|
||||||
|
display_path="newer.pdf",
|
||||||
|
filename="newer.pdf",
|
||||||
|
updated_at=now - timedelta(hours=1),
|
||||||
|
),
|
||||||
|
FileAsset(
|
||||||
|
id="other-newest",
|
||||||
|
tenant_id=TENANT_ID,
|
||||||
|
owner_type="user",
|
||||||
|
owner_user_id=OTHER_USER_ID,
|
||||||
|
display_path="private.pdf",
|
||||||
|
filename="private.pdf",
|
||||||
|
updated_at=now,
|
||||||
|
),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
|
||||||
|
with patch("govoplan_files.backend.storage.files.user_group_ids", return_value=[]):
|
||||||
|
recent = list_recent_assets_for_user(
|
||||||
|
session,
|
||||||
|
tenant_id=TENANT_ID,
|
||||||
|
user_id=USER_ID,
|
||||||
|
limit=1,
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(["owned-newer"], [asset.id for asset in recent])
|
||||||
|
|
||||||
def test_file_access_provider_explains_owner_share_admin_and_missing_resources(self) -> None:
|
def test_file_access_provider_explains_owner_share_admin_and_missing_resources(self) -> None:
|
||||||
session = _session()
|
session = _session()
|
||||||
self.addCleanup(_close_session, session)
|
self.addCleanup(_close_session, session)
|
||||||
|
|||||||
@@ -0,0 +1,219 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from types import SimpleNamespace
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from govoplan_core.core.configuration_packages import (
|
||||||
|
ConfigurationPackageFragment,
|
||||||
|
ConfigurationPreflightContext,
|
||||||
|
)
|
||||||
|
from govoplan_core.core.infrastructure_capabilities import (
|
||||||
|
infrastructure_capability_receipt_from_mapping,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.configuration_provider import (
|
||||||
|
FILES_CONFIGURATION_CAPABILITY,
|
||||||
|
FilesConfigurationProvider,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.manifest import manifest
|
||||||
|
|
||||||
|
|
||||||
|
def _receipt(*, source: str = "host-local", state: str = "configured"):
|
||||||
|
endpoint = (
|
||||||
|
{"kind": "filesystem", "reference": "volume:files-data"}
|
||||||
|
if source == "host-local"
|
||||||
|
else {"scheme": "http", "host": "garage", "port": 3900}
|
||||||
|
)
|
||||||
|
secret_refs = (
|
||||||
|
[]
|
||||||
|
if source == "host-local"
|
||||||
|
else [
|
||||||
|
"env:FILE_STORAGE_S3_ACCESS_KEY_ID",
|
||||||
|
"env:FILE_STORAGE_S3_SECRET_ACCESS_KEY",
|
||||||
|
"env:GARAGE_RPC_SECRET",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
return infrastructure_capability_receipt_from_mapping(
|
||||||
|
{
|
||||||
|
"schema_version": 1,
|
||||||
|
"installation_id": "files-provider-test",
|
||||||
|
"profile": "evaluation",
|
||||||
|
"capabilities": [
|
||||||
|
{
|
||||||
|
"id": "files.storage",
|
||||||
|
"label": "Managed file content storage",
|
||||||
|
"state": state,
|
||||||
|
"source": source,
|
||||||
|
"detail": "Deployment-owned storage binding.",
|
||||||
|
"endpoint": endpoint,
|
||||||
|
"secret_refs": secret_refs,
|
||||||
|
"dependent_modules": ["files"],
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"post_install_tasks": [],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _local_settings(**overrides):
|
||||||
|
values = {
|
||||||
|
"file_storage_backend": "local",
|
||||||
|
"file_storage_local_root": "/var/lib/govoplan/files",
|
||||||
|
}
|
||||||
|
values.update(overrides)
|
||||||
|
return SimpleNamespace(**values)
|
||||||
|
|
||||||
|
|
||||||
|
def _s3_settings(**overrides):
|
||||||
|
values = {
|
||||||
|
"file_storage_backend": "s3",
|
||||||
|
"file_storage_s3_endpoint_url": "http://garage:3900",
|
||||||
|
"file_storage_s3_bucket": "files",
|
||||||
|
"file_storage_s3_deployment_managed": True,
|
||||||
|
"file_storage_s3_endpoint_trusted": False,
|
||||||
|
}
|
||||||
|
values.update(overrides)
|
||||||
|
return SimpleNamespace(**values)
|
||||||
|
|
||||||
|
|
||||||
|
class FilesConfigurationProviderTests(unittest.TestCase):
|
||||||
|
def test_provider_is_registered(self) -> None:
|
||||||
|
self.assertIn(FILES_CONFIGURATION_CAPABILITY, manifest.capability_factories)
|
||||||
|
|
||||||
|
def test_matching_local_storage_is_an_idempotent_noop(self) -> None:
|
||||||
|
provider = FilesConfigurationProvider(
|
||||||
|
settings=_local_settings(),
|
||||||
|
environment={},
|
||||||
|
)
|
||||||
|
context = ConfigurationPreflightContext(
|
||||||
|
infrastructure_receipt=_receipt(),
|
||||||
|
)
|
||||||
|
fragment = ConfigurationPackageFragment(
|
||||||
|
module_id="files",
|
||||||
|
fragment_type="managed_storage",
|
||||||
|
fragment_id="files-storage",
|
||||||
|
payload={
|
||||||
|
"expected_backend": "local",
|
||||||
|
"expected_source": "host-local",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
first = provider.preflight(fragment, context)
|
||||||
|
applied = provider.apply(fragment, {}, context)
|
||||||
|
second = provider.preflight(fragment, context)
|
||||||
|
|
||||||
|
self.assertEqual("skip", first.plan[0].action)
|
||||||
|
self.assertEqual("skip", second.plan[0].action)
|
||||||
|
self.assertEqual((), applied.diagnostics)
|
||||||
|
self.assertEqual({}, applied.created_refs)
|
||||||
|
self.assertEqual({}, applied.updated_refs)
|
||||||
|
|
||||||
|
def test_runtime_backend_mismatch_blocks_without_rewriting_settings(self) -> None:
|
||||||
|
settings = _local_settings()
|
||||||
|
provider = FilesConfigurationProvider(settings=settings, environment={})
|
||||||
|
context = ConfigurationPreflightContext(
|
||||||
|
infrastructure_receipt=_receipt(source="installer-managed-garage"),
|
||||||
|
)
|
||||||
|
fragment = ConfigurationPackageFragment(
|
||||||
|
module_id="files",
|
||||||
|
fragment_type="managed_storage",
|
||||||
|
payload={},
|
||||||
|
)
|
||||||
|
|
||||||
|
result = provider.preflight(fragment, context)
|
||||||
|
|
||||||
|
self.assertEqual("blocked", result.plan[0].action)
|
||||||
|
self.assertIn(
|
||||||
|
"files_storage_runtime_mismatch",
|
||||||
|
{item.code for item in result.diagnostics},
|
||||||
|
)
|
||||||
|
self.assertEqual("local", settings.file_storage_backend)
|
||||||
|
|
||||||
|
def test_garage_binding_requires_only_files_secret_references(self) -> None:
|
||||||
|
fragment = ConfigurationPackageFragment(
|
||||||
|
module_id="files",
|
||||||
|
fragment_type="managed_storage",
|
||||||
|
payload={},
|
||||||
|
)
|
||||||
|
context = ConfigurationPreflightContext(
|
||||||
|
infrastructure_receipt=_receipt(source="installer-managed-garage"),
|
||||||
|
)
|
||||||
|
missing = FilesConfigurationProvider(
|
||||||
|
settings=_s3_settings(),
|
||||||
|
environment={},
|
||||||
|
).preflight(fragment, context)
|
||||||
|
available = FilesConfigurationProvider(
|
||||||
|
settings=_s3_settings(),
|
||||||
|
environment={
|
||||||
|
"FILE_STORAGE_S3_ACCESS_KEY_ID": "reference-resolved",
|
||||||
|
"FILE_STORAGE_S3_SECRET_ACCESS_KEY": "reference-resolved",
|
||||||
|
},
|
||||||
|
).preflight(fragment, context)
|
||||||
|
|
||||||
|
self.assertEqual("blocked", missing.plan[0].action)
|
||||||
|
self.assertEqual(
|
||||||
|
2,
|
||||||
|
sum(
|
||||||
|
item.code == "files_storage_secret_reference_unresolved"
|
||||||
|
for item in missing.diagnostics
|
||||||
|
),
|
||||||
|
)
|
||||||
|
self.assertEqual("skip", available.plan[0].action)
|
||||||
|
|
||||||
|
def test_operator_supplied_s3_requires_explicit_endpoint_trust(self) -> None:
|
||||||
|
receipt = _receipt(source="operator-supplied-s3", state="externally_supplied")
|
||||||
|
fragment = ConfigurationPackageFragment(
|
||||||
|
module_id="files",
|
||||||
|
fragment_type="managed_storage",
|
||||||
|
payload={},
|
||||||
|
)
|
||||||
|
context = ConfigurationPreflightContext(infrastructure_receipt=receipt)
|
||||||
|
environment = {
|
||||||
|
"FILE_STORAGE_S3_ACCESS_KEY_ID": "reference-resolved",
|
||||||
|
"FILE_STORAGE_S3_SECRET_ACCESS_KEY": "reference-resolved",
|
||||||
|
}
|
||||||
|
|
||||||
|
blocked = FilesConfigurationProvider(
|
||||||
|
settings=_s3_settings(
|
||||||
|
file_storage_s3_deployment_managed=False,
|
||||||
|
file_storage_s3_endpoint_trusted=False,
|
||||||
|
),
|
||||||
|
environment=environment,
|
||||||
|
).preflight(fragment, context)
|
||||||
|
ready = FilesConfigurationProvider(
|
||||||
|
settings=_s3_settings(
|
||||||
|
file_storage_s3_deployment_managed=False,
|
||||||
|
file_storage_s3_endpoint_trusted=True,
|
||||||
|
),
|
||||||
|
environment=environment,
|
||||||
|
).preflight(fragment, context)
|
||||||
|
|
||||||
|
self.assertEqual("blocked", blocked.plan[0].action)
|
||||||
|
self.assertIn(
|
||||||
|
"files_storage_trust_boundary_missing",
|
||||||
|
{item.code for item in blocked.diagnostics},
|
||||||
|
)
|
||||||
|
self.assertEqual("skip", ready.plan[0].action)
|
||||||
|
|
||||||
|
def test_inline_storage_secret_is_rejected(self) -> None:
|
||||||
|
provider = FilesConfigurationProvider(
|
||||||
|
settings=_local_settings(),
|
||||||
|
environment={},
|
||||||
|
)
|
||||||
|
result = provider.preflight(
|
||||||
|
ConfigurationPackageFragment(
|
||||||
|
module_id="files",
|
||||||
|
fragment_type="managed_storage",
|
||||||
|
payload={"secret_access_key": "inline"},
|
||||||
|
),
|
||||||
|
ConfigurationPreflightContext(infrastructure_receipt=_receipt()),
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual("blocked", result.plan[0].action)
|
||||||
|
self.assertIn(
|
||||||
|
"files_configuration_secret_forbidden",
|
||||||
|
{item.code for item in result.diagnostics},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -5,7 +5,7 @@ from datetime import UTC, datetime
|
|||||||
from unittest.mock import patch
|
from unittest.mock import patch
|
||||||
|
|
||||||
from govoplan_files.backend.storage.connector_browse import ConnectorBrowseError, _smb_location, browse_connector_profile
|
from govoplan_files.backend.storage.connector_browse import ConnectorBrowseError, _smb_location, browse_connector_profile
|
||||||
from govoplan_files.backend.storage.connector_imports import ConnectorImportError, read_connector_file
|
from govoplan_files.backend.storage.connector_imports import read_connector_file
|
||||||
from govoplan_files.backend.storage.connector_profiles import ConnectorProfile, connector_profiles_from_payload
|
from govoplan_files.backend.storage.connector_profiles import ConnectorProfile, connector_profiles_from_payload
|
||||||
from govoplan_files.backend.storage.connector_providers import connector_provider_descriptors
|
from govoplan_files.backend.storage.connector_providers import connector_provider_descriptors
|
||||||
|
|
||||||
@@ -23,6 +23,10 @@ class FakeS3Client:
|
|||||||
self.list_objects_request: dict[str, object] | None = None
|
self.list_objects_request: dict[str, object] | None = None
|
||||||
self.head_request: dict[str, object] | None = None
|
self.head_request: dict[str, object] | None = None
|
||||||
self.get_request: dict[str, object] | None = None
|
self.get_request: dict[str, object] | None = None
|
||||||
|
self.closed = False
|
||||||
|
|
||||||
|
def close(self) -> None:
|
||||||
|
self.closed = True
|
||||||
|
|
||||||
def list_objects_v2(self, **kwargs: object) -> dict[str, object]:
|
def list_objects_v2(self, **kwargs: object) -> dict[str, object]:
|
||||||
self.list_objects_request = dict(kwargs)
|
self.list_objects_request = dict(kwargs)
|
||||||
@@ -75,42 +79,39 @@ def s3_profile(**overrides: object) -> ConnectorProfile:
|
|||||||
|
|
||||||
|
|
||||||
class ConnectorProviderTests(unittest.TestCase):
|
class ConnectorProviderTests(unittest.TestCase):
|
||||||
def test_smb_sdk_transport_fails_closed_before_client_creation_in_all_modes(self) -> None:
|
def test_smb_browse_uses_the_files_owned_pinned_connection_cache(self) -> None:
|
||||||
profile = ConnectorProfile(
|
profile = ConnectorProfile(
|
||||||
id="smb",
|
id="smb",
|
||||||
label="SMB",
|
label="SMB",
|
||||||
provider="smb",
|
provider="smb",
|
||||||
endpoint_url="smb://files.example.test/share",
|
endpoint_url="smb://files.example.test/share",
|
||||||
)
|
)
|
||||||
for allow_private, address in ((False, "93.184.216.34"), (True, "10.0.0.5")):
|
with patch.dict(
|
||||||
with self.subTest(allow_private=allow_private), patch.dict(
|
|
||||||
"os.environ",
|
"os.environ",
|
||||||
{
|
{"APP_ENV": "production", "GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS": "false"},
|
||||||
"APP_ENV": "production",
|
|
||||||
"GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS": str(allow_private).lower(),
|
|
||||||
},
|
|
||||||
), patch(
|
), patch(
|
||||||
"govoplan_core.security.outbound_http.socket.getaddrinfo",
|
"govoplan_core.security.outbound_http.socket.getaddrinfo",
|
||||||
return_value=[(2, 1, 6, "", (address, 445))],
|
return_value=[(2, 1, 6, "", ("93.184.216.34", 445))],
|
||||||
), patch("govoplan_files.backend.storage.connector_browse._smbclient_module") as sdk, self.assertRaisesRegex(
|
), patch("govoplan_files.backend.storage.connector_browse._smbclient_module") as sdk:
|
||||||
ConnectorBrowseError,
|
sdk.return_value.scandir.return_value.__enter__.return_value = iter(())
|
||||||
"redirects/referrals.*DNS/IP pinning",
|
self.assertEqual([], browse_connector_profile(profile, path=""))
|
||||||
):
|
|
||||||
browse_connector_profile(profile, path="")
|
|
||||||
sdk.assert_not_called()
|
|
||||||
|
|
||||||
def test_smb_explicit_ip_still_fails_closed_because_the_sdk_may_follow_referrals(self) -> None:
|
kwargs = sdk.return_value.scandir.call_args.kwargs
|
||||||
|
self.assertIsInstance(kwargs["connection_cache"], dict)
|
||||||
|
self.assertTrue(kwargs["require_signing"])
|
||||||
|
|
||||||
|
def test_smb_endpoint_preflight_applies_private_network_policy(self) -> None:
|
||||||
profile = ConnectorProfile(id="smb", label="SMB", provider="smb", endpoint_url="smb://10.0.0.5/share")
|
profile = ConnectorProfile(id="smb", label="SMB", provider="smb", endpoint_url="smb://10.0.0.5/share")
|
||||||
with patch.dict(
|
with patch.dict(
|
||||||
"os.environ",
|
"os.environ",
|
||||||
{"APP_ENV": "production", "GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS": "true"},
|
{"APP_ENV": "production", "GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS": "false"},
|
||||||
), patch(
|
), patch(
|
||||||
"govoplan_core.security.outbound_http.socket.getaddrinfo",
|
"govoplan_core.security.outbound_http.socket.getaddrinfo",
|
||||||
return_value=[(2, 1, 6, "", ("10.0.0.5", 445))],
|
return_value=[(2, 1, 6, "", ("10.0.0.5", 445))],
|
||||||
), self.assertRaisesRegex(ConnectorBrowseError, "redirects/referrals.*DNS/IP pinning"):
|
), self.assertRaisesRegex(ConnectorBrowseError, "non-public network"):
|
||||||
_smb_location(profile)
|
_smb_location(profile)
|
||||||
|
|
||||||
def test_smb_import_surfaces_fail_closed_policy_as_an_import_error(self) -> None:
|
def test_smb_import_uses_the_same_pinned_connection_cache(self) -> None:
|
||||||
profile = ConnectorProfile(id="smb", label="SMB", provider="smb", endpoint_url="smb://10.0.0.5/share")
|
profile = ConnectorProfile(id="smb", label="SMB", provider="smb", endpoint_url="smb://10.0.0.5/share")
|
||||||
with patch.dict(
|
with patch.dict(
|
||||||
"os.environ",
|
"os.environ",
|
||||||
@@ -118,12 +119,13 @@ class ConnectorProviderTests(unittest.TestCase):
|
|||||||
), patch(
|
), patch(
|
||||||
"govoplan_core.security.outbound_http.socket.getaddrinfo",
|
"govoplan_core.security.outbound_http.socket.getaddrinfo",
|
||||||
return_value=[(2, 1, 6, "", ("10.0.0.5", 445))],
|
return_value=[(2, 1, 6, "", ("10.0.0.5", 445))],
|
||||||
), patch("govoplan_files.backend.storage.connector_imports._smbclient_module") as sdk, self.assertRaisesRegex(
|
), patch("govoplan_files.backend.storage.connector_imports._smbclient_module") as sdk:
|
||||||
ConnectorImportError,
|
sdk.return_value.stat.return_value.st_size = 4
|
||||||
"redirects/referrals.*DNS/IP pinning",
|
sdk.return_value.open_file.return_value.__enter__.return_value.read.return_value = b"test"
|
||||||
):
|
downloaded = read_connector_file(profile, library_id="", path="notice.txt", max_bytes=1024)
|
||||||
read_connector_file(profile, library_id="", path="notice.txt", max_bytes=1024)
|
|
||||||
sdk.assert_not_called()
|
self.assertEqual(b"test", downloaded.data)
|
||||||
|
self.assertIsInstance(sdk.return_value.stat.call_args.kwargs["connection_cache"], dict)
|
||||||
|
|
||||||
def test_provider_descriptors_include_s3_and_reserved_microsoft_providers(self) -> None:
|
def test_provider_descriptors_include_s3_and_reserved_microsoft_providers(self) -> None:
|
||||||
descriptors = {descriptor.provider: descriptor for descriptor in connector_provider_descriptors()}
|
descriptors = {descriptor.provider: descriptor for descriptor in connector_provider_descriptors()}
|
||||||
@@ -172,6 +174,7 @@ class ConnectorProviderTests(unittest.TestCase):
|
|||||||
self.assertEqual("govoplan:root/report.xlsx", items[1].external_id)
|
self.assertEqual("govoplan:root/report.xlsx", items[1].external_id)
|
||||||
self.assertEqual("root/report.xlsx", items[1].metadata["key"])
|
self.assertEqual("root/report.xlsx", items[1].metadata["key"])
|
||||||
self.assertEqual("next-page", items[1].metadata["next_continuation_token"])
|
self.assertEqual("next-page", items[1].metadata["next_continuation_token"])
|
||||||
|
self.assertTrue(client.closed)
|
||||||
|
|
||||||
def test_s3_browse_lists_buckets_when_profile_has_no_bucket(self) -> None:
|
def test_s3_browse_lists_buckets_when_profile_has_no_bucket(self) -> None:
|
||||||
client = FakeS3Client()
|
client = FakeS3Client()
|
||||||
@@ -181,27 +184,33 @@ class ConnectorProviderTests(unittest.TestCase):
|
|||||||
|
|
||||||
self.assertEqual(["archive"], [item.path for item in items])
|
self.assertEqual(["archive"], [item.path for item in items])
|
||||||
|
|
||||||
def test_s3_sdk_transport_fails_closed_before_client_creation_in_private_mode(self) -> None:
|
def test_s3_client_is_constructed_through_the_pinned_transport(self) -> None:
|
||||||
with patch.dict(
|
with patch.dict(
|
||||||
"os.environ",
|
"os.environ",
|
||||||
{"APP_ENV": "production", "GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS": "true"},
|
{"APP_ENV": "production", "GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS": "true"},
|
||||||
), patch(
|
), patch(
|
||||||
"govoplan_core.security.outbound_http.socket.getaddrinfo",
|
"govoplan_core.security.outbound_http.socket.getaddrinfo",
|
||||||
return_value=[(2, 1, 6, "", ("127.0.0.1", 9000))],
|
return_value=[(2, 1, 6, "", ("127.0.0.1", 9000))],
|
||||||
), patch("govoplan_files.backend.storage.connector_browse.import_module") as importer, self.assertRaisesRegex(
|
), patch(
|
||||||
ConnectorBrowseError,
|
"govoplan_files.backend.storage.connector_browse.create_pinned_s3_client",
|
||||||
"until that transport supports.*DNS/IP pinning",
|
return_value=FakeS3Client(),
|
||||||
):
|
) as factory:
|
||||||
browse_connector_profile(s3_profile(), path="")
|
browse_connector_profile(s3_profile(), path="")
|
||||||
importer.assert_not_called()
|
|
||||||
|
|
||||||
def test_s3_sdk_endpoint_discovery_fails_closed(self) -> None:
|
kwargs = factory.call_args.kwargs
|
||||||
with patch("govoplan_files.backend.storage.connector_browse.import_module") as importer, self.assertRaisesRegex(
|
self.assertEqual("http://127.0.0.1:9000", kwargs["endpoint_url"])
|
||||||
ConnectorBrowseError,
|
self.assertEqual("access-key", kwargs["aws_access_key_id"])
|
||||||
"endpoint discovery.*cannot guarantee.*DNS/IP pinning",
|
self.assertEqual("secret-key", kwargs["aws_secret_access_key"])
|
||||||
):
|
self.assertEqual({}, kwargs["config"].proxies)
|
||||||
|
|
||||||
|
def test_s3_endpoint_discovery_uses_the_same_pinned_transport(self) -> None:
|
||||||
|
with patch(
|
||||||
|
"govoplan_files.backend.storage.connector_browse.create_pinned_s3_client",
|
||||||
|
return_value=FakeS3Client(),
|
||||||
|
) as factory:
|
||||||
browse_connector_profile(s3_profile(endpoint_url=None), path="")
|
browse_connector_profile(s3_profile(endpoint_url=None), path="")
|
||||||
importer.assert_not_called()
|
|
||||||
|
self.assertNotIn("endpoint_url", factory.call_args.kwargs)
|
||||||
|
|
||||||
def test_s3_import_downloads_object_and_preserves_remote_identity(self) -> None:
|
def test_s3_import_downloads_object_and_preserves_remote_identity(self) -> None:
|
||||||
client = FakeS3Client()
|
client = FakeS3Client()
|
||||||
@@ -217,6 +226,7 @@ class ConnectorProviderTests(unittest.TestCase):
|
|||||||
self.assertEqual("govoplan:root/report.txt", downloaded.external_id)
|
self.assertEqual("govoplan:root/report.txt", downloaded.external_id)
|
||||||
self.assertEqual("s3://govoplan/root/report.txt", downloaded.external_url)
|
self.assertEqual("s3://govoplan/root/report.txt", downloaded.external_url)
|
||||||
self.assertEqual("checksum", downloaded.metadata["checksum_sha256"])
|
self.assertEqual("checksum", downloaded.metadata["checksum_sha256"])
|
||||||
|
self.assertTrue(client.closed)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|||||||
@@ -0,0 +1,101 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import unittest
|
||||||
|
from types import SimpleNamespace
|
||||||
|
from unittest.mock import Mock, patch
|
||||||
|
|
||||||
|
from govoplan_files.backend.storage.common import FileStorageError
|
||||||
|
from govoplan_files.backend.storage.connector_profiles import ConnectorProfile
|
||||||
|
from govoplan_files.backend.storage.connector_spaces import (
|
||||||
|
soft_delete_connector_space,
|
||||||
|
validate_connector_space_write_mode,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorSpaceDeletionTests(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
self.session = Mock()
|
||||||
|
self.space = SimpleNamespace(
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
owner_type="user",
|
||||||
|
owner_user_id="owner-1",
|
||||||
|
owner_group_id=None,
|
||||||
|
deleted_at=None,
|
||||||
|
is_active=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
@patch("govoplan_files.backend.storage.connector_spaces.utcnow")
|
||||||
|
@patch("govoplan_files.backend.storage.connector_spaces.ensure_owner_access")
|
||||||
|
def test_owner_can_soft_delete_local_space_definition(
|
||||||
|
self, ensure_owner_access: Mock, utcnow: Mock
|
||||||
|
) -> None:
|
||||||
|
deleted_at = object()
|
||||||
|
utcnow.return_value = deleted_at
|
||||||
|
|
||||||
|
result = soft_delete_connector_space(
|
||||||
|
self.session, self.space, user_id="owner-1"
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertIs(result, self.space)
|
||||||
|
self.assertIs(deleted_at, self.space.deleted_at)
|
||||||
|
self.assertFalse(self.space.is_active)
|
||||||
|
ensure_owner_access.assert_called_once_with(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
owner_type="user",
|
||||||
|
owner_id="owner-1",
|
||||||
|
user_id="owner-1",
|
||||||
|
is_admin=False,
|
||||||
|
)
|
||||||
|
self.session.add.assert_called_once_with(self.space)
|
||||||
|
self.session.flush.assert_called_once_with()
|
||||||
|
|
||||||
|
@patch(
|
||||||
|
"govoplan_files.backend.storage.connector_spaces.ensure_owner_access",
|
||||||
|
side_effect=FileStorageError("File space access denied"),
|
||||||
|
)
|
||||||
|
def test_inaccessible_space_is_blocked_without_mutation(
|
||||||
|
self, _ensure_owner_access: Mock
|
||||||
|
) -> None:
|
||||||
|
with self.assertRaisesRegex(FileStorageError, "access denied"):
|
||||||
|
soft_delete_connector_space(
|
||||||
|
self.session, self.space, user_id="other-user"
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertIsNone(self.space.deleted_at)
|
||||||
|
self.assertTrue(self.space.is_active)
|
||||||
|
self.session.add.assert_not_called()
|
||||||
|
self.session.flush.assert_not_called()
|
||||||
|
|
||||||
|
def test_read_only_mode_does_not_require_remote_write_capability(self) -> None:
|
||||||
|
profile = ConnectorProfile(
|
||||||
|
id="webdav-read",
|
||||||
|
label="Read only",
|
||||||
|
provider="webdav",
|
||||||
|
capabilities=("browse", "import"),
|
||||||
|
)
|
||||||
|
|
||||||
|
validate_connector_space_write_mode(profile, read_only=True)
|
||||||
|
|
||||||
|
def test_two_way_mode_requires_explicit_supported_write_capability(self) -> None:
|
||||||
|
missing_capability = ConnectorProfile(
|
||||||
|
id="s3-read",
|
||||||
|
label="S3 read only",
|
||||||
|
provider="s3",
|
||||||
|
capabilities=("browse",),
|
||||||
|
)
|
||||||
|
unsupported_provider = ConnectorProfile(
|
||||||
|
id="webdav-write",
|
||||||
|
label="WebDAV write",
|
||||||
|
provider="webdav",
|
||||||
|
capabilities=("write",),
|
||||||
|
)
|
||||||
|
|
||||||
|
for profile in (missing_capability, unsupported_provider):
|
||||||
|
with self.subTest(profile=profile.id):
|
||||||
|
with self.assertRaisesRegex(FileStorageError, "requires an S3"):
|
||||||
|
validate_connector_space_write_mode(profile, read_only=False)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -10,6 +10,9 @@ from govoplan_files.backend.storage.connector_visibility import (
|
|||||||
connector_profile_usable_for_import,
|
connector_profile_usable_for_import,
|
||||||
visible_connector_profiles_for_actor,
|
visible_connector_profiles_for_actor,
|
||||||
)
|
)
|
||||||
|
from govoplan_files.backend.storage.connector_providers import (
|
||||||
|
connector_provider_descriptors,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _profile(
|
def _profile(
|
||||||
@@ -168,11 +171,15 @@ class ConnectorVisibilityTests(unittest.TestCase):
|
|||||||
self,
|
self,
|
||||||
) -> None:
|
) -> None:
|
||||||
self.assertTrue(connector_profile_usable_for_import(_profile("webdav")))
|
self.assertTrue(connector_profile_usable_for_import(_profile("webdav")))
|
||||||
self.assertFalse(
|
descriptors = {
|
||||||
connector_profile_usable_for_import(_profile("s3", provider="s3"))
|
item.provider: item for item in connector_provider_descriptors()
|
||||||
)
|
}
|
||||||
self.assertFalse(
|
for provider in ("s3", "smb"):
|
||||||
connector_profile_usable_for_import(_profile("smb", provider="smb"))
|
self.assertEqual(
|
||||||
|
descriptors[provider].installed,
|
||||||
|
connector_profile_usable_for_import(
|
||||||
|
_profile(provider, provider=provider)
|
||||||
|
),
|
||||||
)
|
)
|
||||||
self.assertFalse(
|
self.assertFalse(
|
||||||
connector_profile_usable_for_import(
|
connector_profile_usable_for_import(
|
||||||
|
|||||||
@@ -142,6 +142,7 @@ class FilesRuntimeDocumentationTests(unittest.TestCase):
|
|||||||
base_path="/classified",
|
base_path="/classified",
|
||||||
credential_mode="basic",
|
credential_mode="basic",
|
||||||
password_value="credential-secret",
|
password_value="credential-secret",
|
||||||
|
secret_ref="runtime/connector-secret",
|
||||||
)
|
)
|
||||||
with patch(
|
with patch(
|
||||||
"govoplan_files.backend.documentation.visible_connector_profiles_for_actor",
|
"govoplan_files.backend.documentation.visible_connector_profiles_for_actor",
|
||||||
|
|||||||
@@ -0,0 +1,468 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import unittest
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
|
||||||
|
from sqlalchemy import create_engine
|
||||||
|
from sqlalchemy.orm import sessionmaker
|
||||||
|
|
||||||
|
from govoplan_access.backend.db.models import Account, Group, User
|
||||||
|
from govoplan_core.core.dsar import DsarProvider, DsarSubjectRef
|
||||||
|
from govoplan_core.core.change_sequence import ChangeSequenceEntry
|
||||||
|
from govoplan_core.db.base import Base
|
||||||
|
from govoplan_core.privacy.dsar_workflow import (
|
||||||
|
DataSubjectRequest,
|
||||||
|
create_data_subject_request,
|
||||||
|
execute_data_subject_erasure,
|
||||||
|
plan_data_subject_erasure,
|
||||||
|
search_data_subject_request,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.db.models import (
|
||||||
|
FileAsset,
|
||||||
|
FileBlob,
|
||||||
|
FileConnectorCredential,
|
||||||
|
FileConnectorPolicy,
|
||||||
|
FileConnectorProfile,
|
||||||
|
FileConnectorSpace,
|
||||||
|
FileFolder,
|
||||||
|
FileFormEvidenceGrant,
|
||||||
|
FileIntegrityFinding,
|
||||||
|
FileIntegrityScan,
|
||||||
|
FileShare,
|
||||||
|
FileVersion,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.dsar_provider import (
|
||||||
|
FILES_DSAR_CAPABILITY,
|
||||||
|
FilesDsarProvider,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.manifest import manifest
|
||||||
|
|
||||||
|
|
||||||
|
class _Registry:
|
||||||
|
def __init__(self, provider: FilesDsarProvider, *, files_active: bool = True) -> None:
|
||||||
|
self.provider = provider
|
||||||
|
self.files_active = files_active
|
||||||
|
|
||||||
|
def capability_names(self):
|
||||||
|
return (FILES_DSAR_CAPABILITY,)
|
||||||
|
|
||||||
|
def capability_owner(self, name):
|
||||||
|
self._assert_capability(name)
|
||||||
|
return "files"
|
||||||
|
|
||||||
|
def tenant_entitlement_resolver(self):
|
||||||
|
files_active = self.files_active
|
||||||
|
|
||||||
|
class _Resolver:
|
||||||
|
@staticmethod
|
||||||
|
def resolve(session, tenant_id):
|
||||||
|
del session, tenant_id
|
||||||
|
return type(
|
||||||
|
"State",
|
||||||
|
(),
|
||||||
|
{"effective_modules": ("files",) if files_active else ()},
|
||||||
|
)()
|
||||||
|
|
||||||
|
return _Resolver()
|
||||||
|
|
||||||
|
def require_tenant_capability(self, name, session, **kwargs):
|
||||||
|
del session, kwargs
|
||||||
|
self._assert_capability(name)
|
||||||
|
return self.provider
|
||||||
|
|
||||||
|
def manifests(self):
|
||||||
|
return (type("Manifest", (), {"id": "files"})(),)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _assert_capability(name: str) -> None:
|
||||||
|
if name != FILES_DSAR_CAPABILITY:
|
||||||
|
raise KeyError(name)
|
||||||
|
|
||||||
|
|
||||||
|
class FilesDsarProviderTests(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
self.engine = create_engine("sqlite:///:memory:", future=True)
|
||||||
|
Base.metadata.create_all(
|
||||||
|
bind=self.engine,
|
||||||
|
tables=[
|
||||||
|
Account.__table__,
|
||||||
|
User.__table__,
|
||||||
|
Group.__table__,
|
||||||
|
ChangeSequenceEntry.__table__,
|
||||||
|
DataSubjectRequest.__table__,
|
||||||
|
FileBlob.__table__,
|
||||||
|
FileAsset.__table__,
|
||||||
|
FileVersion.__table__,
|
||||||
|
FileFolder.__table__,
|
||||||
|
FileFormEvidenceGrant.__table__,
|
||||||
|
FileShare.__table__,
|
||||||
|
FileConnectorCredential.__table__,
|
||||||
|
FileConnectorPolicy.__table__,
|
||||||
|
FileConnectorProfile.__table__,
|
||||||
|
FileConnectorSpace.__table__,
|
||||||
|
FileIntegrityScan.__table__,
|
||||||
|
FileIntegrityFinding.__table__,
|
||||||
|
],
|
||||||
|
)
|
||||||
|
self.session = sessionmaker(bind=self.engine, future=True)()
|
||||||
|
self.account = Account(
|
||||||
|
id="account-1",
|
||||||
|
email="subject@example.test",
|
||||||
|
normalized_email="subject@example.test",
|
||||||
|
display_name="Subject",
|
||||||
|
password_hash="not-exported",
|
||||||
|
)
|
||||||
|
self.user = User(
|
||||||
|
id="membership-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
account_id=self.account.id,
|
||||||
|
email="subject@example.test",
|
||||||
|
display_name="Subject",
|
||||||
|
)
|
||||||
|
self.blob = FileBlob(
|
||||||
|
id="blob-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
storage_backend="local",
|
||||||
|
storage_key="private/storage/key-do-not-export",
|
||||||
|
checksum_sha256="a" * 64,
|
||||||
|
size_bytes=12,
|
||||||
|
ref_count=1,
|
||||||
|
)
|
||||||
|
self.asset = FileAsset(
|
||||||
|
id="asset-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
owner_type="user",
|
||||||
|
owner_user_id=self.user.id,
|
||||||
|
created_by_user_id=self.user.id,
|
||||||
|
current_version_id="version-1",
|
||||||
|
display_path="subjects/private.txt",
|
||||||
|
filename="private.txt",
|
||||||
|
description="Subject-provided document",
|
||||||
|
retained_until=datetime.now(timezone.utc) + timedelta(days=30),
|
||||||
|
lifecycle_reason="Pending proceeding",
|
||||||
|
metadata_={"password": "metadata-secret-do-not-export"},
|
||||||
|
)
|
||||||
|
self.version = FileVersion(
|
||||||
|
id="version-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
file_asset_id=self.asset.id,
|
||||||
|
blob_id=self.blob.id,
|
||||||
|
version_number=1,
|
||||||
|
filename_at_upload=self.asset.filename,
|
||||||
|
display_path_at_upload=self.asset.display_path,
|
||||||
|
content_type="text/plain",
|
||||||
|
size_bytes=12,
|
||||||
|
checksum_sha256="a" * 64,
|
||||||
|
created_by_user_id=self.user.id,
|
||||||
|
)
|
||||||
|
self.folder = FileFolder(
|
||||||
|
id="folder-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
owner_type="user",
|
||||||
|
owner_user_id=self.user.id,
|
||||||
|
path="subjects",
|
||||||
|
created_by_user_id=self.user.id,
|
||||||
|
)
|
||||||
|
self.share = FileShare(
|
||||||
|
id="share-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
file_asset_id=self.asset.id,
|
||||||
|
target_type="user",
|
||||||
|
target_id=self.user.id,
|
||||||
|
permission="read",
|
||||||
|
created_by_user_id=self.user.id,
|
||||||
|
)
|
||||||
|
self.form_evidence = FileFormEvidenceGrant(
|
||||||
|
id="evidence-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
form_instance_id="form-instance-1",
|
||||||
|
form_definition_id="application",
|
||||||
|
form_definition_revision="7",
|
||||||
|
token_sha256="b" * 64,
|
||||||
|
idempotency_key="evidence-key-1",
|
||||||
|
request_sha256="c" * 64,
|
||||||
|
custodian_user_id=self.user.id,
|
||||||
|
evidence_kind="attachment",
|
||||||
|
purpose="Submitted application evidence",
|
||||||
|
status="uploaded",
|
||||||
|
expires_at=datetime.now(timezone.utc) + timedelta(days=1),
|
||||||
|
max_size_bytes=1024,
|
||||||
|
allowed_content_types=["text/plain"],
|
||||||
|
file_asset_id=self.asset.id,
|
||||||
|
file_version_id=self.version.id,
|
||||||
|
metadata_={},
|
||||||
|
)
|
||||||
|
self.credential = FileConnectorCredential(
|
||||||
|
id="credential-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
scope_type="tenant",
|
||||||
|
scope_id="tenant-1",
|
||||||
|
label="Subject-created credential",
|
||||||
|
provider="s3",
|
||||||
|
credential_mode="database",
|
||||||
|
username="subject-user",
|
||||||
|
password_encrypted="encrypted-password-do-not-export",
|
||||||
|
token_encrypted="encrypted-token-do-not-export",
|
||||||
|
password_env="PASSWORD_ENV_DO_NOT_EXPORT",
|
||||||
|
secret_ref="vault://do-not-export",
|
||||||
|
created_by_user_id=self.user.id,
|
||||||
|
updated_by_user_id=self.user.id,
|
||||||
|
)
|
||||||
|
self.integrity_scan = FileIntegrityScan(
|
||||||
|
id="scan-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
storage_backend="local",
|
||||||
|
storage_prefix="private-prefix-do-not-export",
|
||||||
|
created_by_user_id=self.user.id,
|
||||||
|
)
|
||||||
|
tenant_two_asset = FileAsset(
|
||||||
|
id="asset-tenant-2",
|
||||||
|
tenant_id="tenant-2",
|
||||||
|
owner_type="user",
|
||||||
|
owner_user_id=self.user.id,
|
||||||
|
display_path="other-tenant.txt",
|
||||||
|
filename="other-tenant.txt",
|
||||||
|
)
|
||||||
|
self.session.add_all(
|
||||||
|
[
|
||||||
|
self.account,
|
||||||
|
self.user,
|
||||||
|
self.blob,
|
||||||
|
self.asset,
|
||||||
|
self.version,
|
||||||
|
self.folder,
|
||||||
|
self.share,
|
||||||
|
self.form_evidence,
|
||||||
|
self.credential,
|
||||||
|
self.integrity_scan,
|
||||||
|
tenant_two_asset,
|
||||||
|
]
|
||||||
|
)
|
||||||
|
self.session.commit()
|
||||||
|
self.provider = FilesDsarProvider()
|
||||||
|
self.subject = DsarSubjectRef(membership_id=self.user.id)
|
||||||
|
|
||||||
|
def tearDown(self) -> None:
|
||||||
|
self.session.close()
|
||||||
|
self.engine.dispose()
|
||||||
|
|
||||||
|
def test_manifest_publishes_protocol_conforming_provider(self) -> None:
|
||||||
|
provided_names = {item.name for item in manifest.provides_interfaces}
|
||||||
|
self.assertIn(FILES_DSAR_CAPABILITY, provided_names)
|
||||||
|
provider = manifest.capability_factories[FILES_DSAR_CAPABILITY](None)
|
||||||
|
self.assertIsInstance(provider, DsarProvider)
|
||||||
|
|
||||||
|
def test_search_is_tenant_scoped_and_excludes_bytes_and_secrets(self) -> None:
|
||||||
|
records = self.provider.search_subject(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
subject=self.subject,
|
||||||
|
)
|
||||||
|
resource_types = {record.resource_type for record in records}
|
||||||
|
self.assertTrue(
|
||||||
|
{
|
||||||
|
"file_asset",
|
||||||
|
"file_version",
|
||||||
|
"file_folder",
|
||||||
|
"file_share",
|
||||||
|
"file_form_evidence",
|
||||||
|
"connector_credential",
|
||||||
|
"file_integrity_scan",
|
||||||
|
}.issubset(resource_types)
|
||||||
|
)
|
||||||
|
serialized = repr([record.to_dict() for record in records])
|
||||||
|
self.assertNotIn("asset-tenant-2", serialized)
|
||||||
|
self.assertNotIn("private/storage/key-do-not-export", serialized)
|
||||||
|
self.assertNotIn("metadata-secret-do-not-export", serialized)
|
||||||
|
self.assertNotIn("encrypted-password-do-not-export", serialized)
|
||||||
|
self.assertNotIn("encrypted-token-do-not-export", serialized)
|
||||||
|
self.assertNotIn("subject-user", serialized)
|
||||||
|
self.assertNotIn("PASSWORD_ENV_DO_NOT_EXPORT", serialized)
|
||||||
|
self.assertNotIn("vault://do-not-export", serialized)
|
||||||
|
self.assertNotIn("private-prefix-do-not-export", serialized)
|
||||||
|
|
||||||
|
def test_conflicting_direct_subject_references_fail_closed(self) -> None:
|
||||||
|
records = self.provider.search_subject(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
subject=DsarSubjectRef(
|
||||||
|
membership_id=self.user.id,
|
||||||
|
external_references={"files.user": "another-membership"},
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual((), records)
|
||||||
|
|
||||||
|
def test_plan_classifies_retention_and_manual_review(self) -> None:
|
||||||
|
records = self.provider.search_subject(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
subject=self.subject,
|
||||||
|
)
|
||||||
|
actions = self.provider.plan_erasure(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
subject=self.subject,
|
||||||
|
records=records,
|
||||||
|
)
|
||||||
|
|
||||||
|
kinds = {action.kind for action in actions}
|
||||||
|
self.assertTrue({"retain", "manual_review", "revoke", "detach"}.issubset(kinds))
|
||||||
|
asset_retention = next(
|
||||||
|
action
|
||||||
|
for action in actions
|
||||||
|
if action.action_id == "files:retain:file_asset:asset-1"
|
||||||
|
)
|
||||||
|
self.assertIn("retained until", asset_retention.rationale)
|
||||||
|
self.assertTrue(
|
||||||
|
any(action.action_id == "files:revoke:file_share:share-1" for action in actions)
|
||||||
|
)
|
||||||
|
self.assertFalse(
|
||||||
|
any(action.kind == "delete" and action.executable for action in actions)
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_execution_is_revalidated_and_idempotent(self) -> None:
|
||||||
|
records = self.provider.search_subject(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
subject=self.subject,
|
||||||
|
)
|
||||||
|
actions = self.provider.plan_erasure(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
subject=self.subject,
|
||||||
|
records=records,
|
||||||
|
)
|
||||||
|
executable = tuple(action for action in actions if action.executable)
|
||||||
|
|
||||||
|
first = self.provider.execute_erasure(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
subject=self.subject,
|
||||||
|
actions=executable,
|
||||||
|
request_id="dsar-1",
|
||||||
|
)
|
||||||
|
self.assertEqual({"executed"}, {result.status for result in first})
|
||||||
|
self.assertIsNotNone(self.share.revoked_at)
|
||||||
|
self.assertIsNone(self.asset.created_by_user_id)
|
||||||
|
self.assertIsNone(self.version.created_by_user_id)
|
||||||
|
self.assertIsNone(self.folder.created_by_user_id)
|
||||||
|
self.assertIsNone(self.credential.created_by_user_id)
|
||||||
|
self.assertIsNone(self.credential.updated_by_user_id)
|
||||||
|
|
||||||
|
repeated = self.provider.execute_erasure(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
subject=self.subject,
|
||||||
|
actions=executable,
|
||||||
|
request_id="dsar-1",
|
||||||
|
)
|
||||||
|
self.assertEqual({"unchanged"}, {result.status for result in repeated})
|
||||||
|
|
||||||
|
def test_execution_blocks_when_reference_changed_after_planning(self) -> None:
|
||||||
|
records = self.provider.search_subject(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
subject=self.subject,
|
||||||
|
)
|
||||||
|
actions = self.provider.plan_erasure(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
subject=self.subject,
|
||||||
|
records=records,
|
||||||
|
)
|
||||||
|
detach = next(
|
||||||
|
action
|
||||||
|
for action in actions
|
||||||
|
if action.action_id
|
||||||
|
== "files:detach:file_version:created_by_user_id:version-1"
|
||||||
|
)
|
||||||
|
self.version.created_by_user_id = "replacement-user"
|
||||||
|
self.session.flush()
|
||||||
|
|
||||||
|
result = self.provider.execute_erasure(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
subject=self.subject,
|
||||||
|
actions=(detach,),
|
||||||
|
request_id="dsar-2",
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual("blocked", result[0].status)
|
||||||
|
self.assertEqual("replacement-user", self.version.created_by_user_id)
|
||||||
|
|
||||||
|
def test_core_workflow_discovers_active_provider_and_skips_it_when_disabled(
|
||||||
|
self,
|
||||||
|
) -> None:
|
||||||
|
request = create_data_subject_request(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
reference="DSAR-FILES-1",
|
||||||
|
request_kind="access_and_erasure",
|
||||||
|
subject=self.subject,
|
||||||
|
purpose="Respond to an authorized privacy request.",
|
||||||
|
legal_basis="Article 15 and 17 GDPR",
|
||||||
|
due_at=None,
|
||||||
|
requested_by_account_id="privacy-officer",
|
||||||
|
)
|
||||||
|
self.session.commit()
|
||||||
|
registry = _Registry(self.provider)
|
||||||
|
|
||||||
|
search_data_subject_request(
|
||||||
|
self.session,
|
||||||
|
registry=registry,
|
||||||
|
row=request,
|
||||||
|
expected_revision=1,
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
"searched", request.status, request.search_result["provider_runs"]
|
||||||
|
)
|
||||||
|
self.assertEqual(["files"], request.coverage["covered_modules"])
|
||||||
|
self.assertEqual([], request.coverage["modules_without_provider"])
|
||||||
|
plan_data_subject_erasure(
|
||||||
|
self.session,
|
||||||
|
registry=registry,
|
||||||
|
row=request,
|
||||||
|
expected_revision=2,
|
||||||
|
)
|
||||||
|
executable_ids = [
|
||||||
|
action["action_id"]
|
||||||
|
for action in request.erasure_plan["actions"]
|
||||||
|
if action["executable"]
|
||||||
|
]
|
||||||
|
execute_data_subject_erasure(
|
||||||
|
self.session,
|
||||||
|
registry=registry,
|
||||||
|
row=request,
|
||||||
|
expected_revision=3,
|
||||||
|
action_ids=executable_ids,
|
||||||
|
)
|
||||||
|
self.assertEqual("completed", request.status)
|
||||||
|
|
||||||
|
disabled = create_data_subject_request(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
reference="DSAR-FILES-DISABLED",
|
||||||
|
request_kind="access",
|
||||||
|
subject=self.subject,
|
||||||
|
purpose="Verify disabled-module coverage.",
|
||||||
|
legal_basis="Article 15 GDPR",
|
||||||
|
due_at=None,
|
||||||
|
requested_by_account_id="privacy-officer",
|
||||||
|
)
|
||||||
|
search_data_subject_request(
|
||||||
|
self.session,
|
||||||
|
registry=_Registry(self.provider, files_active=False),
|
||||||
|
row=disabled,
|
||||||
|
expected_revision=1,
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(0, disabled.search_result["record_count"])
|
||||||
|
self.assertEqual(
|
||||||
|
[FILES_DSAR_CAPABILITY],
|
||||||
|
disabled.coverage["inactive_provider_capabilities"],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,164 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import timedelta
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from sqlalchemy import create_engine
|
||||||
|
from sqlalchemy.orm import sessionmaker
|
||||||
|
|
||||||
|
from govoplan_access.backend.db.models import Account, Group, User
|
||||||
|
from govoplan_core.db.base import Base
|
||||||
|
from govoplan_core.core.change_sequence import ChangeSequenceEntry
|
||||||
|
from govoplan_files.backend.db.models import (
|
||||||
|
FileAsset,
|
||||||
|
FileBlob,
|
||||||
|
FileFolder,
|
||||||
|
FileFormEvidenceGrant,
|
||||||
|
FileShare,
|
||||||
|
FileVersion,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.storage.common import FileStorageError, utcnow
|
||||||
|
from govoplan_files.backend.storage.lifecycle import (
|
||||||
|
preview_asset_purge,
|
||||||
|
restore_asset,
|
||||||
|
set_asset_lifecycle,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class FileLifecycleTests(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
self.engine = create_engine("sqlite:///:memory:", future=True)
|
||||||
|
Base.metadata.create_all(
|
||||||
|
bind=self.engine,
|
||||||
|
tables=[
|
||||||
|
Account.__table__,
|
||||||
|
User.__table__,
|
||||||
|
Group.__table__,
|
||||||
|
ChangeSequenceEntry.__table__,
|
||||||
|
FileBlob.__table__,
|
||||||
|
FileAsset.__table__,
|
||||||
|
FileVersion.__table__,
|
||||||
|
FileFolder.__table__,
|
||||||
|
FileFormEvidenceGrant.__table__,
|
||||||
|
FileShare.__table__,
|
||||||
|
],
|
||||||
|
)
|
||||||
|
self.session = sessionmaker(bind=self.engine, future=True)()
|
||||||
|
self.blob = FileBlob(
|
||||||
|
id="blob-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
storage_backend="local",
|
||||||
|
storage_key="tenants/tenant-1/files/blob-1",
|
||||||
|
checksum_sha256="a" * 64,
|
||||||
|
size_bytes=1,
|
||||||
|
ref_count=1,
|
||||||
|
)
|
||||||
|
self.asset = FileAsset(
|
||||||
|
id="file-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
owner_type="user",
|
||||||
|
owner_user_id="user-1",
|
||||||
|
current_version_id="version-1",
|
||||||
|
display_path="records/report.txt",
|
||||||
|
filename="report.txt",
|
||||||
|
deleted_at=utcnow(),
|
||||||
|
)
|
||||||
|
self.version = FileVersion(
|
||||||
|
id="version-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
file_asset_id="file-1",
|
||||||
|
blob_id="blob-1",
|
||||||
|
version_number=1,
|
||||||
|
filename_at_upload="report.txt",
|
||||||
|
display_path_at_upload="records/report.txt",
|
||||||
|
size_bytes=1,
|
||||||
|
checksum_sha256="a" * 64,
|
||||||
|
)
|
||||||
|
self.session.add_all([self.blob, self.asset, self.version])
|
||||||
|
self.session.commit()
|
||||||
|
|
||||||
|
def tearDown(self) -> None:
|
||||||
|
self.session.close()
|
||||||
|
self.engine.dispose()
|
||||||
|
|
||||||
|
def test_preview_hash_changes_with_retention_and_blocks_active_policy(self) -> None:
|
||||||
|
first = preview_asset_purge(
|
||||||
|
self.session, tenant_id="tenant-1", file_ids=[self.asset.id]
|
||||||
|
)
|
||||||
|
self.assertTrue(first.eligible)
|
||||||
|
|
||||||
|
set_asset_lifecycle(
|
||||||
|
self.session,
|
||||||
|
self.asset,
|
||||||
|
retained_until=utcnow() + timedelta(days=7),
|
||||||
|
legal_hold=True,
|
||||||
|
reason="Active proceeding",
|
||||||
|
expected_revision=1,
|
||||||
|
)
|
||||||
|
self.session.commit()
|
||||||
|
second = preview_asset_purge(
|
||||||
|
self.session, tenant_id="tenant-1", file_ids=[self.asset.id]
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertFalse(second.eligible)
|
||||||
|
self.assertNotEqual(first.preview_sha256, second.preview_sha256)
|
||||||
|
self.assertEqual(
|
||||||
|
{"legal_hold", "retention_active"}, set(second.items[0].blockers)
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_restore_preserves_version_and_provenance_and_rejects_collision(self) -> None:
|
||||||
|
self.asset.metadata_ = {"source": {"provider": "s3", "revision": "v1"}}
|
||||||
|
self.session.commit()
|
||||||
|
|
||||||
|
self.assertTrue(restore_asset(self.session, self.asset))
|
||||||
|
self.assertEqual("version-1", self.asset.current_version_id)
|
||||||
|
self.assertEqual("v1", self.asset.metadata_["source"]["revision"])
|
||||||
|
self.session.commit()
|
||||||
|
|
||||||
|
self.asset.deleted_at = utcnow()
|
||||||
|
collision = FileAsset(
|
||||||
|
id="file-2",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
owner_type="user",
|
||||||
|
owner_user_id="user-1",
|
||||||
|
display_path=self.asset.display_path,
|
||||||
|
filename=self.asset.filename,
|
||||||
|
)
|
||||||
|
self.session.add(collision)
|
||||||
|
self.session.commit()
|
||||||
|
with self.assertRaisesRegex(FileStorageError, "path is already in use"):
|
||||||
|
restore_asset(self.session, self.asset)
|
||||||
|
|
||||||
|
def test_active_share_blocks_purge_until_revoked(self) -> None:
|
||||||
|
share = FileShare(
|
||||||
|
id="share-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
file_asset_id=self.asset.id,
|
||||||
|
target_type="tenant",
|
||||||
|
target_id="tenant-1",
|
||||||
|
permission="read",
|
||||||
|
)
|
||||||
|
self.session.add(share)
|
||||||
|
self.session.commit()
|
||||||
|
|
||||||
|
blocked = preview_asset_purge(
|
||||||
|
self.session, tenant_id="tenant-1", file_ids=[self.asset.id]
|
||||||
|
)
|
||||||
|
self.assertIn("active_share", blocked.items[0].blockers)
|
||||||
|
|
||||||
|
share.revoked_at = utcnow()
|
||||||
|
self.session.commit()
|
||||||
|
allowed = preview_asset_purge(
|
||||||
|
self.session, tenant_id="tenant-1", file_ids=[self.asset.id]
|
||||||
|
)
|
||||||
|
self.assertTrue(allowed.eligible)
|
||||||
|
|
||||||
|
def test_missing_campaign_table_is_an_optional_module_safe_path(self) -> None:
|
||||||
|
preview = preview_asset_purge(
|
||||||
|
self.session, tenant_id="tenant-1", file_ids=[self.asset.id]
|
||||||
|
)
|
||||||
|
self.assertNotIn("campaign_evidence", preview.items[0].blockers)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,333 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import UTC, datetime, timedelta
|
||||||
|
from io import BytesIO
|
||||||
|
from types import SimpleNamespace
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from fastapi import UploadFile
|
||||||
|
from sqlalchemy import create_engine
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
from starlette.datastructures import Headers
|
||||||
|
|
||||||
|
from govoplan_access.backend.db.models import Account, User
|
||||||
|
from govoplan_core.core.access import (
|
||||||
|
CAPABILITY_ACCESS_DIRECTORY,
|
||||||
|
AccessSubjectRef,
|
||||||
|
UserRef,
|
||||||
|
)
|
||||||
|
from govoplan_core.core.change_sequence import ChangeSequenceEntry
|
||||||
|
from govoplan_core.core.form_evidence import (
|
||||||
|
FormEvidenceContractError,
|
||||||
|
FormEvidenceGrantRequest,
|
||||||
|
FormEvidenceInspectionRequest,
|
||||||
|
)
|
||||||
|
from govoplan_core.core.institutional import EvidenceReference, InstitutionalReference
|
||||||
|
from govoplan_core.db.base import Base
|
||||||
|
from govoplan_files.backend.db.models import (
|
||||||
|
FileAsset,
|
||||||
|
FileBlob,
|
||||||
|
FileFormEvidenceGrant,
|
||||||
|
FileVersion,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.form_evidence import FilesFormEvidenceProvider
|
||||||
|
from govoplan_files.backend.routes.form_evidence import upload_form_evidence
|
||||||
|
|
||||||
|
|
||||||
|
class _Directory:
|
||||||
|
def __init__(self, user: UserRef) -> None:
|
||||||
|
self._user = user
|
||||||
|
|
||||||
|
def get_account(self, account_id: str):
|
||||||
|
del account_id
|
||||||
|
return None
|
||||||
|
|
||||||
|
def get_user(self, user_id: str):
|
||||||
|
return self._user if user_id == self._user.id else None
|
||||||
|
|
||||||
|
def get_users(self, user_ids):
|
||||||
|
return {user_id: self._user for user_id in user_ids if user_id == self._user.id}
|
||||||
|
|
||||||
|
def users_for_tenant(self, tenant_id: str):
|
||||||
|
return (self._user,) if tenant_id == self._user.tenant_id else ()
|
||||||
|
|
||||||
|
def get_group(self, group_id: str):
|
||||||
|
del group_id
|
||||||
|
return None
|
||||||
|
|
||||||
|
def get_groups(self, group_ids):
|
||||||
|
del group_ids
|
||||||
|
return {}
|
||||||
|
|
||||||
|
def groups_for_tenant(self, tenant_id: str):
|
||||||
|
del tenant_id
|
||||||
|
return ()
|
||||||
|
|
||||||
|
def groups_for_user(self, user_id: str, *, tenant_id: str):
|
||||||
|
del user_id, tenant_id
|
||||||
|
return ()
|
||||||
|
|
||||||
|
def display_label(self, subject: AccessSubjectRef):
|
||||||
|
del subject
|
||||||
|
return self._user.display_name
|
||||||
|
|
||||||
|
|
||||||
|
class _Registry:
|
||||||
|
def __init__(self, directory: _Directory) -> None:
|
||||||
|
self._directory = directory
|
||||||
|
|
||||||
|
def has_capability(self, name: str) -> bool:
|
||||||
|
return name == CAPABILITY_ACCESS_DIRECTORY
|
||||||
|
|
||||||
|
def require_capability(self, name: str):
|
||||||
|
if not self.has_capability(name):
|
||||||
|
raise KeyError(name)
|
||||||
|
return self._directory
|
||||||
|
|
||||||
|
|
||||||
|
class FilesFormEvidenceTests(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
self.engine = create_engine("sqlite+pysqlite:///:memory:")
|
||||||
|
Base.metadata.create_all(
|
||||||
|
self.engine,
|
||||||
|
tables=[
|
||||||
|
Account.__table__,
|
||||||
|
User.__table__,
|
||||||
|
FileBlob.__table__,
|
||||||
|
FileAsset.__table__,
|
||||||
|
FileVersion.__table__,
|
||||||
|
FileFormEvidenceGrant.__table__,
|
||||||
|
ChangeSequenceEntry.__table__,
|
||||||
|
],
|
||||||
|
)
|
||||||
|
self.session = Session(self.engine)
|
||||||
|
user_ref = UserRef(
|
||||||
|
id="user-1",
|
||||||
|
account_id="account-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
email="user@example.test",
|
||||||
|
display_name="Evidence Custodian",
|
||||||
|
)
|
||||||
|
self.provider = FilesFormEvidenceProvider(
|
||||||
|
_Registry(_Directory(user_ref)),
|
||||||
|
SimpleNamespace(file_upload_max_bytes=2_000_000),
|
||||||
|
)
|
||||||
|
self.principal = SimpleNamespace(tenant_id="tenant-1")
|
||||||
|
self.definition_ref = InstitutionalReference(
|
||||||
|
kind="form",
|
||||||
|
owner_module="forms",
|
||||||
|
object_id="form-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
version="3",
|
||||||
|
)
|
||||||
|
|
||||||
|
def tearDown(self) -> None:
|
||||||
|
self.session.close()
|
||||||
|
self.engine.dispose()
|
||||||
|
|
||||||
|
def request(self, **overrides: object) -> FormEvidenceGrantRequest:
|
||||||
|
values: dict[str, object] = {
|
||||||
|
"tenant_id": "tenant-1",
|
||||||
|
"instance_id": "instance-1",
|
||||||
|
"definition_ref": self.definition_ref,
|
||||||
|
"evidence_kind": "document",
|
||||||
|
"purpose": "supporting document",
|
||||||
|
"idempotency_key": "grant-1",
|
||||||
|
"expires_at": datetime.now(UTC) + timedelta(hours=1),
|
||||||
|
"custodian_ref": "user:user-1",
|
||||||
|
"max_size_bytes": 1_000_000,
|
||||||
|
"allowed_content_types": ("application/pdf",),
|
||||||
|
}
|
||||||
|
values.update(overrides)
|
||||||
|
return FormEvidenceGrantRequest(**values) # type: ignore[arg-type]
|
||||||
|
|
||||||
|
def test_grant_persists_only_token_hash_and_replays_without_secret(self) -> None:
|
||||||
|
request = self.request()
|
||||||
|
issued = self.provider.create_upload_grant(
|
||||||
|
self.session,
|
||||||
|
self.principal,
|
||||||
|
request=request,
|
||||||
|
)
|
||||||
|
row = self.session.query(FileFormEvidenceGrant).one()
|
||||||
|
|
||||||
|
self.assertIsNotNone(issued.upload_token)
|
||||||
|
self.assertNotEqual(issued.upload_token, row.token_sha256)
|
||||||
|
self.assertLessEqual(
|
||||||
|
issued.expires_at,
|
||||||
|
datetime.now(UTC) + timedelta(minutes=15, seconds=1),
|
||||||
|
)
|
||||||
|
replay = self.provider.create_upload_grant(
|
||||||
|
self.session,
|
||||||
|
self.principal,
|
||||||
|
request=request,
|
||||||
|
)
|
||||||
|
self.assertTrue(replay.replayed)
|
||||||
|
self.assertIsNone(replay.upload_token)
|
||||||
|
self.assertEqual(issued.grant_id, replay.grant_id)
|
||||||
|
|
||||||
|
def test_grant_capacity_counts_outstanding_uploads_but_allows_replay(self) -> None:
|
||||||
|
request = self.request(metadata={"remaining_attachments": 1})
|
||||||
|
self.provider.create_upload_grant(self.session, self.principal, request=request)
|
||||||
|
|
||||||
|
replay = self.provider.create_upload_grant(
|
||||||
|
self.session,
|
||||||
|
self.principal,
|
||||||
|
request=request,
|
||||||
|
)
|
||||||
|
self.assertTrue(replay.replayed)
|
||||||
|
with self.assertRaisesRegex(
|
||||||
|
FormEvidenceContractError,
|
||||||
|
"maximum number of active attachment uploads",
|
||||||
|
):
|
||||||
|
self.provider.create_upload_grant(
|
||||||
|
self.session,
|
||||||
|
self.principal,
|
||||||
|
request=self.request(
|
||||||
|
idempotency_key="grant-2",
|
||||||
|
metadata={"remaining_attachments": 1},
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_idempotency_key_cannot_be_reused_for_another_request(self) -> None:
|
||||||
|
self.provider.create_upload_grant(
|
||||||
|
self.session,
|
||||||
|
self.principal,
|
||||||
|
request=self.request(),
|
||||||
|
)
|
||||||
|
with self.assertRaisesRegex(
|
||||||
|
FormEvidenceContractError,
|
||||||
|
"idempotency conflict",
|
||||||
|
):
|
||||||
|
self.provider.create_upload_grant(
|
||||||
|
self.session,
|
||||||
|
self.principal,
|
||||||
|
request=self.request(purpose="another purpose"),
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_inspection_accepts_only_exact_verified_managed_version(self) -> None:
|
||||||
|
issued = self.provider.create_upload_grant(
|
||||||
|
self.session,
|
||||||
|
self.principal,
|
||||||
|
request=self.request(),
|
||||||
|
)
|
||||||
|
grant = self.session.get(FileFormEvidenceGrant, issued.grant_id)
|
||||||
|
assert grant is not None
|
||||||
|
blob = FileBlob(
|
||||||
|
id="blob-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
storage_backend="local",
|
||||||
|
storage_key="tenant-1/blob-1",
|
||||||
|
checksum_sha256="a" * 64,
|
||||||
|
size_bytes=100,
|
||||||
|
integrity_status="verified",
|
||||||
|
)
|
||||||
|
asset = FileAsset(
|
||||||
|
id="asset-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
owner_type="user",
|
||||||
|
owner_user_id="user-1",
|
||||||
|
current_version_id="version-1",
|
||||||
|
display_path="Form submissions/instance-1/evidence.pdf",
|
||||||
|
filename="evidence.pdf",
|
||||||
|
)
|
||||||
|
version = FileVersion(
|
||||||
|
id="version-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
file_asset_id="asset-1",
|
||||||
|
blob_id="blob-1",
|
||||||
|
version_number=1,
|
||||||
|
filename_at_upload="evidence.pdf",
|
||||||
|
display_path_at_upload=asset.display_path,
|
||||||
|
content_type="application/pdf",
|
||||||
|
size_bytes=100,
|
||||||
|
checksum_sha256="a" * 64,
|
||||||
|
)
|
||||||
|
self.session.add_all((blob, asset, version))
|
||||||
|
grant.status = "uploaded"
|
||||||
|
grant.file_asset_id = asset.id
|
||||||
|
grant.file_version_id = version.id
|
||||||
|
self.session.flush()
|
||||||
|
reference = EvidenceReference(
|
||||||
|
kind="document",
|
||||||
|
owner_module="files",
|
||||||
|
evidence_id=asset.id,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
version=version.id,
|
||||||
|
checksum="a" * 64,
|
||||||
|
)
|
||||||
|
|
||||||
|
accepted = self.provider.inspect_evidence(
|
||||||
|
self.session,
|
||||||
|
self.principal,
|
||||||
|
request=FormEvidenceInspectionRequest(
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
instance_id="instance-1",
|
||||||
|
definition_ref=self.definition_ref,
|
||||||
|
evidence=reference,
|
||||||
|
purpose="final submission",
|
||||||
|
final=True,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
self.assertTrue(accepted.accepted)
|
||||||
|
|
||||||
|
rejected = self.provider.inspect_evidence(
|
||||||
|
self.session,
|
||||||
|
self.principal,
|
||||||
|
request=FormEvidenceInspectionRequest(
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
instance_id="another-instance",
|
||||||
|
definition_ref=self.definition_ref,
|
||||||
|
evidence=reference,
|
||||||
|
purpose="final submission",
|
||||||
|
final=True,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
self.assertEqual("rejected", rejected.state)
|
||||||
|
|
||||||
|
def test_public_upload_consumes_grant_and_returns_exact_evidence(self) -> None:
|
||||||
|
issued = self.provider.create_upload_grant(
|
||||||
|
self.session,
|
||||||
|
self.principal,
|
||||||
|
request=self.request(),
|
||||||
|
)
|
||||||
|
assert issued.upload_token is not None
|
||||||
|
stored = SimpleNamespace(
|
||||||
|
asset=SimpleNamespace(id="asset-uploaded"),
|
||||||
|
version=SimpleNamespace(
|
||||||
|
id="version-uploaded",
|
||||||
|
checksum_sha256="c" * 64,
|
||||||
|
size_bytes=8,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
upload = UploadFile(
|
||||||
|
filename="evidence.pdf",
|
||||||
|
file=BytesIO(b"evidence"),
|
||||||
|
headers=Headers({"content-type": "application/pdf"}),
|
||||||
|
)
|
||||||
|
|
||||||
|
with (
|
||||||
|
patch(
|
||||||
|
"govoplan_files.backend.routes.form_evidence.create_file_asset",
|
||||||
|
return_value=stored,
|
||||||
|
) as create,
|
||||||
|
patch("govoplan_files.backend.routes.form_evidence.emit_platform_event"),
|
||||||
|
):
|
||||||
|
response = upload_form_evidence(
|
||||||
|
file=upload,
|
||||||
|
x_form_evidence_token=issued.upload_token,
|
||||||
|
session=self.session,
|
||||||
|
)
|
||||||
|
|
||||||
|
grant = self.session.get(FileFormEvidenceGrant, issued.grant_id)
|
||||||
|
assert grant is not None
|
||||||
|
self.assertEqual("uploaded", grant.status)
|
||||||
|
self.assertEqual("asset-uploaded", grant.file_asset_id)
|
||||||
|
self.assertEqual("version-uploaded", grant.file_version_id)
|
||||||
|
self.assertEqual("asset-uploaded", response["evidence"]["evidence_id"])
|
||||||
|
self.assertEqual(b"evidence", create.call_args.kwargs["data"])
|
||||||
|
self.assertEqual("user-1", create.call_args.kwargs["owner_id"])
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -6,6 +6,7 @@ import unittest
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from unittest.mock import patch
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from fastapi import HTTPException
|
||||||
from sqlalchemy import create_engine
|
from sqlalchemy import create_engine
|
||||||
from sqlalchemy.orm import sessionmaker
|
from sqlalchemy.orm import sessionmaker
|
||||||
|
|
||||||
@@ -16,6 +17,7 @@ from govoplan_files.backend.db.models import (
|
|||||||
FileIntegrityFinding,
|
FileIntegrityFinding,
|
||||||
FileIntegrityScan,
|
FileIntegrityScan,
|
||||||
)
|
)
|
||||||
|
from govoplan_files.backend.routes.integrity import _assert_expected_revision
|
||||||
from govoplan_files.backend.storage.backends import LocalFilesystemStorageBackend
|
from govoplan_files.backend.storage.backends import LocalFilesystemStorageBackend
|
||||||
from govoplan_files.backend.storage.common import FileStorageError
|
from govoplan_files.backend.storage.common import FileStorageError
|
||||||
from govoplan_files.backend.storage.integrity import (
|
from govoplan_files.backend.storage.integrity import (
|
||||||
@@ -82,6 +84,7 @@ class IntegrityReconciliationTests(unittest.TestCase):
|
|||||||
backend=self.backend,
|
backend=self.backend,
|
||||||
)
|
)
|
||||||
self.session.commit()
|
self.session.commit()
|
||||||
|
self.assertEqual(1, scan.revision)
|
||||||
|
|
||||||
invocations = 0
|
invocations = 0
|
||||||
while scan.status != "completed":
|
while scan.status != "completed":
|
||||||
@@ -99,6 +102,7 @@ class IntegrityReconciliationTests(unittest.TestCase):
|
|||||||
self.fail("Integrity scan did not complete")
|
self.fail("Integrity scan did not complete")
|
||||||
|
|
||||||
self.assertGreater(invocations, 3)
|
self.assertGreater(invocations, 3)
|
||||||
|
self.assertEqual(1 + invocations, scan.revision)
|
||||||
self.assertEqual(3, scan.scanned_blob_count)
|
self.assertEqual(3, scan.scanned_blob_count)
|
||||||
self.assertEqual(1, scan.verified_blob_count)
|
self.assertEqual(1, scan.verified_blob_count)
|
||||||
self.assertEqual(2, scan.quarantined_blob_count)
|
self.assertEqual(2, scan.quarantined_blob_count)
|
||||||
@@ -183,6 +187,13 @@ class IntegrityReconciliationTests(unittest.TestCase):
|
|||||||
self.assertEqual("already_deleted", repeated.action)
|
self.assertEqual("already_deleted", repeated.action)
|
||||||
self.assertFalse(repeated.changed)
|
self.assertFalse(repeated.changed)
|
||||||
|
|
||||||
|
def test_stale_integrity_action_revision_is_rejected(self) -> None:
|
||||||
|
with self.assertRaises(HTTPException) as captured:
|
||||||
|
_assert_expected_revision(3, 2)
|
||||||
|
|
||||||
|
self.assertEqual(409, captured.exception.status_code)
|
||||||
|
self.assertIn("reload", str(captured.exception.detail).lower())
|
||||||
|
|
||||||
|
|
||||||
def _blob(blob_id: str, filename: str, expected_data: bytes) -> FileBlob:
|
def _blob(blob_id: str, filename: str, expected_data: bytes) -> FileBlob:
|
||||||
return FileBlob(
|
return FileBlob(
|
||||||
|
|||||||
@@ -4,15 +4,24 @@ import unittest
|
|||||||
|
|
||||||
|
|
||||||
STATIC_TOPIC_IDS = {
|
STATIC_TOPIC_IDS = {
|
||||||
|
"files.configuration-package.managed-storage",
|
||||||
|
"files.quick-access-and-product-area",
|
||||||
|
"files.search.managed-content",
|
||||||
"files.workflow.organize-managed-files",
|
"files.workflow.organize-managed-files",
|
||||||
"files.workflow.find-and-download-files",
|
"files.workflow.find-and-download-files",
|
||||||
"files.workflow.share-managed-files",
|
"files.workflow.share-managed-files",
|
||||||
"files.workflow.delete-managed-files",
|
"files.workflow.delete-managed-files",
|
||||||
|
"files.workflow.restore-retain-and-purge",
|
||||||
|
"files.privacy.data-subject-requests",
|
||||||
"files.governed-connectors-and-provenance",
|
"files.governed-connectors-and-provenance",
|
||||||
"files.reference.integrity-recovery-and-fail-closed-transports",
|
"files.reference.integrity-recovery-and-fail-closed-transports",
|
||||||
"files.reference.shared-storage-profile",
|
"files.reference.shared-storage-profile",
|
||||||
"files.reference.generated-artifact-store",
|
"files.reference.generated-artifact-store",
|
||||||
"files.reference.snapshot-provenance-and-capabilities",
|
"files.reference.snapshot-provenance-and-capabilities",
|
||||||
|
"files.records.exact-version-source",
|
||||||
|
"files.forms-runtime.managed-evidence",
|
||||||
|
"files.postbox.exact-version-references",
|
||||||
|
"files.tabular-content",
|
||||||
"files.assurance.process-and-release-readiness",
|
"files.assurance.process-and-release-readiness",
|
||||||
}
|
}
|
||||||
RUNTIME_TOPIC_IDS = {
|
RUNTIME_TOPIC_IDS = {
|
||||||
@@ -113,9 +122,22 @@ class FilesManifestDocumentationTests(unittest.TestCase):
|
|||||||
|
|
||||||
delete = self.topic("files.workflow.delete-managed-files")
|
delete = self.topic("files.workflow.delete-managed-files")
|
||||||
self.assertIn("Soft-delete", delete.summary)
|
self.assertIn("Soft-delete", delete.summary)
|
||||||
self.assertIn("no self-service restore or hard-purge", delete.body)
|
self.assertIn("Authorized restoration", delete.body)
|
||||||
self.assertTrue(
|
self.assertTrue(
|
||||||
any("not a hard purge" in item for item in delete.metadata["limitations"])
|
any("soft deletion" in item.casefold() for item in delete.metadata["limitations"])
|
||||||
|
)
|
||||||
|
|
||||||
|
lifecycle = self.topic("files.workflow.restore-retain-and-purge")
|
||||||
|
self.assertIn("legal hold", lifecycle.body)
|
||||||
|
self.assertIn("preview hash", lifecycle.body)
|
||||||
|
self.assertIn("recovery ledger", lifecycle.body)
|
||||||
|
|
||||||
|
privacy = self.topic("files.privacy.data-subject-requests")
|
||||||
|
self.assertEqual(("admin",), privacy.documentation_types)
|
||||||
|
self.assertIn("without raw file bytes", privacy.body)
|
||||||
|
self.assertIn("separately authorized Files purge", privacy.body)
|
||||||
|
self.assertTrue(
|
||||||
|
any("membership" in item for item in privacy.metadata["limitations"])
|
||||||
)
|
)
|
||||||
|
|
||||||
def test_admin_topic_covers_policy_redaction_and_atomic_credential_deletion(
|
def test_admin_topic_covers_policy_redaction_and_atomic_credential_deletion(
|
||||||
@@ -138,6 +160,9 @@ class FilesManifestDocumentationTests(unittest.TestCase):
|
|||||||
self.assertIn("deny rules win", topic.body)
|
self.assertIn("deny rules win", topic.body)
|
||||||
self.assertIn("redact secret values", topic.body)
|
self.assertIn("redact secret values", topic.body)
|
||||||
self.assertIn("same transaction", topic.body)
|
self.assertIn("same transaction", topic.body)
|
||||||
|
self.assertIn("read-only by default", topic.body)
|
||||||
|
self.assertIn("S3", topic.body)
|
||||||
|
self.assertIn("Automatic remote deletion", topic.body)
|
||||||
self.assertTrue(
|
self.assertTrue(
|
||||||
any(
|
any(
|
||||||
"non-owned external references" in item
|
"non-owned external references" in item
|
||||||
@@ -152,7 +177,7 @@ class FilesManifestDocumentationTests(unittest.TestCase):
|
|||||||
"/api/v1/files/connectors/credentials", {link.href for link in topic.links}
|
"/api/v1/files/connectors/credentials", {link.href for link in topic.links}
|
||||||
)
|
)
|
||||||
|
|
||||||
def test_operator_topic_covers_recovery_and_fail_closed_s3_smb(self) -> None:
|
def test_operator_topic_covers_recovery_and_pinned_s3_smb(self) -> None:
|
||||||
topic = self.topic(
|
topic = self.topic(
|
||||||
"files.reference.integrity-recovery-and-fail-closed-transports"
|
"files.reference.integrity-recovery-and-fail-closed-transports"
|
||||||
)
|
)
|
||||||
@@ -163,18 +188,31 @@ class FilesManifestDocumentationTests(unittest.TestCase):
|
|||||||
self.assertIn("SMB", topic.body)
|
self.assertIn("SMB", topic.body)
|
||||||
self.assertIn("fail closed", topic.body)
|
self.assertIn("fail closed", topic.body)
|
||||||
self.assertIn("DFS referrals", topic.body)
|
self.assertIn("DFS referrals", topic.body)
|
||||||
|
self.assertIn("ambient credential discovery", topic.body)
|
||||||
self.assertIn("does not remove backend blob objects", topic.body)
|
self.assertIn("does not remove backend blob objects", topic.body)
|
||||||
self.assertIn("bounded resumable integrity scan", topic.body)
|
self.assertIn("bounded resumable integrity scan", topic.body)
|
||||||
self.assertIn("quarantined", topic.body)
|
self.assertIn("quarantined", topic.body)
|
||||||
self.assertIn("MASTER_KEY_B64", topic.metadata["recovery_unit"])
|
self.assertIn("MASTER_KEY_B64", topic.metadata["recovery_unit"])
|
||||||
self.assertIn("Encryption envelope and wrapped-key rows", topic.metadata["recovery_unit"])
|
self.assertIn(
|
||||||
|
"Encryption envelope and wrapped-key rows", topic.metadata["recovery_unit"]
|
||||||
|
)
|
||||||
self.assertIn("lease-fenced Core recovery", topic.body)
|
self.assertIn("lease-fenced Core recovery", topic.body)
|
||||||
self.assertIn("Ops", topic.body)
|
self.assertIn("Ops", topic.body)
|
||||||
|
self.assertIn("Hard purge", topic.body)
|
||||||
|
self.assertIn("S3 connector write-back", topic.body)
|
||||||
self.assertTrue(topic.metadata["verification"])
|
self.assertTrue(topic.metadata["verification"])
|
||||||
self.assertIn(
|
self.assertIn(
|
||||||
"/api/v1/files/integrity/scans",
|
"/api/v1/files/integrity/scans",
|
||||||
{link.href for link in topic.links},
|
{link.href for link in topic.links},
|
||||||
)
|
)
|
||||||
|
self.assertIn(
|
||||||
|
"/admin?section=tenant-file-integrity",
|
||||||
|
{link.href for link in topic.links},
|
||||||
|
)
|
||||||
|
self.assertIn(
|
||||||
|
"files.admin.tenant-integrity",
|
||||||
|
{surface.id for surface in self.manifest.frontend.view_surfaces},
|
||||||
|
)
|
||||||
self.assertIn(
|
self.assertIn(
|
||||||
"GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS", topic.configuration_keys
|
"GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS", topic.configuration_keys
|
||||||
)
|
)
|
||||||
@@ -185,7 +223,11 @@ class FilesManifestDocumentationTests(unittest.TestCase):
|
|||||||
self.assertEqual(("admin", "user"), topic.documentation_types)
|
self.assertEqual(("admin", "user"), topic.documentation_types)
|
||||||
self.assertEqual("reference", topic.metadata["kind"])
|
self.assertEqual("reference", topic.metadata["kind"])
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
["files.access@0.1.6", "files.campaign_attachments@0.1.6"],
|
[
|
||||||
|
"files.access@0.1.6",
|
||||||
|
"files.campaign_attachments@0.1.6",
|
||||||
|
"forms_runtime.evidence.files@1.0.0",
|
||||||
|
],
|
||||||
topic.metadata["provided_interfaces"],
|
topic.metadata["provided_interfaces"],
|
||||||
)
|
)
|
||||||
self.assertIn("revision", topic.metadata["provenance_fields"])
|
self.assertIn("revision", topic.metadata["provenance_fields"])
|
||||||
@@ -203,8 +245,9 @@ class FilesManifestDocumentationTests(unittest.TestCase):
|
|||||||
self.assertIn("process_owner", topic.audience)
|
self.assertIn("process_owner", topic.audience)
|
||||||
self.assertIn("release_manager", topic.audience)
|
self.assertIn("release_manager", topic.audience)
|
||||||
self.assertIn("versions align", topic.body)
|
self.assertIn("versions align", topic.body)
|
||||||
self.assertIn("Share grant, change, expiry, and revocation", topic.body)
|
self.assertIn("share lifecycle", topic.body)
|
||||||
self.assertIn("no enforced retention or legal hold", topic.body)
|
self.assertIn("legal hold", topic.body)
|
||||||
|
self.assertIn("S3 write-back", topic.body)
|
||||||
for key in ("prerequisites", "steps", "outcome", "verification"):
|
for key in ("prerequisites", "steps", "outcome", "verification"):
|
||||||
self.assertTrue(topic.metadata[key])
|
self.assertTrue(topic.metadata[key])
|
||||||
self.assertTrue(
|
self.assertTrue(
|
||||||
|
|||||||
@@ -0,0 +1,42 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from alembic.runtime.migration import MigrationContext
|
||||||
|
from sqlalchemy import create_engine, inspect
|
||||||
|
|
||||||
|
from govoplan_access.backend.manifest import get_manifest as get_access_manifest
|
||||||
|
from govoplan_core.db.migrations import migrate_database
|
||||||
|
from govoplan_files.backend.manifest import get_manifest as get_files_manifest
|
||||||
|
|
||||||
|
|
||||||
|
class FilesMigrationTests(unittest.TestCase):
|
||||||
|
def test_fresh_migration_creates_form_evidence_grants_and_head(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory(
|
||||||
|
prefix="govoplan-files-migration-"
|
||||||
|
) as directory:
|
||||||
|
url = f"sqlite:///{Path(directory) / 'files.db'}"
|
||||||
|
migrate_database(
|
||||||
|
database_url=url,
|
||||||
|
enabled_modules=("access", "files"),
|
||||||
|
manifest_factories=(get_access_manifest, get_files_manifest),
|
||||||
|
)
|
||||||
|
engine = create_engine(url)
|
||||||
|
try:
|
||||||
|
self.assertIn(
|
||||||
|
"file_form_evidence_grants",
|
||||||
|
inspect(engine).get_table_names(),
|
||||||
|
)
|
||||||
|
with engine.connect() as connection:
|
||||||
|
self.assertIn(
|
||||||
|
"a2b3c4d5e6f9",
|
||||||
|
set(MigrationContext.configure(connection).get_current_heads()),
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
engine.dispose()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,160 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from types import SimpleNamespace
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from govoplan_core.core.files import PostboxFileReferenceRequest
|
||||||
|
from govoplan_files.backend.capabilities import FilesPostboxReferenceService
|
||||||
|
from govoplan_files.backend.db.models import FileAsset, FileBlob, FileVersion
|
||||||
|
from govoplan_files.backend.storage.common import FileStorageError
|
||||||
|
|
||||||
|
|
||||||
|
class _Principal:
|
||||||
|
def __init__(self, *, tenant_id: str = "tenant-1", scopes: set[str] | None = None):
|
||||||
|
self.tenant_id = tenant_id
|
||||||
|
self.user = SimpleNamespace(id="user-1")
|
||||||
|
self._scopes = {"files:file:download"} if scopes is None else scopes
|
||||||
|
|
||||||
|
def has(self, scope: str) -> bool:
|
||||||
|
return scope in self._scopes
|
||||||
|
|
||||||
|
|
||||||
|
class _Session:
|
||||||
|
def __init__(self, values: dict[tuple[type[object], str], object]):
|
||||||
|
self.values = values
|
||||||
|
|
||||||
|
def get(self, model: type[object], resource_id: str):
|
||||||
|
return self.values.get((model, resource_id))
|
||||||
|
|
||||||
|
|
||||||
|
def _request(reference_type: str = "file_version", reference_id: str = "version-1"):
|
||||||
|
return PostboxFileReferenceRequest(
|
||||||
|
postbox_id="postbox-1",
|
||||||
|
message_id="message-1",
|
||||||
|
reference_type=reference_type,
|
||||||
|
reference_id=reference_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _objects():
|
||||||
|
asset = SimpleNamespace(
|
||||||
|
id="file-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
current_version_id="version-2",
|
||||||
|
deleted_at=None,
|
||||||
|
filename="current.txt",
|
||||||
|
display_path="Evidence/current.txt",
|
||||||
|
)
|
||||||
|
version = SimpleNamespace(
|
||||||
|
id="version-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
file_asset_id="file-1",
|
||||||
|
blob_id="blob-1",
|
||||||
|
filename_at_upload="evidence.txt",
|
||||||
|
content_type="text/plain",
|
||||||
|
size_bytes=8,
|
||||||
|
checksum_sha256="a" * 64,
|
||||||
|
)
|
||||||
|
blob = SimpleNamespace(
|
||||||
|
id="blob-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
content_type="text/plain",
|
||||||
|
)
|
||||||
|
return asset, version, blob
|
||||||
|
|
||||||
|
|
||||||
|
class FilesPostboxReferenceTests(unittest.TestCase):
|
||||||
|
def test_resolves_the_referenced_version_instead_of_the_current_version(self) -> None:
|
||||||
|
asset, version, blob = _objects()
|
||||||
|
session = _Session(
|
||||||
|
{
|
||||||
|
(FileAsset, asset.id): asset,
|
||||||
|
(FileVersion, version.id): version,
|
||||||
|
(FileBlob, blob.id): blob,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
with patch(
|
||||||
|
"govoplan_files.backend.capabilities.get_asset_for_user",
|
||||||
|
return_value=asset,
|
||||||
|
):
|
||||||
|
result = FilesPostboxReferenceService().resolve_postbox_references(
|
||||||
|
session,
|
||||||
|
_Principal(),
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
requests=(_request(),),
|
||||||
|
)[0]
|
||||||
|
|
||||||
|
self.assertTrue(result.available)
|
||||||
|
self.assertEqual("version-1", result.file_version_id)
|
||||||
|
self.assertEqual(
|
||||||
|
"/api/v1/files/file-1/versions/version-1/download",
|
||||||
|
result.download_path,
|
||||||
|
)
|
||||||
|
self.assertTrue(result.provenance["exact_version"])
|
||||||
|
|
||||||
|
def test_fails_closed_for_tenant_permission_and_file_access_mismatches(self) -> None:
|
||||||
|
service = FilesPostboxReferenceService()
|
||||||
|
empty = _Session({})
|
||||||
|
|
||||||
|
tenant_result = service.resolve_postbox_references(
|
||||||
|
empty,
|
||||||
|
_Principal(tenant_id="tenant-2"),
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
requests=(_request(),),
|
||||||
|
)[0]
|
||||||
|
permission_result = service.resolve_postbox_references(
|
||||||
|
empty,
|
||||||
|
_Principal(scopes=set()),
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
requests=(_request(),),
|
||||||
|
)[0]
|
||||||
|
|
||||||
|
asset, version, blob = _objects()
|
||||||
|
populated = _Session(
|
||||||
|
{
|
||||||
|
(FileAsset, asset.id): asset,
|
||||||
|
(FileVersion, version.id): version,
|
||||||
|
(FileBlob, blob.id): blob,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
with patch(
|
||||||
|
"govoplan_files.backend.capabilities.get_asset_for_user",
|
||||||
|
side_effect=FileStorageError("denied"),
|
||||||
|
):
|
||||||
|
access_result = service.resolve_postbox_references(
|
||||||
|
populated,
|
||||||
|
_Principal(),
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
requests=(_request(),),
|
||||||
|
)[0]
|
||||||
|
|
||||||
|
self.assertEqual("tenant_mismatch", tenant_result.reason_code)
|
||||||
|
self.assertEqual("download_permission_missing", permission_result.reason_code)
|
||||||
|
self.assertEqual("file_access_denied", access_result.reason_code)
|
||||||
|
self.assertFalse(access_result.available)
|
||||||
|
|
||||||
|
def test_asset_reference_does_not_drift_to_a_later_current_version(self) -> None:
|
||||||
|
asset, version, blob = _objects()
|
||||||
|
session = _Session(
|
||||||
|
{
|
||||||
|
(FileAsset, asset.id): asset,
|
||||||
|
(FileVersion, version.id): version,
|
||||||
|
(FileBlob, blob.id): blob,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
result = FilesPostboxReferenceService().resolve_postbox_references(
|
||||||
|
session,
|
||||||
|
_Principal(),
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
requests=(_request("file", asset.id),),
|
||||||
|
)[0]
|
||||||
|
|
||||||
|
self.assertFalse(result.available)
|
||||||
|
self.assertEqual("exact_version_required", result.reason_code)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,38 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from govoplan_files.backend.manifest import manifest
|
||||||
|
|
||||||
|
|
||||||
|
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
|
||||||
|
|
||||||
|
class FilesQuickAccessContractTests(unittest.TestCase):
|
||||||
|
def test_manifest_declares_exact_version_result_and_fallback(self) -> None:
|
||||||
|
tool = manifest.frontend.quick_access_tools[0]
|
||||||
|
|
||||||
|
self.assertEqual("files.recent", tool.id)
|
||||||
|
self.assertEqual(("files.file-version",), tool.returned_reference_kinds)
|
||||||
|
self.assertEqual("files.quick_access.files", tool.help_context_id)
|
||||||
|
self.assertEqual("/files", tool.full_page_path)
|
||||||
|
|
||||||
|
def test_renderer_and_page_keep_recent_selection_and_upload_bounded(self) -> None:
|
||||||
|
renderer = (
|
||||||
|
REPOSITORY_ROOT / "webui/src/features/files/FileQuickAccess.tsx"
|
||||||
|
).read_text(encoding="utf-8")
|
||||||
|
page = (
|
||||||
|
REPOSITORY_ROOT / "webui/src/features/files/FilesPage.tsx"
|
||||||
|
).read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
self.assertIn('sort: "recent", page_size: 7', renderer)
|
||||||
|
self.assertIn('kind: "file-version"', renderer)
|
||||||
|
self.assertIn("quickAccessLaunchState(launchContext)", renderer)
|
||||||
|
self.assertIn('to="/files?quickAction=upload"', renderer)
|
||||||
|
self.assertIn('parameters.get("quickAction") !== "upload"', page)
|
||||||
|
self.assertIn('openDialog("upload"', page)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,108 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from types import SimpleNamespace
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import MagicMock, patch
|
||||||
|
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from govoplan_core.core.records import RecordContractError, RecordSourceLocator
|
||||||
|
from govoplan_files.backend.db.models import FileAsset, FileBlob, FileVersion
|
||||||
|
from govoplan_files.backend.record_source import FilesRecordSource
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class Principal:
|
||||||
|
tenant_id: str = "tenant-1"
|
||||||
|
membership_id: str = "user-1"
|
||||||
|
user: object = field(default_factory=lambda: SimpleNamespace(id="user-1"))
|
||||||
|
|
||||||
|
def has(self, scope: str) -> bool:
|
||||||
|
return scope in {"files:file:read"}
|
||||||
|
|
||||||
|
|
||||||
|
class FilesRecordSourceTests(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
self.session = MagicMock(spec=Session)
|
||||||
|
self.asset = FileAsset(
|
||||||
|
id="asset-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
owner_type="user",
|
||||||
|
owner_user_id="user-1",
|
||||||
|
current_version_id="version-1",
|
||||||
|
display_path="Evidence/Decision.pdf",
|
||||||
|
filename="Decision.pdf",
|
||||||
|
)
|
||||||
|
self.blob = FileBlob(
|
||||||
|
id="blob-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
storage_backend="local",
|
||||||
|
storage_key="tenant-1/blob-1",
|
||||||
|
checksum_sha256="a" * 64,
|
||||||
|
size_bytes=1024,
|
||||||
|
integrity_status="verified",
|
||||||
|
)
|
||||||
|
self.version = FileVersion(
|
||||||
|
id="version-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
file_asset_id="asset-1",
|
||||||
|
blob_id="blob-1",
|
||||||
|
version_number=1,
|
||||||
|
filename_at_upload="Decision.pdf",
|
||||||
|
display_path_at_upload="Evidence/Decision.pdf",
|
||||||
|
content_type="application/pdf",
|
||||||
|
size_bytes=1024,
|
||||||
|
checksum_sha256="a" * 64,
|
||||||
|
)
|
||||||
|
self.version.created_at = datetime.now(UTC)
|
||||||
|
version_query = self.session.query.return_value.filter.return_value
|
||||||
|
version_query.filter.return_value.one_or_none.return_value = self.version
|
||||||
|
self.session.get.return_value = self.blob
|
||||||
|
|
||||||
|
def locator(self) -> RecordSourceLocator:
|
||||||
|
return RecordSourceLocator(
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
source_module="files",
|
||||||
|
resource_type="file_version",
|
||||||
|
resource_id="asset-1",
|
||||||
|
source_revision="version-1",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_resolves_exact_authorized_version_and_digest(self) -> None:
|
||||||
|
with patch(
|
||||||
|
"govoplan_files.backend.record_source.get_asset_for_user",
|
||||||
|
return_value=self.asset,
|
||||||
|
):
|
||||||
|
result = FilesRecordSource().resolve(
|
||||||
|
self.session,
|
||||||
|
Principal(),
|
||||||
|
locator=self.locator(),
|
||||||
|
purpose="document decision basis",
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual("version-1", result.locator.source_revision)
|
||||||
|
self.assertEqual("a" * 64, result.content_sha256)
|
||||||
|
self.assertEqual("verified", result.metadata["integrity_status"])
|
||||||
|
|
||||||
|
def test_quarantined_version_fails_closed(self) -> None:
|
||||||
|
self.blob.quarantined_at = datetime.now(UTC)
|
||||||
|
self.session.flush()
|
||||||
|
with (
|
||||||
|
patch(
|
||||||
|
"govoplan_files.backend.record_source.get_asset_for_user",
|
||||||
|
return_value=self.asset,
|
||||||
|
),
|
||||||
|
self.assertRaisesRegex(RecordContractError, "integrity gate"),
|
||||||
|
):
|
||||||
|
FilesRecordSource().resolve(
|
||||||
|
self.session,
|
||||||
|
Principal(),
|
||||||
|
locator=self.locator(),
|
||||||
|
purpose="document decision basis",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -7,11 +7,17 @@ from inspect import signature
|
|||||||
from govoplan_files.backend.router import router
|
from govoplan_files.backend.router import router
|
||||||
from govoplan_files.backend.routes.assets import router as assets_router
|
from govoplan_files.backend.routes.assets import router as assets_router
|
||||||
from govoplan_files.backend.routes.connector_io import router as connector_io_router
|
from govoplan_files.backend.routes.connector_io import router as connector_io_router
|
||||||
from govoplan_files.backend.routes.connector_profiles import router as connector_profiles_router
|
from govoplan_files.backend.routes.connector_profiles import (
|
||||||
from govoplan_files.backend.routes.connector_settings import router as connector_settings_router
|
router as connector_profiles_router,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.routes.connector_settings import (
|
||||||
|
router as connector_settings_router,
|
||||||
|
)
|
||||||
from govoplan_files.backend.routes.folders import router as folders_router
|
from govoplan_files.backend.routes.folders import router as folders_router
|
||||||
|
from govoplan_files.backend.routes.form_evidence import router as form_evidence_router
|
||||||
from govoplan_files.backend.routes.integrity import router as integrity_router
|
from govoplan_files.backend.routes.integrity import router as integrity_router
|
||||||
from govoplan_files.backend.routes.listing import router as listing_router
|
from govoplan_files.backend.routes.listing import router as listing_router
|
||||||
|
from govoplan_files.backend.routes.lifecycle import router as lifecycle_router
|
||||||
from govoplan_files.backend.routes.shares import router as shares_router
|
from govoplan_files.backend.routes.shares import router as shares_router
|
||||||
from govoplan_files.backend.routes.spaces import router as spaces_router
|
from govoplan_files.backend.routes.spaces import router as spaces_router
|
||||||
from govoplan_files.backend.routes.transfers import router as transfers_router
|
from govoplan_files.backend.routes.transfers import router as transfers_router
|
||||||
@@ -31,7 +37,9 @@ class FilesRouterContractTests(unittest.TestCase):
|
|||||||
workflow_routers = (
|
workflow_routers = (
|
||||||
spaces_router,
|
spaces_router,
|
||||||
folders_router,
|
folders_router,
|
||||||
|
form_evidence_router,
|
||||||
integrity_router,
|
integrity_router,
|
||||||
|
lifecycle_router,
|
||||||
listing_router,
|
listing_router,
|
||||||
uploads_router,
|
uploads_router,
|
||||||
connector_settings_router,
|
connector_settings_router,
|
||||||
@@ -49,22 +57,23 @@ class FilesRouterContractTests(unittest.TestCase):
|
|||||||
actual = self._operation_keys(router)
|
actual = self._operation_keys(router)
|
||||||
|
|
||||||
self.assertEqual(expected, actual)
|
self.assertEqual(expected, actual)
|
||||||
self.assertEqual(52, len(actual))
|
self.assertEqual(62, len(actual))
|
||||||
self.assertFalse(
|
self.assertFalse(
|
||||||
[operation for operation, count in Counter(actual).items() if count > 1]
|
[operation for operation, count in Counter(actual).items() if count > 1]
|
||||||
)
|
)
|
||||||
|
|
||||||
def test_archive_preview_and_confirmation_routes_are_exposed(self) -> None:
|
def test_archive_preview_and_confirmation_routes_are_exposed(self) -> None:
|
||||||
routes = {
|
routes = {
|
||||||
(tuple(sorted(route.methods or ())), route.path)
|
(tuple(sorted(route.methods or ())), route.path) for route in router.routes
|
||||||
for route in router.routes
|
|
||||||
}
|
}
|
||||||
|
|
||||||
self.assertIn((("POST",), "/files/archive-preview"), routes)
|
self.assertIn((("POST",), "/files/archive-preview"), routes)
|
||||||
self.assertIn((("POST",), "/files/archive-confirm"), routes)
|
self.assertIn((("POST",), "/files/archive-confirm"), routes)
|
||||||
|
|
||||||
def test_connector_routes_keep_existing_api_paths(self) -> None:
|
def test_connector_routes_keep_existing_api_paths(self) -> None:
|
||||||
routes = {(tuple(sorted(route.methods or ())), route.path) for route in router.routes}
|
routes = {
|
||||||
|
(tuple(sorted(route.methods or ())), route.path) for route in router.routes
|
||||||
|
}
|
||||||
|
|
||||||
expected = {
|
expected = {
|
||||||
(("GET",), "/files/connectors/providers"),
|
(("GET",), "/files/connectors/providers"),
|
||||||
@@ -74,28 +83,56 @@ class FilesRouterContractTests(unittest.TestCase):
|
|||||||
(("GET",), "/files/connectors/profiles/{profile_id}/browse"),
|
(("GET",), "/files/connectors/profiles/{profile_id}/browse"),
|
||||||
(("POST",), "/files/connectors/profiles/{profile_id}/import"),
|
(("POST",), "/files/connectors/profiles/{profile_id}/import"),
|
||||||
(("POST",), "/files/connectors/profiles/{profile_id}/sync"),
|
(("POST",), "/files/connectors/profiles/{profile_id}/sync"),
|
||||||
|
(("POST",), "/files/connector-spaces/{space_id}/write-back"),
|
||||||
(("GET",), "/files/connectors/credentials"),
|
(("GET",), "/files/connectors/credentials"),
|
||||||
(("POST",), "/files/connectors/credentials"),
|
(("POST",), "/files/connectors/credentials"),
|
||||||
(("GET",), "/files/connector-spaces"),
|
(("GET",), "/files/connector-spaces"),
|
||||||
(("POST",), "/files/connector-spaces"),
|
(("POST",), "/files/connector-spaces"),
|
||||||
|
(("PATCH",), "/files/connector-spaces/{space_id}"),
|
||||||
|
(("DELETE",), "/files/connector-spaces/{space_id}"),
|
||||||
}
|
}
|
||||||
|
|
||||||
self.assertTrue(expected.issubset(routes))
|
self.assertTrue(expected.issubset(routes))
|
||||||
|
|
||||||
def test_bulk_organize_routes_keep_existing_api_paths(self) -> None:
|
def test_bulk_organize_routes_keep_existing_api_paths(self) -> None:
|
||||||
routes = {(tuple(sorted(route.methods or ())), route.path) for route in router.routes}
|
routes = {
|
||||||
|
(tuple(sorted(route.methods or ())), route.path) for route in router.routes
|
||||||
|
}
|
||||||
|
|
||||||
self.assertIn((("POST",), "/files/bulk-rename"), routes)
|
self.assertIn((("POST",), "/files/bulk-rename"), routes)
|
||||||
self.assertIn((("POST",), "/files/transfer"), routes)
|
self.assertIn((("POST",), "/files/transfer"), routes)
|
||||||
|
|
||||||
def test_share_lifecycle_routes_are_exposed(self) -> None:
|
def test_share_lifecycle_routes_are_exposed(self) -> None:
|
||||||
routes = {(tuple(sorted(route.methods or ())), route.path) for route in router.routes}
|
routes = {
|
||||||
|
(tuple(sorted(route.methods or ())), route.path) for route in router.routes
|
||||||
|
}
|
||||||
|
|
||||||
self.assertIn((("GET",), "/files/{file_id}/shares"), routes)
|
self.assertIn((("GET",), "/files/{file_id}/shares"), routes)
|
||||||
self.assertIn((("POST",), "/files/{file_id}/shares"), routes)
|
self.assertIn((("POST",), "/files/{file_id}/shares"), routes)
|
||||||
self.assertIn((("DELETE",), "/files/{file_id}/shares/{share_id}"), routes)
|
self.assertIn((("DELETE",), "/files/{file_id}/shares/{share_id}"), routes)
|
||||||
self.assertIn((("GET",), "/files/{file_id}/share-target-options"), routes)
|
self.assertIn((("GET",), "/files/{file_id}/share-target-options"), routes)
|
||||||
|
|
||||||
|
def test_exact_version_download_route_is_exposed(self) -> None:
|
||||||
|
routes = {
|
||||||
|
(tuple(sorted(route.methods or ())), route.path) for route in router.routes
|
||||||
|
}
|
||||||
|
|
||||||
|
self.assertIn(
|
||||||
|
(("GET",), "/files/{file_id}/versions/{version_id}/download"),
|
||||||
|
routes,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_governed_lifecycle_routes_are_exposed(self) -> None:
|
||||||
|
routes = {
|
||||||
|
(tuple(sorted(route.methods or ())), route.path) for route in router.routes
|
||||||
|
}
|
||||||
|
|
||||||
|
self.assertIn((("PATCH",), "/files/{file_id}/lifecycle"), routes)
|
||||||
|
self.assertIn((("POST",), "/files/assets/{file_id}/restore"), routes)
|
||||||
|
self.assertIn((("POST",), "/files/purge/preview"), routes)
|
||||||
|
self.assertIn((("POST",), "/files/purge/execute"), routes)
|
||||||
|
self.assertIn((("POST",), "/files/purge/blobs"), routes)
|
||||||
|
|
||||||
def test_file_listing_exposes_structured_property_filters(self) -> None:
|
def test_file_listing_exposes_structured_property_filters(self) -> None:
|
||||||
route = next(
|
route = next(
|
||||||
route
|
route
|
||||||
@@ -106,6 +143,7 @@ class FilesRouterContractTests(unittest.TestCase):
|
|||||||
|
|
||||||
self.assertIn("campaign_usage", parameters)
|
self.assertIn("campaign_usage", parameters)
|
||||||
self.assertIn("audit_relevant", parameters)
|
self.assertIn("audit_relevant", parameters)
|
||||||
|
self.assertIn("sort", parameters)
|
||||||
self.assertIn("total", route.response_model.model_fields)
|
self.assertIn("total", route.response_model.model_fields)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,294 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import socket
|
||||||
|
import sys
|
||||||
|
import threading
|
||||||
|
import types
|
||||||
|
import unittest
|
||||||
|
from importlib.util import find_spec
|
||||||
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from govoplan_core.security.outbound_http import (
|
||||||
|
OutboundHttpBlocked,
|
||||||
|
create_outbound_connection as core_create_outbound_connection,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.storage.sdk_peer_pinning import (
|
||||||
|
SdkPeerPinningError,
|
||||||
|
_botocore_transport_types,
|
||||||
|
create_pinned_s3_client,
|
||||||
|
install_pinned_smb_transport,
|
||||||
|
pinned_smb_connection_cache,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class _Socket:
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.connected_to: object | None = None
|
||||||
|
|
||||||
|
def settimeout(self, _value: object) -> None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def setsockopt(self, *_args: object) -> None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def connect(self, value: object) -> None:
|
||||||
|
self.connected_to = value
|
||||||
|
|
||||||
|
def close(self) -> None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
class _S3Handler(BaseHTTPRequestHandler):
|
||||||
|
def do_GET(self) -> None: # noqa: N802 - stdlib handler contract
|
||||||
|
body = (
|
||||||
|
b'<?xml version="1.0" encoding="UTF-8"?>'
|
||||||
|
b'<ListAllMyBucketsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">'
|
||||||
|
b'<Owner><ID>govoplan</ID><DisplayName>GovOPlaN</DisplayName></Owner>'
|
||||||
|
b'<Buckets><Bucket><Name>evidence</Name><CreationDate>2026-08-04T00:00:00Z</CreationDate>'
|
||||||
|
b'</Bucket></Buckets></ListAllMyBucketsResult>'
|
||||||
|
)
|
||||||
|
self.send_response(200)
|
||||||
|
self.send_header("Content-Type", "application/xml")
|
||||||
|
self.send_header("Content-Length", str(len(body)))
|
||||||
|
self.end_headers()
|
||||||
|
self.wfile.write(body)
|
||||||
|
|
||||||
|
def log_message(self, _format: str, *_args: object) -> None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
@unittest.skipUnless(find_spec("botocore") and find_spec("boto3"), "boto3 extra is not installed")
|
||||||
|
class S3PeerPinningTests(unittest.TestCase):
|
||||||
|
def test_client_is_born_with_the_pinned_http_session(self) -> None:
|
||||||
|
client = create_pinned_s3_client(
|
||||||
|
endpoint_url="http://127.0.0.1:9000",
|
||||||
|
region_name="eu-central-1",
|
||||||
|
aws_access_key_id="access",
|
||||||
|
aws_secret_access_key="secret",
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
session_type, _, _ = _botocore_transport_types()
|
||||||
|
self.assertIsInstance(client._endpoint.http_session, session_type)
|
||||||
|
self.assertEqual({}, client._endpoint.http_session._proxy_config._proxies)
|
||||||
|
finally:
|
||||||
|
client.close()
|
||||||
|
|
||||||
|
def test_each_connection_attempt_resolves_and_pins_again(self) -> None:
|
||||||
|
_, http_connection, _ = _botocore_transport_types()
|
||||||
|
connection = http_connection(host="objects.example.test", port=80)
|
||||||
|
sockets = [_Socket(), _Socket()]
|
||||||
|
with patch(
|
||||||
|
"govoplan_files.backend.storage.sdk_peer_pinning.create_outbound_connection",
|
||||||
|
side_effect=sockets,
|
||||||
|
) as connector:
|
||||||
|
self.assertIs(sockets[0], connection._new_conn())
|
||||||
|
self.assertIs(sockets[1], connection._new_conn())
|
||||||
|
|
||||||
|
self.assertEqual(2, connector.call_count)
|
||||||
|
self.assertEqual("objects.example.test", connector.call_args.args[0])
|
||||||
|
|
||||||
|
def test_real_sdk_request_uses_the_pinned_socket_path(self) -> None:
|
||||||
|
server = ThreadingHTTPServer(("127.0.0.1", 0), _S3Handler)
|
||||||
|
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||||
|
thread.start()
|
||||||
|
endpoint = f"http://127.0.0.1:{server.server_port}"
|
||||||
|
try:
|
||||||
|
with patch.dict(
|
||||||
|
"os.environ",
|
||||||
|
{"APP_ENV": "production", "GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS": "true"},
|
||||||
|
), patch(
|
||||||
|
"govoplan_files.backend.storage.sdk_peer_pinning.create_outbound_connection",
|
||||||
|
wraps=core_create_outbound_connection,
|
||||||
|
) as connector:
|
||||||
|
client = create_pinned_s3_client(
|
||||||
|
endpoint_url=endpoint,
|
||||||
|
region_name="eu-central-1",
|
||||||
|
aws_access_key_id="access",
|
||||||
|
aws_secret_access_key="secret",
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
response = client.list_buckets()
|
||||||
|
finally:
|
||||||
|
client.close()
|
||||||
|
finally:
|
||||||
|
server.shutdown()
|
||||||
|
server.server_close()
|
||||||
|
thread.join(timeout=2)
|
||||||
|
|
||||||
|
self.assertEqual("evidence", response["Buckets"][0]["Name"])
|
||||||
|
self.assertEqual("127.0.0.1", connector.call_args.args[0])
|
||||||
|
|
||||||
|
def test_every_sdk_selected_origin_uses_a_pinned_pool(self) -> None:
|
||||||
|
session_type, http_connection, https_connection = _botocore_transport_types()
|
||||||
|
session = session_type(proxies={})
|
||||||
|
try:
|
||||||
|
first = session._manager.connection_from_url("http://one.example.test/root")
|
||||||
|
redirected = session._manager.connection_from_url("https://two.example.test/root")
|
||||||
|
self.assertIs(first.ConnectionCls, http_connection)
|
||||||
|
self.assertIs(redirected.ConnectionCls, https_connection)
|
||||||
|
finally:
|
||||||
|
session.close()
|
||||||
|
|
||||||
|
def test_tls_connection_retains_the_configured_authority_for_sni(self) -> None:
|
||||||
|
_, _, https_connection = _botocore_transport_types()
|
||||||
|
connection = https_connection(host="objects.example.test", port=443)
|
||||||
|
with patch(
|
||||||
|
"govoplan_files.backend.storage.sdk_peer_pinning.create_outbound_connection",
|
||||||
|
return_value=_Socket(),
|
||||||
|
) as connector:
|
||||||
|
connection._new_conn()
|
||||||
|
|
||||||
|
self.assertEqual("objects.example.test", connection.host)
|
||||||
|
self.assertEqual("objects.example.test", connection._dns_host)
|
||||||
|
self.assertEqual("objects.example.test", connector.call_args.args[0])
|
||||||
|
|
||||||
|
def test_mixed_answer_and_peer_change_fail_closed(self) -> None:
|
||||||
|
_, http_connection, _ = _botocore_transport_types()
|
||||||
|
connection = http_connection(host="objects.example.test", port=80)
|
||||||
|
records = [
|
||||||
|
[(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("93.184.216.34", 80))],
|
||||||
|
[
|
||||||
|
(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("93.184.216.34", 80)),
|
||||||
|
(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("10.0.0.5", 80)),
|
||||||
|
],
|
||||||
|
]
|
||||||
|
with patch.dict(
|
||||||
|
"os.environ",
|
||||||
|
{"APP_ENV": "production", "GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS": "false"},
|
||||||
|
), patch(
|
||||||
|
"govoplan_core.security.outbound_http.socket.getaddrinfo",
|
||||||
|
side_effect=records,
|
||||||
|
), patch(
|
||||||
|
"govoplan_core.security.outbound_http.socket.socket",
|
||||||
|
return_value=_Socket(),
|
||||||
|
):
|
||||||
|
connection._new_conn()
|
||||||
|
with self.assertRaisesRegex(Exception, "non-public network"):
|
||||||
|
connection._new_conn()
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeTcp:
|
||||||
|
def __init__(self, server: str, port: int, timeout: float | None = None) -> None:
|
||||||
|
self.server = server
|
||||||
|
self.port = port
|
||||||
|
self.timeout = timeout
|
||||||
|
self.connected = False
|
||||||
|
self._sock = None
|
||||||
|
self._sock_lock = threading.Lock()
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeConnection:
|
||||||
|
def __init__(self, _guid: object, server: str, port: int, *, require_signing: bool) -> None:
|
||||||
|
self.server = server
|
||||||
|
self.port = port
|
||||||
|
self.require_signing = require_signing
|
||||||
|
self.transport = _FakeTcp(server, port)
|
||||||
|
self.session_table: dict[str, object] = {}
|
||||||
|
|
||||||
|
def connect(self, *, timeout: float) -> None:
|
||||||
|
self.transport.timeout = timeout
|
||||||
|
self.transport.connect()
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeSession:
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
connection: _FakeConnection,
|
||||||
|
*,
|
||||||
|
username: str | None,
|
||||||
|
password: str | None,
|
||||||
|
require_encryption: bool,
|
||||||
|
auth_protocol: str,
|
||||||
|
) -> None:
|
||||||
|
self.connection = connection
|
||||||
|
self.username = username
|
||||||
|
self.password = password
|
||||||
|
self.encrypt_data = require_encryption
|
||||||
|
self.auth_protocol = auth_protocol
|
||||||
|
self.encrypt = require_encryption
|
||||||
|
|
||||||
|
def connect(self) -> None:
|
||||||
|
self.connection.session_table[self.username or "anonymous"] = self
|
||||||
|
|
||||||
|
|
||||||
|
class SmbPeerPinningTests(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
pinned_smb_connection_cache.cache_clear()
|
||||||
|
|
||||||
|
def _modules(self) -> tuple[types.ModuleType, dict[str, types.ModuleType]]:
|
||||||
|
smbclient = types.ModuleType("smbclient")
|
||||||
|
pool = types.ModuleType("smbclient._pool")
|
||||||
|
|
||||||
|
def original_register_session(
|
||||||
|
server: str,
|
||||||
|
username: str | None = None,
|
||||||
|
password: str | None = None,
|
||||||
|
port: int = 445,
|
||||||
|
encrypt: bool | None = None,
|
||||||
|
connection_timeout: float = 60,
|
||||||
|
connection_cache: dict[str, object] | None = None,
|
||||||
|
auth_protocol: str = "negotiate",
|
||||||
|
require_signing: bool = True,
|
||||||
|
) -> None:
|
||||||
|
del server, username, password, port, encrypt, connection_timeout
|
||||||
|
del connection_cache, auth_protocol, require_signing
|
||||||
|
|
||||||
|
pool.register_session = original_register_session
|
||||||
|
pool.ClientConfig = lambda: types.SimpleNamespace(client_guid="guid")
|
||||||
|
connection = types.ModuleType("smbprotocol.connection")
|
||||||
|
connection.Connection = _FakeConnection
|
||||||
|
connection.Tcp = _FakeTcp
|
||||||
|
session = types.ModuleType("smbprotocol.session")
|
||||||
|
session.Session = _FakeSession
|
||||||
|
transport = types.ModuleType("smbprotocol.transport")
|
||||||
|
transport.Tcp = _FakeTcp
|
||||||
|
return smbclient, {
|
||||||
|
"smbclient._pool": pool,
|
||||||
|
"smbprotocol.connection": connection,
|
||||||
|
"smbprotocol.session": session,
|
||||||
|
"smbprotocol.transport": transport,
|
||||||
|
}
|
||||||
|
|
||||||
|
def test_initial_reconnect_and_referral_hosts_are_each_pinned(self) -> None:
|
||||||
|
smbclient, modules = self._modules()
|
||||||
|
sockets = [_Socket(), _Socket(), _Socket()]
|
||||||
|
with patch.dict(sys.modules, modules), patch(
|
||||||
|
"govoplan_files.backend.storage.sdk_peer_pinning.create_outbound_connection",
|
||||||
|
side_effect=sockets,
|
||||||
|
) as connector:
|
||||||
|
install_pinned_smb_transport(smbclient)
|
||||||
|
register = modules["smbclient._pool"].register_session
|
||||||
|
cache: dict[str, object] = {}
|
||||||
|
first = register("files.example.test", connection_cache=cache)
|
||||||
|
first.connection.transport.connected = False
|
||||||
|
register("files.example.test", connection_cache=cache)
|
||||||
|
register("dfs-target.example.test", connection_cache=cache)
|
||||||
|
|
||||||
|
self.assertEqual(
|
||||||
|
["files.example.test", "files.example.test", "dfs-target.example.test"],
|
||||||
|
[call.args[0] for call in connector.call_args_list],
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_referral_to_disallowed_peer_fails_before_session_creation(self) -> None:
|
||||||
|
smbclient, modules = self._modules()
|
||||||
|
with patch.dict(sys.modules, modules), patch(
|
||||||
|
"govoplan_files.backend.storage.sdk_peer_pinning.create_outbound_connection",
|
||||||
|
side_effect=OutboundHttpBlocked("non-public network"),
|
||||||
|
):
|
||||||
|
install_pinned_smb_transport(smbclient)
|
||||||
|
register = modules["smbclient._pool"].register_session
|
||||||
|
with self.assertRaisesRegex(ValueError, "non-public network"):
|
||||||
|
register("private-referral.example.test", connection_cache={})
|
||||||
|
|
||||||
|
def test_transport_tampering_after_installation_fails_closed(self) -> None:
|
||||||
|
smbclient, modules = self._modules()
|
||||||
|
with patch.dict(sys.modules, modules):
|
||||||
|
install_pinned_smb_transport(smbclient)
|
||||||
|
modules["smbprotocol.connection"].Tcp = _FakeTcp
|
||||||
|
with self.assertRaisesRegex(SdkPeerPinningError, "changed after peer pinning"):
|
||||||
|
install_pinned_smb_transport(smbclient)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,145 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from types import SimpleNamespace
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from sqlalchemy import create_engine
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from govoplan_access.backend.db.models import Account, Group, User
|
||||||
|
from govoplan_core.auth import ApiPrincipal
|
||||||
|
from govoplan_core.core.access import PrincipalRef
|
||||||
|
from govoplan_core.core.change_sequence import ChangeSequenceEntry
|
||||||
|
from govoplan_core.core.events import EventObjectRef, EventTenantRef, PlatformEvent
|
||||||
|
from govoplan_core.core.search import (
|
||||||
|
SearchAuthorizationRequest,
|
||||||
|
SearchBackfillRequest,
|
||||||
|
SearchResourceReference,
|
||||||
|
)
|
||||||
|
from govoplan_core.db.base import Base
|
||||||
|
from govoplan_files.backend.db.models import FileAsset, FileFolder, FileShare
|
||||||
|
from govoplan_files.backend.search_source import (
|
||||||
|
FilesSearchSource,
|
||||||
|
PROVIDER_ID,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class FilesSearchSourceTests(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
self.engine = create_engine("sqlite://")
|
||||||
|
Base.metadata.create_all(
|
||||||
|
self.engine,
|
||||||
|
tables=(
|
||||||
|
Account.__table__,
|
||||||
|
User.__table__,
|
||||||
|
Group.__table__,
|
||||||
|
FileAsset.__table__,
|
||||||
|
FileFolder.__table__,
|
||||||
|
FileShare.__table__,
|
||||||
|
ChangeSequenceEntry.__table__,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
self.session = Session(self.engine)
|
||||||
|
self.session.add_all(
|
||||||
|
(
|
||||||
|
Account(
|
||||||
|
id="account-1",
|
||||||
|
email="one@example.test",
|
||||||
|
normalized_email="one@example.test",
|
||||||
|
),
|
||||||
|
User(
|
||||||
|
id="user-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
account_id="account-1",
|
||||||
|
email="one@example.test",
|
||||||
|
),
|
||||||
|
FileAsset(
|
||||||
|
id="file-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
owner_type="user",
|
||||||
|
owner_user_id="user-1",
|
||||||
|
display_path="records/permit.pdf",
|
||||||
|
filename="permit.pdf",
|
||||||
|
description="Monthly permit evidence",
|
||||||
|
),
|
||||||
|
FileAsset(
|
||||||
|
id="file-other",
|
||||||
|
tenant_id="tenant-2",
|
||||||
|
owner_type="user",
|
||||||
|
display_path="other.pdf",
|
||||||
|
filename="other.pdf",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
self.session.commit()
|
||||||
|
self.source = FilesSearchSource()
|
||||||
|
|
||||||
|
def tearDown(self) -> None:
|
||||||
|
self.session.close()
|
||||||
|
self.engine.dispose()
|
||||||
|
|
||||||
|
def test_backfill_and_authorization_are_tenant_bounded(self) -> None:
|
||||||
|
page = self.source.backfill(
|
||||||
|
self.session,
|
||||||
|
request=SearchBackfillRequest(
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
provider_id=PROVIDER_ID,
|
||||||
|
resource_type="file",
|
||||||
|
rebuild_id="rebuild-1",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
self.assertEqual(("file-1",), tuple(doc.resource_id for doc in page.documents))
|
||||||
|
reference = SearchResourceReference(
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
module_id="files",
|
||||||
|
resource_type="file",
|
||||||
|
resource_id="file-1",
|
||||||
|
)
|
||||||
|
request = SearchAuthorizationRequest(reference=reference, source_revision="1")
|
||||||
|
self.assertTrue(
|
||||||
|
self.source.authorize(
|
||||||
|
self.session,
|
||||||
|
_principal({"files:file:read"}),
|
||||||
|
requests=(request,),
|
||||||
|
)[reference.key]
|
||||||
|
)
|
||||||
|
self.assertFalse(
|
||||||
|
self.source.authorize(
|
||||||
|
self.session,
|
||||||
|
_principal(set()),
|
||||||
|
requests=(request,),
|
||||||
|
)[reference.key]
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_committed_file_event_produces_authoritative_upsert(self) -> None:
|
||||||
|
event = PlatformEvent(
|
||||||
|
type="files.file.updated",
|
||||||
|
module_id="files",
|
||||||
|
tenant=EventTenantRef(id="tenant-1"),
|
||||||
|
resource=EventObjectRef(type="file", id="file-1"),
|
||||||
|
)
|
||||||
|
changes = self.source.index_changes_for_event(
|
||||||
|
self.session,
|
||||||
|
event=event,
|
||||||
|
delivery_key="delivery-1",
|
||||||
|
)
|
||||||
|
self.assertEqual(1, len(changes))
|
||||||
|
self.assertEqual("upsert", changes[0].kind)
|
||||||
|
self.assertEqual(event.event_id, changes[0].cursor)
|
||||||
|
|
||||||
|
|
||||||
|
def _principal(scopes: set[str]) -> ApiPrincipal:
|
||||||
|
return ApiPrincipal(
|
||||||
|
principal=PrincipalRef(
|
||||||
|
account_id="account-1",
|
||||||
|
membership_id="user-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
scopes=frozenset(scopes),
|
||||||
|
),
|
||||||
|
account=SimpleNamespace(id="account-1"),
|
||||||
|
user=SimpleNamespace(id="user-1"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -5,6 +5,7 @@ from datetime import timedelta
|
|||||||
from unittest.mock import patch
|
from unittest.mock import patch
|
||||||
|
|
||||||
from sqlalchemy import create_engine
|
from sqlalchemy import create_engine
|
||||||
|
from sqlalchemy.dialects import postgresql
|
||||||
from sqlalchemy.orm import sessionmaker
|
from sqlalchemy.orm import sessionmaker
|
||||||
|
|
||||||
from govoplan_access.backend.db.models import Account, Group, User
|
from govoplan_access.backend.db.models import Account, Group, User
|
||||||
@@ -13,6 +14,7 @@ from govoplan_core.db.base import Base
|
|||||||
from govoplan_files.backend.db.models import FileAsset, FileShare
|
from govoplan_files.backend.db.models import FileAsset, FileShare
|
||||||
from govoplan_files.backend.storage.common import FileStorageError, utcnow
|
from govoplan_files.backend.storage.common import FileStorageError, utcnow
|
||||||
from govoplan_files.backend.storage.files import (
|
from govoplan_files.backend.storage.files import (
|
||||||
|
_asset_visibility_query_for_user,
|
||||||
get_asset_for_user,
|
get_asset_for_user,
|
||||||
list_file_shares,
|
list_file_shares,
|
||||||
revoke_file_share,
|
revoke_file_share,
|
||||||
@@ -214,6 +216,22 @@ class FileShareLifecycleTests(unittest.TestCase):
|
|||||||
expires_at=utcnow() - timedelta(seconds=1),
|
expires_at=utcnow() - timedelta(seconds=1),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@patch(
|
||||||
|
"govoplan_files.backend.storage.files.user_group_ids",
|
||||||
|
return_value=[],
|
||||||
|
)
|
||||||
|
def test_visibility_query_is_postgresql_json_safe(self, _groups) -> None:
|
||||||
|
query = _asset_visibility_query_for_user(
|
||||||
|
self.session,
|
||||||
|
tenant_id=TENANT_ID,
|
||||||
|
user_id=RECIPIENT_ID,
|
||||||
|
)
|
||||||
|
|
||||||
|
compiled = str(query.statement.compile(dialect=postgresql.dialect()))
|
||||||
|
|
||||||
|
self.assertNotIn("SELECT DISTINCT", compiled.upper())
|
||||||
|
self.assertIn("EXISTS", compiled.upper())
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import hashlib
|
import hashlib
|
||||||
|
from datetime import UTC, datetime
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
import tempfile
|
import tempfile
|
||||||
import unittest
|
import unittest
|
||||||
@@ -15,6 +16,7 @@ from govoplan_core.core.recovery import (
|
|||||||
RecoveryOperation,
|
RecoveryOperation,
|
||||||
RecoveryStatus,
|
RecoveryStatus,
|
||||||
)
|
)
|
||||||
|
from govoplan_core.core.change_sequence import ChangeSequenceEntry
|
||||||
from govoplan_core.core.runtime_coordination import (
|
from govoplan_core.core.runtime_coordination import (
|
||||||
DistributedLease,
|
DistributedLease,
|
||||||
RuntimeIdentity,
|
RuntimeIdentity,
|
||||||
@@ -23,9 +25,13 @@ from govoplan_core.core.runtime_coordination import (
|
|||||||
from govoplan_core.db.base import Base
|
from govoplan_core.db.base import Base
|
||||||
from govoplan_core.db.session import configure_database, reset_database
|
from govoplan_core.db.session import configure_database, reset_database
|
||||||
from govoplan_files.backend.db.models import (
|
from govoplan_files.backend.db.models import (
|
||||||
|
FileAsset,
|
||||||
FileBlob,
|
FileBlob,
|
||||||
|
FileFormEvidenceGrant,
|
||||||
FileIntegrityFinding,
|
FileIntegrityFinding,
|
||||||
FileIntegrityScan,
|
FileIntegrityScan,
|
||||||
|
FileShare,
|
||||||
|
FileVersion,
|
||||||
)
|
)
|
||||||
from govoplan_files.backend.storage.backends import (
|
from govoplan_files.backend.storage.backends import (
|
||||||
LocalFilesystemStorageBackend,
|
LocalFilesystemStorageBackend,
|
||||||
@@ -34,12 +40,61 @@ from govoplan_files.backend.storage.common import FileStorageError
|
|||||||
from govoplan_files.backend.storage.files import _get_or_create_blob
|
from govoplan_files.backend.storage.files import _get_or_create_blob
|
||||||
from govoplan_files.backend.storage.integrity import cleanup_orphan_finding
|
from govoplan_files.backend.storage.integrity import cleanup_orphan_finding
|
||||||
from govoplan_files.backend.storage.recovery import begin_blob_write_recovery
|
from govoplan_files.backend.storage.recovery import begin_blob_write_recovery
|
||||||
|
from govoplan_files.backend.storage.connector_profiles import ConnectorProfile
|
||||||
|
from govoplan_files.backend.storage.connector_writes import write_connector_file
|
||||||
|
from govoplan_files.backend.storage.lifecycle import (
|
||||||
|
execute_asset_purge,
|
||||||
|
garbage_collect_unreferenced_blobs,
|
||||||
|
preview_asset_purge,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
TENANT_ID = "tenant-1"
|
TENANT_ID = "tenant-1"
|
||||||
USER_ID = "user-1"
|
USER_ID = "user-1"
|
||||||
|
|
||||||
|
|
||||||
|
class _MissingS3Object(Exception):
|
||||||
|
response = {
|
||||||
|
"Error": {"Code": "NoSuchKey"},
|
||||||
|
"ResponseMetadata": {"HTTPStatusCode": 404},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class _WriteS3Client:
|
||||||
|
def __init__(
|
||||||
|
self, *, tamper_metadata: bool = False, fail_post_write_probe: bool = False
|
||||||
|
) -> None:
|
||||||
|
self.object: dict[str, object] | None = None
|
||||||
|
self.tamper_metadata = tamper_metadata
|
||||||
|
self.fail_post_write_probe = fail_post_write_probe
|
||||||
|
self.put_request: dict[str, object] | None = None
|
||||||
|
self.closed = False
|
||||||
|
self.write_completed = False
|
||||||
|
|
||||||
|
def head_object(self, **_kwargs: object) -> dict[str, object]:
|
||||||
|
if self.write_completed and self.fail_post_write_probe:
|
||||||
|
raise RuntimeError("provider unavailable")
|
||||||
|
if self.object is None:
|
||||||
|
raise _MissingS3Object()
|
||||||
|
return dict(self.object)
|
||||||
|
|
||||||
|
def put_object(self, **kwargs: object) -> dict[str, object]:
|
||||||
|
self.put_request = dict(kwargs)
|
||||||
|
metadata = dict(kwargs.get("Metadata") or {})
|
||||||
|
if self.tamper_metadata:
|
||||||
|
metadata["govoplan-sha256"] = "0" * 64
|
||||||
|
self.object = {
|
||||||
|
"ETag": '"etag-written"',
|
||||||
|
"VersionId": "version-written",
|
||||||
|
"Metadata": metadata,
|
||||||
|
}
|
||||||
|
self.write_completed = True
|
||||||
|
return {"ETag": '"etag-written"', "VersionId": "version-written"}
|
||||||
|
|
||||||
|
def close(self) -> None:
|
||||||
|
self.closed = True
|
||||||
|
|
||||||
|
|
||||||
class StorageRecoveryTests(unittest.TestCase):
|
class StorageRecoveryTests(unittest.TestCase):
|
||||||
def setUp(self) -> None:
|
def setUp(self) -> None:
|
||||||
self.temporary_directory = tempfile.TemporaryDirectory()
|
self.temporary_directory = tempfile.TemporaryDirectory()
|
||||||
@@ -54,10 +109,15 @@ class StorageRecoveryTests(unittest.TestCase):
|
|||||||
Account.__table__,
|
Account.__table__,
|
||||||
User.__table__,
|
User.__table__,
|
||||||
Group.__table__,
|
Group.__table__,
|
||||||
|
ChangeSequenceEntry.__table__,
|
||||||
DistributedLease.__table__,
|
DistributedLease.__table__,
|
||||||
RecoveryOperation.__table__,
|
RecoveryOperation.__table__,
|
||||||
RecoveryCheckpoint.__table__,
|
RecoveryCheckpoint.__table__,
|
||||||
FileBlob.__table__,
|
FileBlob.__table__,
|
||||||
|
FileAsset.__table__,
|
||||||
|
FileVersion.__table__,
|
||||||
|
FileFormEvidenceGrant.__table__,
|
||||||
|
FileShare.__table__,
|
||||||
FileIntegrityScan.__table__,
|
FileIntegrityScan.__table__,
|
||||||
FileIntegrityFinding.__table__,
|
FileIntegrityFinding.__table__,
|
||||||
],
|
],
|
||||||
@@ -115,9 +175,10 @@ class StorageRecoveryTests(unittest.TestCase):
|
|||||||
|
|
||||||
def put_bytes(backend_self, key, data, *, content_type=None):
|
def put_bytes(backend_self, key, data, *, content_type=None):
|
||||||
with self.Session() as evidence_session:
|
with self.Session() as evidence_session:
|
||||||
operation = evidence_session.query(RecoveryOperation).one()
|
operation = evidence_session.query(RecoveryOperation).one_or_none()
|
||||||
observed_running_operation.append(
|
observed_running_operation.append(
|
||||||
operation.status == RecoveryStatus.RUNNING.value
|
operation is not None
|
||||||
|
and operation.status == RecoveryStatus.RUNNING.value
|
||||||
and len(operation.request_sha256) == 64
|
and len(operation.request_sha256) == 64
|
||||||
)
|
)
|
||||||
put_bytes(key, data, content_type=content_type)
|
put_bytes(key, data, content_type=content_type)
|
||||||
@@ -140,7 +201,14 @@ class StorageRecoveryTests(unittest.TestCase):
|
|||||||
|
|
||||||
operation = self._only_operation()
|
operation = self._only_operation()
|
||||||
self.assertEqual(RecoveryStatus.SUCCEEDED.value, operation.status)
|
self.assertEqual(RecoveryStatus.SUCCEEDED.value, operation.status)
|
||||||
self.assertEqual([True], observed_running_operation)
|
# SQLite uses an explicit caller-transaction mode because it permits
|
||||||
|
# only one writer. The intent becomes visible with the business commit;
|
||||||
|
# PostgreSQL retains the independent pre-effect commit guarantee.
|
||||||
|
self.assertEqual([False], observed_running_operation)
|
||||||
|
self.assertEqual(
|
||||||
|
"sqlite_caller_transaction",
|
||||||
|
operation.metadata_["durability_mode"],
|
||||||
|
)
|
||||||
self.assertTrue(self.backend.exists(blob.storage_key))
|
self.assertTrue(self.backend.exists(blob.storage_key))
|
||||||
self.assertNotIn("private-name", blob.storage_key)
|
self.assertNotIn("private-name", blob.storage_key)
|
||||||
self.assertEqual(".blob", Path(blob.storage_key).suffix)
|
self.assertEqual(".blob", Path(blob.storage_key).suffix)
|
||||||
@@ -164,10 +232,54 @@ class StorageRecoveryTests(unittest.TestCase):
|
|||||||
|
|
||||||
operation = self._only_operation()
|
operation = self._only_operation()
|
||||||
self.assertEqual(RecoveryStatus.RECOVERED.value, operation.status)
|
self.assertEqual(RecoveryStatus.RECOVERED.value, operation.status)
|
||||||
|
self.assertEqual(
|
||||||
|
"sqlite_post_rollback_reconstruction",
|
||||||
|
operation.metadata_["durability_mode"],
|
||||||
|
)
|
||||||
self.assertFalse(self.backend.exists(storage_key))
|
self.assertFalse(self.backend.exists(storage_key))
|
||||||
with self.Session() as evidence_session:
|
with self.Session() as evidence_session:
|
||||||
self.assertIsNone(evidence_session.get(FileBlob, blob.id))
|
self.assertIsNone(evidence_session.get(FileBlob, blob.id))
|
||||||
|
|
||||||
|
def test_multiple_blob_writes_share_one_sqlite_business_transaction(self) -> None:
|
||||||
|
with patch(
|
||||||
|
"govoplan_files.backend.storage.files.get_storage_backend",
|
||||||
|
return_value=self.backend,
|
||||||
|
):
|
||||||
|
first = _get_or_create_blob(
|
||||||
|
self.session,
|
||||||
|
tenant_id=TENANT_ID,
|
||||||
|
data=b"first archive member",
|
||||||
|
filename="first.txt",
|
||||||
|
content_type="text/plain",
|
||||||
|
actor_id=USER_ID,
|
||||||
|
)
|
||||||
|
second = _get_or_create_blob(
|
||||||
|
self.session,
|
||||||
|
tenant_id=TENANT_ID,
|
||||||
|
data=b"second archive member",
|
||||||
|
filename="second.txt",
|
||||||
|
content_type="text/plain",
|
||||||
|
actor_id=USER_ID,
|
||||||
|
)
|
||||||
|
self.session.commit()
|
||||||
|
|
||||||
|
self.assertTrue(self.backend.exists(first.storage_key))
|
||||||
|
self.assertTrue(self.backend.exists(second.storage_key))
|
||||||
|
with self.Session() as evidence_session:
|
||||||
|
operations = evidence_session.query(RecoveryOperation).all()
|
||||||
|
self.assertEqual(2, len(operations))
|
||||||
|
self.assertEqual(
|
||||||
|
{RecoveryStatus.SUCCEEDED.value},
|
||||||
|
{operation.status for operation in operations},
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
{"sqlite_caller_transaction"},
|
||||||
|
{
|
||||||
|
operation.metadata_["durability_mode"]
|
||||||
|
for operation in operations
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
def test_post_write_tamper_is_quarantined_and_recovery_required(self) -> None:
|
def test_post_write_tamper_is_quarantined_and_recovery_required(self) -> None:
|
||||||
with patch(
|
with patch(
|
||||||
"govoplan_files.backend.storage.files.get_storage_backend",
|
"govoplan_files.backend.storage.files.get_storage_backend",
|
||||||
@@ -314,6 +426,214 @@ class StorageRecoveryTests(unittest.TestCase):
|
|||||||
self.session.rollback()
|
self.session.rollback()
|
||||||
self.assertEqual(RecoveryStatus.REJECTED.value, self._only_operation().status)
|
self.assertEqual(RecoveryStatus.REJECTED.value, self._only_operation().status)
|
||||||
|
|
||||||
|
def test_purge_is_idempotent_and_gc_deletes_only_after_reference_check(self) -> None:
|
||||||
|
key = f"tenants/{TENANT_ID}/files/purge-me.blob"
|
||||||
|
self.backend.put_bytes(key, b"purge-me")
|
||||||
|
blob = FileBlob(
|
||||||
|
id="blob-purge",
|
||||||
|
tenant_id=TENANT_ID,
|
||||||
|
storage_backend=self.backend.name,
|
||||||
|
storage_key=key,
|
||||||
|
checksum_sha256=hashlib.sha256(b"purge-me").hexdigest(),
|
||||||
|
size_bytes=8,
|
||||||
|
ref_count=1,
|
||||||
|
)
|
||||||
|
asset = FileAsset(
|
||||||
|
id="asset-purge",
|
||||||
|
tenant_id=TENANT_ID,
|
||||||
|
owner_type="user",
|
||||||
|
owner_user_id=USER_ID,
|
||||||
|
current_version_id="version-purge",
|
||||||
|
display_path="purge-me.txt",
|
||||||
|
filename="purge-me.txt",
|
||||||
|
deleted_at=datetime.now(UTC),
|
||||||
|
)
|
||||||
|
version = FileVersion(
|
||||||
|
id="version-purge",
|
||||||
|
tenant_id=TENANT_ID,
|
||||||
|
file_asset_id=asset.id,
|
||||||
|
blob_id=blob.id,
|
||||||
|
version_number=1,
|
||||||
|
filename_at_upload=asset.filename,
|
||||||
|
display_path_at_upload=asset.display_path,
|
||||||
|
size_bytes=blob.size_bytes,
|
||||||
|
checksum_sha256=blob.checksum_sha256,
|
||||||
|
)
|
||||||
|
self.session.add_all([blob, asset, version])
|
||||||
|
self.session.commit()
|
||||||
|
preview = preview_asset_purge(
|
||||||
|
self.session, tenant_id=TENANT_ID, file_ids=[asset.id]
|
||||||
|
)
|
||||||
|
|
||||||
|
result = execute_asset_purge(
|
||||||
|
self.session,
|
||||||
|
tenant_id=TENANT_ID,
|
||||||
|
file_ids=[asset.id],
|
||||||
|
preview_sha256=preview.preview_sha256,
|
||||||
|
idempotency_key="purge-request-1",
|
||||||
|
approval_reference="approval-1",
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(1, result.purged_files)
|
||||||
|
self.assertEqual(1, result.released_blobs)
|
||||||
|
self.assertIsNone(self.session.get(FileAsset, asset.id))
|
||||||
|
retained_blob = self.session.get(FileBlob, blob.id)
|
||||||
|
self.assertIsNotNone(retained_blob)
|
||||||
|
self.assertEqual(0, retained_blob.ref_count)
|
||||||
|
self.assertTrue(self.backend.exists(key))
|
||||||
|
|
||||||
|
replay = execute_asset_purge(
|
||||||
|
self.session,
|
||||||
|
tenant_id=TENANT_ID,
|
||||||
|
file_ids=[asset.id],
|
||||||
|
preview_sha256=preview.preview_sha256,
|
||||||
|
idempotency_key="purge-request-1",
|
||||||
|
approval_reference="approval-1",
|
||||||
|
)
|
||||||
|
self.assertTrue(replay.replayed)
|
||||||
|
|
||||||
|
with patch(
|
||||||
|
"govoplan_files.backend.storage.lifecycle.get_storage_backend",
|
||||||
|
return_value=self.backend,
|
||||||
|
):
|
||||||
|
gc_result = garbage_collect_unreferenced_blobs(
|
||||||
|
self.session,
|
||||||
|
tenant_id=TENANT_ID,
|
||||||
|
limit=10,
|
||||||
|
approval_reference="approval-gc-1",
|
||||||
|
)
|
||||||
|
self.assertEqual(1, gc_result.deleted_blobs)
|
||||||
|
self.assertFalse(self.backend.exists(key))
|
||||||
|
self.assertIsNone(self.session.get(FileBlob, blob.id))
|
||||||
|
|
||||||
|
with self.Session() as evidence_session:
|
||||||
|
operations = evidence_session.query(RecoveryOperation).all()
|
||||||
|
self.assertEqual(2, len(operations))
|
||||||
|
self.assertEqual(
|
||||||
|
{RecoveryStatus.SUCCEEDED.value},
|
||||||
|
{operation.status for operation in operations},
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_s3_write_is_conditional_verified_and_idempotent(self) -> None:
|
||||||
|
client = _WriteS3Client()
|
||||||
|
profile = ConnectorProfile(
|
||||||
|
id="s3-write",
|
||||||
|
label="S3 write",
|
||||||
|
provider="s3",
|
||||||
|
base_path="root",
|
||||||
|
capabilities=("browse", "write"),
|
||||||
|
metadata={"bucket": "files"},
|
||||||
|
)
|
||||||
|
with patch(
|
||||||
|
"govoplan_files.backend.storage.connector_writes._s3_client",
|
||||||
|
return_value=client,
|
||||||
|
):
|
||||||
|
result = write_connector_file(
|
||||||
|
profile,
|
||||||
|
tenant_id=TENANT_ID,
|
||||||
|
library_id=None,
|
||||||
|
remote_path="out/report.txt",
|
||||||
|
data=b"report",
|
||||||
|
content_type="text/plain",
|
||||||
|
expected_revision=None,
|
||||||
|
idempotency_key="connector-write-1",
|
||||||
|
)
|
||||||
|
replay = write_connector_file(
|
||||||
|
profile,
|
||||||
|
tenant_id=TENANT_ID,
|
||||||
|
library_id=None,
|
||||||
|
remote_path="out/report.txt",
|
||||||
|
data=b"report",
|
||||||
|
content_type="text/plain",
|
||||||
|
expected_revision=None,
|
||||||
|
idempotency_key="connector-write-1",
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(RecoveryStatus.SUCCEEDED.value, result.status)
|
||||||
|
self.assertTrue(replay.replayed)
|
||||||
|
self.assertEqual("*", client.put_request["IfNoneMatch"])
|
||||||
|
self.assertEqual("root/out/report.txt", client.put_request["Key"])
|
||||||
|
self.assertNotIn(b"report", repr(self._only_operation().metadata_).encode())
|
||||||
|
self.assertTrue(client.closed)
|
||||||
|
|
||||||
|
def test_s3_write_tamper_is_visible_and_blocks_another_writer(self) -> None:
|
||||||
|
client = _WriteS3Client(tamper_metadata=True)
|
||||||
|
profile = ConnectorProfile(
|
||||||
|
id="s3-write",
|
||||||
|
label="S3 write",
|
||||||
|
provider="s3",
|
||||||
|
capabilities=("write",),
|
||||||
|
metadata={"bucket": "files"},
|
||||||
|
)
|
||||||
|
with patch(
|
||||||
|
"govoplan_files.backend.storage.connector_writes._s3_client",
|
||||||
|
return_value=client,
|
||||||
|
):
|
||||||
|
result = write_connector_file(
|
||||||
|
profile,
|
||||||
|
tenant_id=TENANT_ID,
|
||||||
|
library_id=None,
|
||||||
|
remote_path="report.txt",
|
||||||
|
data=b"report",
|
||||||
|
content_type="text/plain",
|
||||||
|
expected_revision=None,
|
||||||
|
idempotency_key="connector-write-tampered",
|
||||||
|
)
|
||||||
|
with self.assertRaisesRegex(FileStorageError, "unresolved recovery"):
|
||||||
|
write_connector_file(
|
||||||
|
profile,
|
||||||
|
tenant_id=TENANT_ID,
|
||||||
|
library_id=None,
|
||||||
|
remote_path="report.txt",
|
||||||
|
data=b"replacement",
|
||||||
|
content_type="text/plain",
|
||||||
|
expected_revision="version-written",
|
||||||
|
idempotency_key="connector-write-after-tamper",
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(RecoveryStatus.RECOVERY_REQUIRED.value, result.status)
|
||||||
|
self.assertEqual(
|
||||||
|
RecoveryStatus.RECOVERY_REQUIRED.value, self._only_operation().status
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_s3_write_probe_failure_is_outcome_unknown_not_confirmed_absent(self) -> None:
|
||||||
|
client = _WriteS3Client(fail_post_write_probe=True)
|
||||||
|
profile = ConnectorProfile(
|
||||||
|
id="s3-write",
|
||||||
|
label="S3 write",
|
||||||
|
provider="s3",
|
||||||
|
capabilities=("write",),
|
||||||
|
metadata={"bucket": "files"},
|
||||||
|
)
|
||||||
|
with patch(
|
||||||
|
"govoplan_files.backend.storage.connector_writes._s3_client",
|
||||||
|
return_value=client,
|
||||||
|
):
|
||||||
|
result = write_connector_file(
|
||||||
|
profile,
|
||||||
|
tenant_id=TENANT_ID,
|
||||||
|
library_id=None,
|
||||||
|
remote_path="report.txt",
|
||||||
|
data=b"report",
|
||||||
|
content_type="text/plain",
|
||||||
|
expected_revision=None,
|
||||||
|
idempotency_key="connector-write-probe-failed",
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(RecoveryStatus.OUTCOME_UNKNOWN.value, result.status)
|
||||||
|
operation = self._only_operation()
|
||||||
|
self.assertEqual(RecoveryStatus.OUTCOME_UNKNOWN.value, operation.status)
|
||||||
|
with self.Session() as session:
|
||||||
|
checkpoint = (
|
||||||
|
session.query(RecoveryCheckpoint)
|
||||||
|
.filter(RecoveryCheckpoint.operation_id == operation.id)
|
||||||
|
.order_by(RecoveryCheckpoint.sequence.desc())
|
||||||
|
.first()
|
||||||
|
)
|
||||||
|
self.assertIsNotNone(checkpoint)
|
||||||
|
self.assertFalse(checkpoint.evidence["observed"]["probe_verified"])
|
||||||
|
self.assertFalse(checkpoint.evidence["observed"]["present"])
|
||||||
|
|
||||||
def _only_operation(self) -> RecoveryOperation:
|
def _only_operation(self) -> RecoveryOperation:
|
||||||
with self.Session() as session:
|
with self.Session() as session:
|
||||||
operations = session.query(RecoveryOperation).all()
|
operations = session.query(RecoveryOperation).all()
|
||||||
|
|||||||
@@ -0,0 +1,175 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import unittest
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from types import SimpleNamespace
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from sqlalchemy import create_engine
|
||||||
|
from sqlalchemy.orm import sessionmaker
|
||||||
|
|
||||||
|
from govoplan_core.auth import ApiPrincipal
|
||||||
|
from govoplan_core.core.access import PrincipalRef
|
||||||
|
from govoplan_core.core.files import (
|
||||||
|
ManagedTabularFileAccessError,
|
||||||
|
ManagedTabularFileValidationError,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.capabilities import FilesManagedTabularFileService
|
||||||
|
from govoplan_files.backend.storage.common import FileStorageError
|
||||||
|
|
||||||
|
|
||||||
|
def principal(
|
||||||
|
tenant_id: str = "tenant-1",
|
||||||
|
*,
|
||||||
|
scopes: tuple[str, ...] = (
|
||||||
|
"files:file:read",
|
||||||
|
"files:file:download",
|
||||||
|
"files:file:admin",
|
||||||
|
),
|
||||||
|
) -> ApiPrincipal:
|
||||||
|
return ApiPrincipal(
|
||||||
|
principal=PrincipalRef(
|
||||||
|
account_id="account-1",
|
||||||
|
membership_id="membership-1",
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
scopes=frozenset(scopes),
|
||||||
|
),
|
||||||
|
account=object(),
|
||||||
|
user=SimpleNamespace(id="user-1"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class FilesManagedTabularContentTests(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
self.engine = create_engine("sqlite+pysqlite:///:memory:")
|
||||||
|
self.session = sessionmaker(bind=self.engine)()
|
||||||
|
self.blob = SimpleNamespace(
|
||||||
|
id="blob-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
storage_backend="local",
|
||||||
|
storage_key="tenants/tenant-1/files/blob-1",
|
||||||
|
checksum_sha256="a" * 64,
|
||||||
|
size_bytes=8,
|
||||||
|
content_type="text/csv",
|
||||||
|
integrity_status="verified",
|
||||||
|
)
|
||||||
|
self.asset = SimpleNamespace(
|
||||||
|
id="asset-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
owner_type="user",
|
||||||
|
owner_user_id="user-1",
|
||||||
|
current_version_id="version-1",
|
||||||
|
display_path="Imports/source.csv",
|
||||||
|
filename="source.csv",
|
||||||
|
description=None,
|
||||||
|
updated_at=datetime(2026, 8, 21, tzinfo=UTC),
|
||||||
|
)
|
||||||
|
self.version = SimpleNamespace(
|
||||||
|
id="version-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
file_asset_id="asset-1",
|
||||||
|
blob_id="blob-1",
|
||||||
|
version_number=1,
|
||||||
|
filename_at_upload="source.csv",
|
||||||
|
display_path_at_upload="Imports/source.csv",
|
||||||
|
content_type="text/csv",
|
||||||
|
size_bytes=8,
|
||||||
|
checksum_sha256="a" * 64,
|
||||||
|
created_by_user_id="user-1",
|
||||||
|
created_at=datetime(2026, 8, 21, tzinfo=UTC),
|
||||||
|
)
|
||||||
|
self.provider = FilesManagedTabularFileService()
|
||||||
|
|
||||||
|
def tearDown(self) -> None:
|
||||||
|
self.session.close()
|
||||||
|
self.engine.dispose()
|
||||||
|
|
||||||
|
def test_lists_and_reads_only_exact_authorized_tabular_versions(self) -> None:
|
||||||
|
with (
|
||||||
|
patch(
|
||||||
|
"govoplan_files.backend.capabilities.list_recent_assets_for_user",
|
||||||
|
return_value=[self.asset],
|
||||||
|
),
|
||||||
|
patch(
|
||||||
|
"govoplan_files.backend.capabilities.current_version_and_blob",
|
||||||
|
return_value=(self.version, self.blob),
|
||||||
|
),
|
||||||
|
patch(
|
||||||
|
"govoplan_files.backend.capabilities.get_asset_for_user",
|
||||||
|
return_value=self.asset,
|
||||||
|
),
|
||||||
|
patch.object(
|
||||||
|
self.session,
|
||||||
|
"get",
|
||||||
|
side_effect=lambda model, object_id: {
|
||||||
|
"version-1": self.version,
|
||||||
|
"blob-1": self.blob,
|
||||||
|
}.get(object_id),
|
||||||
|
),
|
||||||
|
patch(
|
||||||
|
"govoplan_files.backend.capabilities.read_asset_version_bytes",
|
||||||
|
return_value=(b"id\n1\n", self.version, self.blob),
|
||||||
|
) as read_bytes,
|
||||||
|
patch("govoplan_files.backend.capabilities.audit_from_principal"),
|
||||||
|
):
|
||||||
|
listed = self.provider.list_tabular_files(self.session, principal())
|
||||||
|
result = self.provider.read_tabular_file(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
file_asset_id="asset-1",
|
||||||
|
file_version_id="version-1",
|
||||||
|
max_bytes=100,
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(("version-1",), tuple(item.file_version_id for item in listed))
|
||||||
|
self.assertEqual(b"id\n1\n", result.payload)
|
||||||
|
self.assertTrue(result.file.current_version)
|
||||||
|
read_bytes.assert_called_once()
|
||||||
|
|
||||||
|
def test_tenant_scope_permissions_and_pre_read_size_limit_fail_closed(self) -> None:
|
||||||
|
with patch(
|
||||||
|
"govoplan_files.backend.capabilities.get_asset_for_user",
|
||||||
|
side_effect=FileStorageError("File not found"),
|
||||||
|
):
|
||||||
|
self.assertIsNone(
|
||||||
|
self.provider.get_tabular_file(
|
||||||
|
self.session,
|
||||||
|
principal("tenant-2"),
|
||||||
|
file_asset_id="asset-1",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
with self.assertRaises(ManagedTabularFileAccessError):
|
||||||
|
self.provider.list_tabular_files(
|
||||||
|
self.session,
|
||||||
|
principal(scopes=()),
|
||||||
|
)
|
||||||
|
with (
|
||||||
|
patch(
|
||||||
|
"govoplan_files.backend.capabilities.get_asset_for_user",
|
||||||
|
return_value=self.asset,
|
||||||
|
),
|
||||||
|
patch.object(
|
||||||
|
self.session,
|
||||||
|
"get",
|
||||||
|
side_effect=lambda model, object_id: {
|
||||||
|
"version-1": self.version,
|
||||||
|
"blob-1": self.blob,
|
||||||
|
}.get(object_id),
|
||||||
|
),
|
||||||
|
patch(
|
||||||
|
"govoplan_files.backend.capabilities.read_asset_version_bytes"
|
||||||
|
) as read_bytes,
|
||||||
|
self.assertRaises(ManagedTabularFileValidationError),
|
||||||
|
):
|
||||||
|
self.provider.read_tabular_file(
|
||||||
|
self.session,
|
||||||
|
principal(),
|
||||||
|
file_asset_id="asset-1",
|
||||||
|
file_version_id="version-1",
|
||||||
|
max_bytes=4,
|
||||||
|
)
|
||||||
|
read_bytes.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -1,5 +1,6 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import UTC, datetime
|
||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
from sqlalchemy import create_engine, event
|
from sqlalchemy import create_engine, event
|
||||||
@@ -14,6 +15,7 @@ from govoplan_files.backend.db.models import (
|
|||||||
FileConnectorPolicy,
|
FileConnectorPolicy,
|
||||||
FileConnectorProfile,
|
FileConnectorProfile,
|
||||||
FileConnectorSpace,
|
FileConnectorSpace,
|
||||||
|
FileFormEvidenceGrant,
|
||||||
)
|
)
|
||||||
from govoplan_files.backend.manifest import _tenant_summary_batch
|
from govoplan_files.backend.manifest import _tenant_summary_batch
|
||||||
|
|
||||||
@@ -32,6 +34,7 @@ class FilesTenantSummaryBatchTests(unittest.TestCase):
|
|||||||
FileConnectorPolicy.__table__,
|
FileConnectorPolicy.__table__,
|
||||||
FileConnectorProfile.__table__,
|
FileConnectorProfile.__table__,
|
||||||
FileConnectorSpace.__table__,
|
FileConnectorSpace.__table__,
|
||||||
|
FileFormEvidenceGrant.__table__,
|
||||||
ChangeSequenceEntry.__table__,
|
ChangeSequenceEntry.__table__,
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
@@ -70,6 +73,21 @@ class FilesTenantSummaryBatchTests(unittest.TestCase):
|
|||||||
connector_profile_id="profile-1",
|
connector_profile_id="profile-1",
|
||||||
provider="webdav",
|
provider="webdav",
|
||||||
),
|
),
|
||||||
|
FileFormEvidenceGrant(
|
||||||
|
id="grant-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
form_instance_id="form-instance-1",
|
||||||
|
form_definition_id="form-1",
|
||||||
|
form_definition_revision="1",
|
||||||
|
token_sha256="a" * 64,
|
||||||
|
idempotency_key="grant-1",
|
||||||
|
request_sha256="b" * 64,
|
||||||
|
custodian_user_id="user-1",
|
||||||
|
evidence_kind="document",
|
||||||
|
purpose="submission attachment",
|
||||||
|
expires_at=datetime.now(UTC),
|
||||||
|
max_size_bytes=1024,
|
||||||
|
),
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
session.commit()
|
session.commit()
|
||||||
@@ -88,7 +106,7 @@ class FilesTenantSummaryBatchTests(unittest.TestCase):
|
|||||||
finally:
|
finally:
|
||||||
event.remove(engine, "before_cursor_execute", count_query)
|
event.remove(engine, "before_cursor_execute", count_query)
|
||||||
|
|
||||||
self.assertEqual(5, query_count)
|
self.assertEqual(6, query_count)
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
{
|
{
|
||||||
"files": 1,
|
"files": 1,
|
||||||
@@ -96,6 +114,7 @@ class FilesTenantSummaryBatchTests(unittest.TestCase):
|
|||||||
"connector_policies": 1,
|
"connector_policies": 1,
|
||||||
"connector_profiles": 1,
|
"connector_profiles": 1,
|
||||||
"connector_spaces": 1,
|
"connector_spaces": 1,
|
||||||
|
"form_evidence_upload_grants": 1,
|
||||||
},
|
},
|
||||||
counts["tenant-1"],
|
counts["tenant-1"],
|
||||||
)
|
)
|
||||||
|
|||||||
+3
-2
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@govoplan/files-webui",
|
"name": "@govoplan/files-webui",
|
||||||
"version": "0.1.9",
|
"version": "0.1.19",
|
||||||
"private": true,
|
"private": true,
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"main": "src/index.ts",
|
"main": "src/index.ts",
|
||||||
@@ -16,6 +16,7 @@
|
|||||||
"scripts": {
|
"scripts": {
|
||||||
"test:file-drop-target": "node scripts/test-file-drop-target-structure.mjs",
|
"test:file-drop-target": "node scripts/test-file-drop-target-structure.mjs",
|
||||||
"test:file-property-filters": "node scripts/test-file-property-filters-structure.mjs",
|
"test:file-property-filters": "node scripts/test-file-property-filters-structure.mjs",
|
||||||
|
"test:connector-space-removal": "node scripts/test-connector-space-removal-structure.mjs",
|
||||||
"test:interface-pattern-language": "node scripts/test-interface-pattern-language.mjs"
|
"test:interface-pattern-language": "node scripts/test-interface-pattern-language.mjs"
|
||||||
},
|
},
|
||||||
"peerDependencies": {
|
"peerDependencies": {
|
||||||
@@ -26,7 +27,7 @@
|
|||||||
"react-dom": ">=19.2.7 <20",
|
"react-dom": ">=19.2.7 <20",
|
||||||
"react-router": ">=8.3.0 <9",
|
"react-router": ">=8.3.0 <9",
|
||||||
"lucide-react": "^1.23.0",
|
"lucide-react": "^1.23.0",
|
||||||
"@govoplan/core-webui": "^0.1.9"
|
"@govoplan/core-webui": "^0.1.18"
|
||||||
},
|
},
|
||||||
"peerDependenciesMeta": {
|
"peerDependenciesMeta": {
|
||||||
"@govoplan/core-webui": {
|
"@govoplan/core-webui": {
|
||||||
|
|||||||
@@ -0,0 +1,12 @@
|
|||||||
|
import assert from "node:assert/strict";
|
||||||
|
import { readFileSync } from "node:fs";
|
||||||
|
|
||||||
|
const source = readFileSync(new URL("../src/features/files/FilesPage.tsx", import.meta.url), "utf8");
|
||||||
|
|
||||||
|
assert.match(source, /deleteFileConnectorSpace\(settings, target\.connector_space_id\)/);
|
||||||
|
assert.match(source, /activeSpaceIsConnector &&[\s\S]*setConnectorSpaceRemovalTarget\(activeSpace\)/);
|
||||||
|
assert.match(source, /Remote files and folders remain at the provider/);
|
||||||
|
assert.match(source, /Imported GovOPlaN files, metadata and shares remain managed/);
|
||||||
|
assert.match(source, /connector profile, credentials and remote references are not deleted/);
|
||||||
|
|
||||||
|
console.log("Connector-space removal structure checks passed.");
|
||||||
@@ -7,6 +7,8 @@ function read(relativePath) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const connector = read("../src/features/files/FileConnectorSettingsPanel.tsx");
|
const connector = read("../src/features/files/FileConnectorSettingsPanel.tsx");
|
||||||
|
const integrity = read("../src/features/files/FileIntegrityPanel.tsx");
|
||||||
|
const filesApi = read("../src/api/files.ts");
|
||||||
const filesPage = read("../src/features/files/FilesPage.tsx");
|
const filesPage = read("../src/features/files/FilesPage.tsx");
|
||||||
const moduleSource = read("../src/module.ts");
|
const moduleSource = read("../src/module.ts");
|
||||||
const styles = read("../src/styles/file-manager.css");
|
const styles = read("../src/styles/file-manager.css");
|
||||||
@@ -23,6 +25,14 @@ assert.match(connector, /<AdvancedOptionsPanel title="Connection metadata JSON"/
|
|||||||
assert.match(connector, /<AdvancedOptionsPanel title="Credential metadata JSON"/);
|
assert.match(connector, /<AdvancedOptionsPanel title="Credential metadata JSON"/);
|
||||||
assert.doesNotMatch(connector, /window\.(?:alert|confirm)\(/);
|
assert.doesNotMatch(connector, /window\.(?:alert|confirm)\(/);
|
||||||
|
|
||||||
|
assert.match(integrity, /File integrity/);
|
||||||
|
assert.match(filesApi, /expected_revision/);
|
||||||
|
assert.match(integrity, /cleanupFileIntegrityFinding\(settings, finding, true\)/);
|
||||||
|
assert.match(integrity, /<ConfirmDialog[\s\S]*Delete unreferenced storage object/);
|
||||||
|
assert.match(integrity, /files\.reference\.integrity-recovery-and-fail-closed-transports/);
|
||||||
|
assert.match(moduleSource, /files\.admin\.tenant-integrity/);
|
||||||
|
assert.doesNotMatch(integrity, /window\.(?:alert|confirm)\(/);
|
||||||
|
|
||||||
assert.match(filesPage, /DocumentationHelpLink/);
|
assert.match(filesPage, /DocumentationHelpLink/);
|
||||||
assert.match(filesPage, /topicId: "files\.workflow\.organize-managed-files"/);
|
assert.match(filesPage, /topicId: "files\.workflow\.organize-managed-files"/);
|
||||||
assert.match(filesPage, /disabledReason=\{uploadBlocker\}/);
|
assert.match(filesPage, /disabledReason=\{uploadBlocker\}/);
|
||||||
@@ -31,10 +41,10 @@ assert.match(filesPage, /<ConfirmDialog[\s\S]*tone="danger"/);
|
|||||||
assert.match(filesPage, /className="workspace-data-page module-entry-page file-manager-page file-manager-fullscreen files-page"/);
|
assert.match(filesPage, /className="workspace-data-page module-entry-page file-manager-page file-manager-fullscreen files-page"/);
|
||||||
assert.doesNotMatch(filesPage, /window\.(?:alert|confirm)\(/);
|
assert.doesNotMatch(filesPage, /window\.(?:alert|confirm)\(/);
|
||||||
|
|
||||||
assert.doesNotMatch(`${connector}\n${filesPage}\n${moduleSource}`, /@govoplan\/(?:campaign|mail|docs)-webui|govoplan_(?:campaign|mail|docs)/);
|
assert.doesNotMatch(`${connector}\n${integrity}\n${filesPage}\n${moduleSource}`, /@govoplan\/(?:campaign|mail|docs)-webui|govoplan_(?:campaign|mail|docs)/);
|
||||||
assert.match(moduleSource, /"files\.connectors"/);
|
assert.match(moduleSource, /"files\.connectors"/);
|
||||||
assert.match(moduleSource, /"files\.fileExplorer"/);
|
assert.match(moduleSource, /"files\.fileExplorer"/);
|
||||||
assert.match(styles, /@media \(max-width: 1050px\)[\s\S]*\.files-page \.file-manager-shell[\s\S]*grid-template-columns: 1fr/);
|
assert.match(styles, /@media \(max-width: 1100px\)[\s\S]*\.files-page \.file-manager-shell[\s\S]*grid-template-columns: 1fr/);
|
||||||
assert.match(styles, /@media \(max-width: 760px\)[\s\S]*\.file-connector-profile-row[\s\S]*grid-template-columns: 1fr/);
|
assert.match(styles, /@media \(max-width: 760px\)[\s\S]*\.file-connector-profile-row[\s\S]*grid-template-columns: 1fr/);
|
||||||
|
|
||||||
for (const archetype of ["Directory/explorer", "Adaptive create/edit", "Effective-policy editor", "Dashboard widget"]) {
|
for (const archetype of ["Directory/explorer", "Adaptive create/edit", "Effective-policy editor", "Dashboard widget"]) {
|
||||||
|
|||||||
+271
-3
@@ -10,11 +10,15 @@ import {
|
|||||||
type FilesManagedFileLinkTarget,
|
type FilesManagedFileLinkTarget,
|
||||||
type ReferenceOptionProvider
|
type ReferenceOptionProvider
|
||||||
} from "@govoplan/core-webui";
|
} from "@govoplan/core-webui";
|
||||||
export { fetchResourceAccessExplanation } from "@govoplan/core-webui";
|
export {
|
||||||
|
fetchResourceAccessExplanation,
|
||||||
|
fetchResourceAccessExplanationSubjects
|
||||||
|
} from "@govoplan/core-webui";
|
||||||
export type {
|
export type {
|
||||||
AccessDecisionProvenanceItem,
|
AccessDecisionProvenanceItem,
|
||||||
ResourceAccessExplanationUser as AccessExplanationUser,
|
ResourceAccessExplanationUser as AccessExplanationUser,
|
||||||
ResourceAccessExplanationResponse
|
ResourceAccessExplanationResponse,
|
||||||
|
ResourceAccessExplanationSubjectsResponse
|
||||||
} from "@govoplan/core-webui";
|
} from "@govoplan/core-webui";
|
||||||
|
|
||||||
export type FileSpace = {
|
export type FileSpace = {
|
||||||
@@ -207,6 +211,57 @@ export type FileConnectorCredentialUpdatePayload = Partial<Omit<FileConnectorCre
|
|||||||
clear_token?: boolean;
|
clear_token?: boolean;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export type FileIntegrityScan = {
|
||||||
|
id: string;
|
||||||
|
tenant_id: string;
|
||||||
|
storage_backend: string;
|
||||||
|
storage_prefix: string;
|
||||||
|
status: string;
|
||||||
|
revision: number;
|
||||||
|
phase: string;
|
||||||
|
verify_checksums: boolean;
|
||||||
|
batch_size: number;
|
||||||
|
scanned_blob_count: number;
|
||||||
|
verified_blob_count: number;
|
||||||
|
quarantined_blob_count: number;
|
||||||
|
scanned_object_count: number;
|
||||||
|
orphan_object_count: number;
|
||||||
|
created_by_user_id?: string | null;
|
||||||
|
started_at?: string | null;
|
||||||
|
completed_at?: string | null;
|
||||||
|
last_error?: string | null;
|
||||||
|
created_at: string;
|
||||||
|
updated_at: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type FileIntegrityFinding = {
|
||||||
|
id: string;
|
||||||
|
scan_id: string;
|
||||||
|
tenant_id: string;
|
||||||
|
kind: string;
|
||||||
|
state: string;
|
||||||
|
revision: number;
|
||||||
|
blob_id?: string | null;
|
||||||
|
storage_key: string;
|
||||||
|
expected_size_bytes?: number | null;
|
||||||
|
observed_size_bytes?: number | null;
|
||||||
|
expected_checksum_sha256?: string | null;
|
||||||
|
observed_checksum_sha256?: string | null;
|
||||||
|
resolved_at?: string | null;
|
||||||
|
resolved_by_user_id?: string | null;
|
||||||
|
created_at: string;
|
||||||
|
updated_at: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type FileIntegrityActionResult = {
|
||||||
|
action: string;
|
||||||
|
changed: boolean;
|
||||||
|
dry_run: boolean;
|
||||||
|
finding: FileIntegrityFinding;
|
||||||
|
inspection_kind?: string | null;
|
||||||
|
inspection_valid?: boolean | null;
|
||||||
|
};
|
||||||
|
|
||||||
export type FileConnectorBrowseItem = {
|
export type FileConnectorBrowseItem = {
|
||||||
kind: "library" | "folder" | "file";
|
kind: "library" | "folder" | "file";
|
||||||
name: string;
|
name: string;
|
||||||
@@ -240,6 +295,7 @@ export type FileConnectorProvider = {
|
|||||||
installed: boolean;
|
installed: boolean;
|
||||||
browse_supported: boolean;
|
browse_supported: boolean;
|
||||||
import_supported: boolean;
|
import_supported: boolean;
|
||||||
|
write_supported: boolean;
|
||||||
optional_dependency?: string | null;
|
optional_dependency?: string | null;
|
||||||
permission_model: string;
|
permission_model: string;
|
||||||
sync_strategy: string;
|
sync_strategy: string;
|
||||||
@@ -288,6 +344,7 @@ export type FileConnectorSpacePayload = {
|
|||||||
library_id?: string | null;
|
library_id?: string | null;
|
||||||
remote_path?: string;
|
remote_path?: string;
|
||||||
sync_mode?: "manual";
|
sync_mode?: "manual";
|
||||||
|
read_only?: boolean;
|
||||||
metadata?: Record<string, unknown>;
|
metadata?: Record<string, unknown>;
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -306,6 +363,10 @@ export type ManagedFile = {
|
|||||||
created_at: string;
|
created_at: string;
|
||||||
updated_at: string;
|
updated_at: string;
|
||||||
deleted_at?: string | null;
|
deleted_at?: string | null;
|
||||||
|
retained_until?: string | null;
|
||||||
|
legal_hold?: boolean;
|
||||||
|
lifecycle_revision?: number;
|
||||||
|
lifecycle_reason?: string | null;
|
||||||
audit_relevant: boolean;
|
audit_relevant: boolean;
|
||||||
metadata?: Record<string, unknown> | null;
|
metadata?: Record<string, unknown> | null;
|
||||||
source_provenance?: FileSourceProvenance | null;
|
source_provenance?: FileSourceProvenance | null;
|
||||||
@@ -365,6 +426,22 @@ export type FileConnectorSyncResponse = {
|
|||||||
previous_version_id?: string | null;
|
previous_version_id?: string | null;
|
||||||
current_version_id: string;
|
current_version_id: string;
|
||||||
};
|
};
|
||||||
|
export type FileConnectorWritePayload = {
|
||||||
|
file_id: string;
|
||||||
|
remote_path: string;
|
||||||
|
idempotency_key: string;
|
||||||
|
expected_revision?: string | null;
|
||||||
|
};
|
||||||
|
export type FileConnectorWriteResponse = {
|
||||||
|
recovery_operation_id: string;
|
||||||
|
status: string;
|
||||||
|
replayed: boolean;
|
||||||
|
provider: string;
|
||||||
|
remote_path: string;
|
||||||
|
revision?: string | null;
|
||||||
|
checksum_sha256: string;
|
||||||
|
size_bytes: number;
|
||||||
|
};
|
||||||
export type FileFolder = {
|
export type FileFolder = {
|
||||||
id: string;
|
id: string;
|
||||||
tenant_id: string;
|
tenant_id: string;
|
||||||
@@ -379,7 +456,49 @@ export type FileFoldersResponse = {folders: FileFolder[];cursor?: string | null;
|
|||||||
|
|
||||||
const DEFAULT_MANAGED_FILE_WINDOW_SIZE = 500;
|
const DEFAULT_MANAGED_FILE_WINDOW_SIZE = 500;
|
||||||
export type FolderDeleteResponse = {deleted_folders: number;deleted_files: number;};
|
export type FolderDeleteResponse = {deleted_folders: number;deleted_files: number;};
|
||||||
|
export type FolderRestoreResponse = {restored_folders: number;restored_files: number;};
|
||||||
export type BulkDeleteResponse = {deleted_count: number;};
|
export type BulkDeleteResponse = {deleted_count: number;};
|
||||||
|
export type FileRestoreResponse = {restored_count: number;};
|
||||||
|
export type FileLifecycleUpdatePayload = {
|
||||||
|
retained_until?: string | null;
|
||||||
|
legal_hold: boolean;
|
||||||
|
reason: string;
|
||||||
|
expected_revision: number;
|
||||||
|
};
|
||||||
|
export type FilePurgePreviewItem = {
|
||||||
|
file_id: string;
|
||||||
|
filename: string;
|
||||||
|
lifecycle_revision: number;
|
||||||
|
deleted_at?: string | null;
|
||||||
|
retained_until?: string | null;
|
||||||
|
legal_hold: boolean;
|
||||||
|
blockers: string[];
|
||||||
|
blob_ids: string[];
|
||||||
|
};
|
||||||
|
export type FilePurgePreviewResponse = {
|
||||||
|
preview_sha256: string;
|
||||||
|
eligible: boolean;
|
||||||
|
items: FilePurgePreviewItem[];
|
||||||
|
};
|
||||||
|
export type FilePurgeExecutePayload = {
|
||||||
|
file_ids: string[];
|
||||||
|
preview_sha256: string;
|
||||||
|
idempotency_key: string;
|
||||||
|
approval_reference: string;
|
||||||
|
confirmation: "PURGE";
|
||||||
|
};
|
||||||
|
export type FilePurgeResponse = {
|
||||||
|
recovery_operation_id: string;
|
||||||
|
status: string;
|
||||||
|
replayed: boolean;
|
||||||
|
purged_files: number;
|
||||||
|
released_blobs: number;
|
||||||
|
};
|
||||||
|
export type FileBlobGcResponse = {
|
||||||
|
inspected_blobs: number;
|
||||||
|
deleted_blobs: number;
|
||||||
|
unresolved_operation_ids: string[];
|
||||||
|
};
|
||||||
export type RenameResponse = {dry_run: boolean;items: {kind: "file" | "folder";id: string;file_id?: string | null;folder_path?: string | null;old_path: string;new_path: string;}[];};
|
export type RenameResponse = {dry_run: boolean;items: {kind: "file" | "folder";id: string;file_id?: string | null;folder_path?: string | null;old_path: string;new_path: string;}[];};
|
||||||
export type TransferResponse = {operation: "move" | "copy";files: number;folders: number;};
|
export type TransferResponse = {operation: "move" | "copy";files: number;folders: number;};
|
||||||
export type ConflictAction = "overwrite" | "rename" | "skip";
|
export type ConflictAction = "overwrite" | "rename" | "skip";
|
||||||
@@ -419,7 +538,7 @@ payload: {owner_type: "user" | "group";owner_id: string;path: string;recursive?:
|
|||||||
return apiFetch<FolderDeleteResponse>(settings, "/api/v1/files/folders/delete", { method: "POST", body: JSON.stringify({ recursive: true, ...payload }) });
|
return apiFetch<FolderDeleteResponse>(settings, "/api/v1/files/folders/delete", { method: "POST", body: JSON.stringify({ recursive: true, ...payload }) });
|
||||||
}
|
}
|
||||||
|
|
||||||
export function listFiles(settings: ApiSettings, params: {owner_type?: string;owner_id?: string;campaign_id?: string;path_prefix?: string;campaign_usage?: FileCampaignUsageFilter;audit_relevant?: boolean;page_size?: number;cursor?: string | null;} = {}): Promise<FileListResponse> {
|
export function listFiles(settings: ApiSettings, params: {owner_type?: string;owner_id?: string;campaign_id?: string;path_prefix?: string;campaign_usage?: FileCampaignUsageFilter;audit_relevant?: boolean;sort?: "path" | "recent";page_size?: number;cursor?: string | null;} = {}): Promise<FileListResponse> {
|
||||||
const search = new URLSearchParams();
|
const search = new URLSearchParams();
|
||||||
for (const [key, value] of Object.entries(params)) {
|
for (const [key, value] of Object.entries(params)) {
|
||||||
if (value !== undefined && value !== null && value !== "") search.set(key, String(value));
|
if (value !== undefined && value !== null && value !== "") search.set(key, String(value));
|
||||||
@@ -693,6 +812,55 @@ export function bulkDeleteFiles(settings: ApiSettings, fileIds: string[]): Promi
|
|||||||
return apiFetch<BulkDeleteResponse>(settings, "/api/v1/files/bulk-delete", { method: "POST", body: JSON.stringify({ file_ids: fileIds }) });
|
return apiFetch<BulkDeleteResponse>(settings, "/api/v1/files/bulk-delete", { method: "POST", body: JSON.stringify({ file_ids: fileIds }) });
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function updateFileLifecycle(
|
||||||
|
settings: ApiSettings,
|
||||||
|
fileId: string,
|
||||||
|
payload: FileLifecycleUpdatePayload)
|
||||||
|
: Promise<ManagedFile> {
|
||||||
|
return apiFetch<ManagedFile>(settings, `/api/v1/files/${encodeURIComponent(fileId)}/lifecycle`, {
|
||||||
|
method: "PATCH",
|
||||||
|
body: JSON.stringify(payload)
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export function restoreFile(settings: ApiSettings, fileId: string): Promise<FileRestoreResponse> {
|
||||||
|
return apiFetch<FileRestoreResponse>(settings, `/api/v1/files/assets/${encodeURIComponent(fileId)}/restore`, { method: "POST" });
|
||||||
|
}
|
||||||
|
|
||||||
|
export function restoreFolder(
|
||||||
|
settings: ApiSettings,
|
||||||
|
payload: {owner_type: "user" | "group";owner_id: string;path: string;recursive?: boolean;})
|
||||||
|
: Promise<FolderRestoreResponse> {
|
||||||
|
return apiFetch<FolderRestoreResponse>(settings, "/api/v1/files/folders/restore", {
|
||||||
|
method: "POST",
|
||||||
|
body: JSON.stringify({ recursive: true, ...payload })
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export function previewFilePurge(settings: ApiSettings, fileIds: string[]): Promise<FilePurgePreviewResponse> {
|
||||||
|
return apiFetch<FilePurgePreviewResponse>(settings, "/api/v1/files/purge/preview", {
|
||||||
|
method: "POST",
|
||||||
|
body: JSON.stringify({ file_ids: fileIds })
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export function executeFilePurge(settings: ApiSettings, payload: FilePurgeExecutePayload): Promise<FilePurgeResponse> {
|
||||||
|
return apiFetch<FilePurgeResponse>(settings, "/api/v1/files/purge/execute", {
|
||||||
|
method: "POST",
|
||||||
|
body: JSON.stringify(payload)
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export function garbageCollectFileBlobs(
|
||||||
|
settings: ApiSettings,
|
||||||
|
payload: {limit?: number;approval_reference: string;})
|
||||||
|
: Promise<FileBlobGcResponse> {
|
||||||
|
return apiFetch<FileBlobGcResponse>(settings, "/api/v1/files/purge/blobs", {
|
||||||
|
method: "POST",
|
||||||
|
body: JSON.stringify(payload)
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
export type FileBulkShareResponse = {shares: FileShare[];shared_count: number;};
|
export type FileBulkShareResponse = {shares: FileShare[];shared_count: number;};
|
||||||
export type FileSharePayload = {
|
export type FileSharePayload = {
|
||||||
target_type: "user" | "group" | "tenant" | "campaign";
|
target_type: "user" | "group" | "tenant" | "campaign";
|
||||||
@@ -923,6 +1091,21 @@ export function deleteFileConnectorSpace(settings: ApiSettings, spaceId: string)
|
|||||||
return apiFetch<FileConnectorSpace>(settings, `/api/v1/files/connector-spaces/${encodeURIComponent(spaceId)}`, { method: "DELETE" });
|
return apiFetch<FileConnectorSpace>(settings, `/api/v1/files/connector-spaces/${encodeURIComponent(spaceId)}`, { method: "DELETE" });
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function restoreFileConnectorSpace(settings: ApiSettings, spaceId: string): Promise<FileConnectorSpace> {
|
||||||
|
return apiFetch<FileConnectorSpace>(settings, `/api/v1/files/connector-spaces/${encodeURIComponent(spaceId)}/restore`, { method: "POST" });
|
||||||
|
}
|
||||||
|
|
||||||
|
export function writeBackFileConnectorSpace(
|
||||||
|
settings: ApiSettings,
|
||||||
|
spaceId: string,
|
||||||
|
payload: FileConnectorWritePayload)
|
||||||
|
: Promise<FileConnectorWriteResponse> {
|
||||||
|
return apiFetch<FileConnectorWriteResponse>(settings, `/api/v1/files/connector-spaces/${encodeURIComponent(spaceId)}/write-back`, {
|
||||||
|
method: "POST",
|
||||||
|
body: JSON.stringify(payload)
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
export function getFileConnectorProfile(
|
export function getFileConnectorProfile(
|
||||||
settings: ApiSettings,
|
settings: ApiSettings,
|
||||||
profileId: string,
|
profileId: string,
|
||||||
@@ -1039,6 +1222,91 @@ payload: {
|
|||||||
return apiFetch<TransferResponse>(settings, "/api/v1/files/transfer", { method: "POST", body: JSON.stringify(payload) });
|
return apiFetch<TransferResponse>(settings, "/api/v1/files/transfer", { method: "POST", body: JSON.stringify(payload) });
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export async function listFileIntegrityScans(
|
||||||
|
settings: ApiSettings,
|
||||||
|
limit = 100
|
||||||
|
): Promise<FileIntegrityScan[]> {
|
||||||
|
const response = await apiFetch<{ scans: FileIntegrityScan[] }>(
|
||||||
|
settings,
|
||||||
|
`/api/v1/files/integrity/scans?limit=${Math.max(1, Math.min(limit, 200))}`
|
||||||
|
);
|
||||||
|
return response.scans;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createFileIntegrityScan(
|
||||||
|
settings: ApiSettings,
|
||||||
|
payload: { verify_checksums: boolean; batch_size: number }
|
||||||
|
): Promise<FileIntegrityScan> {
|
||||||
|
return apiFetch<FileIntegrityScan>(settings, "/api/v1/files/integrity/scans", {
|
||||||
|
method: "POST",
|
||||||
|
body: JSON.stringify(payload)
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export function runFileIntegrityScanBatch(
|
||||||
|
settings: ApiSettings,
|
||||||
|
scan: Pick<FileIntegrityScan, "id" | "revision">
|
||||||
|
): Promise<FileIntegrityScan> {
|
||||||
|
return apiFetch<FileIntegrityScan>(
|
||||||
|
settings,
|
||||||
|
`/api/v1/files/integrity/scans/${encodeURIComponent(scan.id)}/run`,
|
||||||
|
{
|
||||||
|
method: "POST",
|
||||||
|
body: JSON.stringify({ expected_revision: scan.revision })
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function listFileIntegrityFindings(
|
||||||
|
settings: ApiSettings,
|
||||||
|
scanId: string,
|
||||||
|
state?: string
|
||||||
|
): Promise<FileIntegrityFinding[]> {
|
||||||
|
const search = new URLSearchParams({ limit: "1000" });
|
||||||
|
if (state) search.set("state", state);
|
||||||
|
const response = await apiFetch<{ findings: FileIntegrityFinding[] }>(
|
||||||
|
settings,
|
||||||
|
`/api/v1/files/integrity/scans/${encodeURIComponent(scanId)}/findings?${search}`
|
||||||
|
);
|
||||||
|
return response.findings;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function recheckFileIntegrityFinding(
|
||||||
|
settings: ApiSettings,
|
||||||
|
finding: Pick<FileIntegrityFinding, "id" | "revision">,
|
||||||
|
dryRun = false
|
||||||
|
): Promise<FileIntegrityActionResult> {
|
||||||
|
return apiFetch<FileIntegrityActionResult>(
|
||||||
|
settings,
|
||||||
|
`/api/v1/files/integrity/findings/${encodeURIComponent(finding.id)}/recheck`,
|
||||||
|
{
|
||||||
|
method: "POST",
|
||||||
|
body: JSON.stringify({
|
||||||
|
dry_run: dryRun,
|
||||||
|
expected_revision: finding.revision
|
||||||
|
})
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function cleanupFileIntegrityFinding(
|
||||||
|
settings: ApiSettings,
|
||||||
|
finding: Pick<FileIntegrityFinding, "id" | "revision">,
|
||||||
|
dryRun: boolean
|
||||||
|
): Promise<FileIntegrityActionResult> {
|
||||||
|
return apiFetch<FileIntegrityActionResult>(
|
||||||
|
settings,
|
||||||
|
`/api/v1/files/integrity/findings/${encodeURIComponent(finding.id)}/cleanup`,
|
||||||
|
{
|
||||||
|
method: "POST",
|
||||||
|
body: JSON.stringify({
|
||||||
|
dry_run: dryRun,
|
||||||
|
expected_revision: finding.revision
|
||||||
|
})
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
export async function downloadFile(settings: ApiSettings, file: ManagedFile): Promise<void> {
|
export async function downloadFile(settings: ApiSettings, file: ManagedFile): Promise<void> {
|
||||||
const response = await fetch(apiUrl(settings, `/api/v1/files/${file.id}/download`), { headers: authHeaders(settings), credentials: "include" });
|
const response = await fetch(apiUrl(settings, `/api/v1/files/${file.id}/download`), { headers: authHeaders(settings), credentials: "include" });
|
||||||
if (!response.ok) throw new Error(`${response.status} ${response.statusText}: ${await response.text()}`);
|
if (!response.ok) throw new Error(`${response.status} ${response.statusText}: ${await response.text()}`);
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import { useEffect, useMemo, useState } from "react";
|
import { useEffect, useMemo, useState } from "react";
|
||||||
import { Edit3, Plus, RefreshCw, Save, ShieldCheck, Trash2, UserRoundKey } from "lucide-react";
|
import { Edit3, Plus, RefreshCw, Save, ShieldCheck, Trash2, UserRoundKey } from "lucide-react";
|
||||||
import {
|
import { FormGrid,
|
||||||
Button,
|
Button,
|
||||||
Card,
|
Card,
|
||||||
ConnectionTree,
|
ConnectionTree,
|
||||||
@@ -76,6 +76,7 @@ type ConnectorProfileDraft = {
|
|||||||
canBrowse: boolean;
|
canBrowse: boolean;
|
||||||
canImport: boolean;
|
canImport: boolean;
|
||||||
canSync: boolean;
|
canSync: boolean;
|
||||||
|
canWriteRemote: boolean;
|
||||||
policyMode: PolicyMode;
|
policyMode: PolicyMode;
|
||||||
allowedPaths: string;
|
allowedPaths: string;
|
||||||
deniedPaths: string;
|
deniedPaths: string;
|
||||||
@@ -509,7 +510,8 @@ export default function FileConnectorSettingsPanel({
|
|||||||
const capabilities = [
|
const capabilities = [
|
||||||
draft.canBrowse ? "browse" : "",
|
draft.canBrowse ? "browse" : "",
|
||||||
draft.canImport ? "import" : "",
|
draft.canImport ? "import" : "",
|
||||||
draft.canSync ? "sync" : ""].
|
draft.canSync ? "sync" : "",
|
||||||
|
draft.canWriteRemote ? "write" : ""].
|
||||||
filter(Boolean);
|
filter(Boolean);
|
||||||
const sharedPayload = {
|
const sharedPayload = {
|
||||||
label,
|
label,
|
||||||
@@ -963,7 +965,7 @@ export default function FileConnectorSettingsPanel({
|
|||||||
|
|
||||||
<LoadingFrame loading={loading} label="i18n:govoplan-files.loading_connector_policy.f12ab285">
|
<LoadingFrame loading={loading} label="i18n:govoplan-files.loading_connector_policy.f12ab285">
|
||||||
<>
|
<>
|
||||||
<div className="form-grid two">
|
<FormGrid columns={2} collapseAt="standard" className="">
|
||||||
<FormField label="i18n:govoplan-files.allowed_connection_ids.0df854c8">
|
<FormField label="i18n:govoplan-files.allowed_connection_ids.0df854c8">
|
||||||
<ReferenceMultiSelect
|
<ReferenceMultiSelect
|
||||||
values={lines(policyDraft.allowedConnectors)}
|
values={lines(policyDraft.allowedConnectors)}
|
||||||
@@ -1042,7 +1044,7 @@ export default function FileConnectorSettingsPanel({
|
|||||||
<FormField label="i18n:govoplan-files.denied_endpoint_urls.1d8d4369">
|
<FormField label="i18n:govoplan-files.denied_endpoint_urls.1d8d4369">
|
||||||
<textarea value={policyDraft.deniedUrls} disabled={saving || !canWrite} onChange={(event) => patchPolicyDraft({ deniedUrls: event.target.value })} rows={3} placeholder={"http://*\n*://legacy.example.test/*"} />
|
<textarea value={policyDraft.deniedUrls} disabled={saving || !canWrite} onChange={(event) => patchPolicyDraft({ deniedUrls: event.target.value })} rows={3} placeholder={"http://*\n*://legacy.example.test/*"} />
|
||||||
</FormField>
|
</FormField>
|
||||||
</div>
|
</FormGrid>
|
||||||
{policyConflicts.length ? (
|
{policyConflicts.length ? (
|
||||||
<DismissibleAlert
|
<DismissibleAlert
|
||||||
tone="warning"
|
tone="warning"
|
||||||
@@ -1075,11 +1077,11 @@ export default function FileConnectorSettingsPanel({
|
|||||||
</LoadingFrame>
|
</LoadingFrame>
|
||||||
</Card>
|
</Card>
|
||||||
|
|
||||||
<Dialog
|
<Dialog variant="administration" size="wide"
|
||||||
open={Boolean(credentialDraft)}
|
open={Boolean(credentialDraft)}
|
||||||
title={editingCredentialId ? "i18n:govoplan-files.edit_file_credential.bc49d44f" : "i18n:govoplan-files.new_file_credential.4c061082"}
|
title={editingCredentialId ? "i18n:govoplan-files.edit_file_credential.bc49d44f" : "i18n:govoplan-files.new_file_credential.4c061082"}
|
||||||
onClose={closeCredentialDialog}
|
onClose={closeCredentialDialog}
|
||||||
className="admin-dialog admin-dialog-wide file-connector-dialog adaptive-config-dialog"
|
className="file-connector-dialog"
|
||||||
bodyClassName="adaptive-config-dialog-body"
|
bodyClassName="adaptive-config-dialog-body"
|
||||||
closeDisabled={saving}
|
closeDisabled={saving}
|
||||||
footer={credentialDraft ?
|
footer={credentialDraft ?
|
||||||
@@ -1098,7 +1100,7 @@ export default function FileConnectorSettingsPanel({
|
|||||||
<h3>Credential</h3>
|
<h3>Credential</h3>
|
||||||
<p>Choose where this credential can be used and how it signs in.</p>
|
<p>Choose where this credential can be used and how it signs in.</p>
|
||||||
</header>
|
</header>
|
||||||
<div className="form-grid two">
|
<FormGrid columns={2} collapseAt="standard" className="">
|
||||||
<FormField label="i18n:govoplan-files.credential_id.9432a6e1">
|
<FormField label="i18n:govoplan-files.credential_id.9432a6e1">
|
||||||
<input className={!editingCredentialId && !credentialDraft.id.trim() ? "field-input-missing" : undefined} aria-invalid={!editingCredentialId && !credentialDraft.id.trim() || undefined} value={credentialDraft.id} disabled={Boolean(editingCredentialId) || saving} onChange={(event) => patchCredentialDraft({ id: event.target.value })} placeholder={`${scopeType}-webdav-credentials`} />
|
<input className={!editingCredentialId && !credentialDraft.id.trim() ? "field-input-missing" : undefined} aria-invalid={!editingCredentialId && !credentialDraft.id.trim() || undefined} value={credentialDraft.id} disabled={Boolean(editingCredentialId) || saving} onChange={(event) => patchCredentialDraft({ id: event.target.value })} placeholder={`${scopeType}-webdav-credentials`} />
|
||||||
</FormField>
|
</FormField>
|
||||||
@@ -1137,7 +1139,7 @@ export default function FileConnectorSettingsPanel({
|
|||||||
<option value="secret_ref">i18n:govoplan-files.secret_reference.04ed2221</option>
|
<option value="secret_ref">i18n:govoplan-files.secret_reference.04ed2221</option>
|
||||||
</select>
|
</select>
|
||||||
</FormField>
|
</FormField>
|
||||||
</div>
|
</FormGrid>
|
||||||
<div className="file-connector-settings-section">
|
<div className="file-connector-settings-section">
|
||||||
<ToggleSwitch label="i18n:govoplan-files.enabled.df174a3f" checked={credentialDraft.enabled} disabled={saving} onChange={(enabled) => patchCredentialDraft({ enabled })} />
|
<ToggleSwitch label="i18n:govoplan-files.enabled.df174a3f" checked={credentialDraft.enabled} disabled={saving} onChange={(enabled) => patchCredentialDraft({ enabled })} />
|
||||||
{editingCredentialId && <ToggleSwitch label="i18n:govoplan-files.clear_saved_password.7442260d" checked={credentialDraft.clearPassword} disabled={saving} onChange={(clearPassword) => patchCredentialDraft({ clearPassword })} />}
|
{editingCredentialId && <ToggleSwitch label="i18n:govoplan-files.clear_saved_password.7442260d" checked={credentialDraft.clearPassword} disabled={saving} onChange={(clearPassword) => patchCredentialDraft({ clearPassword })} />}
|
||||||
@@ -1176,11 +1178,11 @@ export default function FileConnectorSettingsPanel({
|
|||||||
onChange={patchCredentialValues}
|
onChange={patchCredentialValues}
|
||||||
disabled={saving}
|
disabled={saving}
|
||||||
showPassword={false} />
|
showPassword={false} />
|
||||||
<div className="form-grid two">
|
<FormGrid columns={2} collapseAt="standard" className="">
|
||||||
<FormField label="i18n:govoplan-files.secret_reference.04ed2221">
|
<FormField label="i18n:govoplan-files.secret_reference.04ed2221">
|
||||||
<input value={credentialDraft.secretRef} disabled={saving} onChange={(event) => patchCredentialDraft({ secretRef: event.target.value })} />
|
<input value={credentialDraft.secretRef} disabled={saving} onChange={(event) => patchCredentialDraft({ secretRef: event.target.value })} />
|
||||||
</FormField>
|
</FormField>
|
||||||
</div>
|
</FormGrid>
|
||||||
</>
|
</>
|
||||||
}
|
}
|
||||||
{credentialDraft.credentialMode === "token" &&
|
{credentialDraft.credentialMode === "token" &&
|
||||||
@@ -1212,7 +1214,7 @@ export default function FileConnectorSettingsPanel({
|
|||||||
<h3>Policy</h3>
|
<h3>Policy</h3>
|
||||||
<p>Limit where lower levels may use this credential.</p>
|
<p>Limit where lower levels may use this credential.</p>
|
||||||
</header>
|
</header>
|
||||||
<div className="form-grid two">
|
<FormGrid columns={2} collapseAt="standard" className="">
|
||||||
<FormField label="i18n:govoplan-files.policy.bb9cf141" documentation={CONNECTOR_DOCUMENTATION}>
|
<FormField label="i18n:govoplan-files.policy.bb9cf141" documentation={CONNECTOR_DOCUMENTATION}>
|
||||||
<select value={credentialDraft.policyMode} disabled={saving} onChange={(event) => patchCredentialDraft({ policyMode: event.target.value as PolicyMode })}>
|
<select value={credentialDraft.policyMode} disabled={saving} onChange={(event) => patchCredentialDraft({ policyMode: event.target.value as PolicyMode })}>
|
||||||
<option value="allow-provider">i18n:govoplan-files.can_use_this_credential.f4a41971</option>
|
<option value="allow-provider">i18n:govoplan-files.can_use_this_credential.f4a41971</option>
|
||||||
@@ -1228,7 +1230,7 @@ export default function FileConnectorSettingsPanel({
|
|||||||
<FormField label="i18n:govoplan-files.denied_paths.d25e4315">
|
<FormField label="i18n:govoplan-files.denied_paths.d25e4315">
|
||||||
<textarea value={credentialDraft.deniedPaths} disabled={saving} onChange={(event) => patchCredentialDraft({ deniedPaths: event.target.value })} rows={3} placeholder={"private\narchive/closed"} />
|
<textarea value={credentialDraft.deniedPaths} disabled={saving} onChange={(event) => patchCredentialDraft({ deniedPaths: event.target.value })} rows={3} placeholder={"private\narchive/closed"} />
|
||||||
</FormField>
|
</FormField>
|
||||||
</div>
|
</FormGrid>
|
||||||
<AdvancedOptionsPanel title="Credential metadata JSON" summary="For diagnostics or provider-specific compatibility flags. Normal setup should not need this.">
|
<AdvancedOptionsPanel title="Credential metadata JSON" summary="For diagnostics or provider-specific compatibility flags. Normal setup should not need this.">
|
||||||
<FormField label="i18n:govoplan-files.metadata_json.b0e4c283" documentation={CONNECTOR_RECOVERY_DOCUMENTATION}>
|
<FormField label="i18n:govoplan-files.metadata_json.b0e4c283" documentation={CONNECTOR_RECOVERY_DOCUMENTATION}>
|
||||||
<textarea value={credentialDraft.metadataJson} disabled={saving} onChange={(event) => patchCredentialDraft({ metadataJson: event.target.value })} rows={5} spellCheck={false} />
|
<textarea value={credentialDraft.metadataJson} disabled={saving} onChange={(event) => patchCredentialDraft({ metadataJson: event.target.value })} rows={5} spellCheck={false} />
|
||||||
@@ -1239,11 +1241,11 @@ export default function FileConnectorSettingsPanel({
|
|||||||
}
|
}
|
||||||
</Dialog>
|
</Dialog>
|
||||||
|
|
||||||
<Dialog
|
<Dialog variant="administration" size="wide"
|
||||||
open={Boolean(draft)}
|
open={Boolean(draft)}
|
||||||
title={editingProfileId ? "i18n:govoplan-files.edit_file_connection.78698154" : "i18n:govoplan-files.new_file_connection.2ec6eb56"}
|
title={editingProfileId ? "i18n:govoplan-files.edit_file_connection.78698154" : "i18n:govoplan-files.new_file_connection.2ec6eb56"}
|
||||||
onClose={closeProfileDialog}
|
onClose={closeProfileDialog}
|
||||||
className="admin-dialog admin-dialog-wide file-connector-dialog adaptive-config-dialog"
|
className="file-connector-dialog"
|
||||||
bodyClassName="adaptive-config-dialog-body"
|
bodyClassName="adaptive-config-dialog-body"
|
||||||
closeDisabled={saving}
|
closeDisabled={saving}
|
||||||
footer={draft ?
|
footer={draft ?
|
||||||
@@ -1262,7 +1264,7 @@ export default function FileConnectorSettingsPanel({
|
|||||||
<h3>Connection</h3>
|
<h3>Connection</h3>
|
||||||
<p>Name the file server connection and choose the provider.</p>
|
<p>Name the file server connection and choose the provider.</p>
|
||||||
</header>
|
</header>
|
||||||
<div className="form-grid two">
|
<FormGrid columns={2} collapseAt="standard" className="">
|
||||||
<FormField label="i18n:govoplan-files.profile_id.25961d68">
|
<FormField label="i18n:govoplan-files.profile_id.25961d68">
|
||||||
<input className={!editingProfileId && !draft.id.trim() ? "field-input-missing" : undefined} aria-invalid={!editingProfileId && !draft.id.trim() || undefined} value={draft.id} disabled={Boolean(editingProfileId) || saving} onChange={(event) => patchDraft({ id: event.target.value })} placeholder={`${scopeType}-webdav`} />
|
<input className={!editingProfileId && !draft.id.trim() ? "field-input-missing" : undefined} aria-invalid={!editingProfileId && !draft.id.trim() || undefined} value={draft.id} disabled={Boolean(editingProfileId) || saving} onChange={(event) => patchDraft({ id: event.target.value })} placeholder={`${scopeType}-webdav`} />
|
||||||
</FormField>
|
</FormField>
|
||||||
@@ -1287,7 +1289,7 @@ export default function FileConnectorSettingsPanel({
|
|||||||
<div className="field-block file-connector-settings-toggle-field">
|
<div className="field-block file-connector-settings-toggle-field">
|
||||||
<ToggleSwitch label="i18n:govoplan-files.enabled.df174a3f" checked={draft.enabled} disabled={saving} onChange={(enabled) => patchDraft({ enabled })} />
|
<ToggleSwitch label="i18n:govoplan-files.enabled.df174a3f" checked={draft.enabled} disabled={saving} onChange={(enabled) => patchDraft({ enabled })} />
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</FormGrid>
|
||||||
</section>
|
</section>
|
||||||
|
|
||||||
<section className="adaptive-config-section">
|
<section className="adaptive-config-section">
|
||||||
@@ -1295,7 +1297,7 @@ export default function FileConnectorSettingsPanel({
|
|||||||
<h3>Location and credentials</h3>
|
<h3>Location and credentials</h3>
|
||||||
<p>Only fields relevant for the selected provider and credential mode are shown.</p>
|
<p>Only fields relevant for the selected provider and credential mode are shown.</p>
|
||||||
</header>
|
</header>
|
||||||
<div className="form-grid two">
|
<FormGrid columns={2} collapseAt="standard" className="">
|
||||||
<FormField label="i18n:govoplan-files.endpoint_url.65aaaa45" documentation={CONNECTOR_DOCUMENTATION}>
|
<FormField label="i18n:govoplan-files.endpoint_url.65aaaa45" documentation={CONNECTOR_DOCUMENTATION}>
|
||||||
<input value={draft.endpointUrl} disabled={saving} onChange={(event) => patchDraft({ endpointUrl: event.target.value })} placeholder={ENDPOINT_PLACEHOLDERS[draft.provider]} />
|
<input value={draft.endpointUrl} disabled={saving} onChange={(event) => patchDraft({ endpointUrl: event.target.value })} placeholder={ENDPOINT_PLACEHOLDERS[draft.provider]} />
|
||||||
</FormField>
|
</FormField>
|
||||||
@@ -1317,7 +1319,7 @@ export default function FileConnectorSettingsPanel({
|
|||||||
{profileCredentialOptions.map((credential) => <option key={credential.id} value={credential.id}>{credential.label}</option>)}
|
{profileCredentialOptions.map((credential) => <option key={credential.id} value={credential.id}>{credential.label}</option>)}
|
||||||
</select>
|
</select>
|
||||||
</FormField>
|
</FormField>
|
||||||
</div>
|
</FormGrid>
|
||||||
<div className="button-row compact-actions">
|
<div className="button-row compact-actions">
|
||||||
<Button type="button" onClick={() => patchDraft({ endpointUrl: ENDPOINT_PLACEHOLDERS[draft.provider] })} disabled={saving} disabledReason={saving ? "Wait for the current connector change to finish." : undefined}>Use provider example</Button>
|
<Button type="button" onClick={() => patchDraft({ endpointUrl: ENDPOINT_PLACEHOLDERS[draft.provider] })} disabled={saving} disabledReason={saving ? "Wait for the current connector change to finish." : undefined}>Use provider example</Button>
|
||||||
{canDiscoverProfileEndpoint &&
|
{canDiscoverProfileEndpoint &&
|
||||||
@@ -1332,7 +1334,7 @@ export default function FileConnectorSettingsPanel({
|
|||||||
</DismissibleAlert>
|
</DismissibleAlert>
|
||||||
}
|
}
|
||||||
<AdvancedOptionsPanel title="Protocol and compatibility options" summary="Change these only when the provider or network requires an override.">
|
<AdvancedOptionsPanel title="Protocol and compatibility options" summary="Change these only when the provider or network requires an override.">
|
||||||
<div className="form-grid two">
|
<FormGrid columns={2} collapseAt="standard" className="">
|
||||||
<FormField label="i18n:govoplan-files.browse_protocol.d1f27c90">
|
<FormField label="i18n:govoplan-files.browse_protocol.d1f27c90">
|
||||||
<select value={draft.browseProtocol} disabled={saving} onChange={(event) => patchDraft({ browseProtocol: event.target.value })}>
|
<select value={draft.browseProtocol} disabled={saving} onChange={(event) => patchDraft({ browseProtocol: event.target.value })}>
|
||||||
<option value="">i18n:govoplan-files.native_default.ba32f7b6</option>
|
<option value="">i18n:govoplan-files.native_default.ba32f7b6</option>
|
||||||
@@ -1352,7 +1354,7 @@ export default function FileConnectorSettingsPanel({
|
|||||||
</div>
|
</div>
|
||||||
</>
|
</>
|
||||||
}
|
}
|
||||||
</div>
|
</FormGrid>
|
||||||
</AdvancedOptionsPanel>
|
</AdvancedOptionsPanel>
|
||||||
</section>
|
</section>
|
||||||
|
|
||||||
@@ -1365,8 +1367,9 @@ export default function FileConnectorSettingsPanel({
|
|||||||
<ToggleSwitch label="i18n:govoplan-files.browse.2f3b5c55" checked={draft.canBrowse} disabled={saving} onChange={(canBrowse) => patchDraft({ canBrowse })} />
|
<ToggleSwitch label="i18n:govoplan-files.browse.2f3b5c55" checked={draft.canBrowse} disabled={saving} onChange={(canBrowse) => patchDraft({ canBrowse })} />
|
||||||
<ToggleSwitch label="i18n:govoplan-files.import.d6fbc9d2" checked={draft.canImport} disabled={saving} onChange={(canImport) => patchDraft({ canImport })} />
|
<ToggleSwitch label="i18n:govoplan-files.import.d6fbc9d2" checked={draft.canImport} disabled={saving} onChange={(canImport) => patchDraft({ canImport })} />
|
||||||
<ToggleSwitch label="i18n:govoplan-files.sync.905f6309" checked={draft.canSync} disabled={saving} onChange={(canSync) => patchDraft({ canSync })} />
|
<ToggleSwitch label="i18n:govoplan-files.sync.905f6309" checked={draft.canSync} disabled={saving} onChange={(canSync) => patchDraft({ canSync })} />
|
||||||
|
<ToggleSwitch label="Remote write" checked={draft.canWriteRemote} disabled={saving || draft.provider !== "s3"} onChange={(canWriteRemote) => patchDraft({ canWriteRemote })} />
|
||||||
</div>
|
</div>
|
||||||
<div className="form-grid two">
|
<FormGrid columns={2} collapseAt="standard" className="">
|
||||||
<FormField label="i18n:govoplan-files.policy.bb9cf141" documentation={CONNECTOR_DOCUMENTATION}>
|
<FormField label="i18n:govoplan-files.policy.bb9cf141" documentation={CONNECTOR_DOCUMENTATION}>
|
||||||
<select value={draft.policyMode} disabled={saving} onChange={(event) => patchDraft({ policyMode: event.target.value as PolicyMode })}>
|
<select value={draft.policyMode} disabled={saving} onChange={(event) => patchDraft({ policyMode: event.target.value as PolicyMode })}>
|
||||||
<option value="allow-provider">i18n:govoplan-files.can_use_this_connection.5091a74c</option>
|
<option value="allow-provider">i18n:govoplan-files.can_use_this_connection.5091a74c</option>
|
||||||
@@ -1382,7 +1385,7 @@ export default function FileConnectorSettingsPanel({
|
|||||||
<FormField label="i18n:govoplan-files.denied_paths.d25e4315">
|
<FormField label="i18n:govoplan-files.denied_paths.d25e4315">
|
||||||
<textarea value={draft.deniedPaths} disabled={saving} onChange={(event) => patchDraft({ deniedPaths: event.target.value })} rows={3} placeholder={"private\narchive/closed"} />
|
<textarea value={draft.deniedPaths} disabled={saving} onChange={(event) => patchDraft({ deniedPaths: event.target.value })} rows={3} placeholder={"private\narchive/closed"} />
|
||||||
</FormField>
|
</FormField>
|
||||||
</div>
|
</FormGrid>
|
||||||
<AdvancedOptionsPanel title="Connection metadata JSON" summary="For diagnostics or provider-specific compatibility flags. Normal setup should not need this.">
|
<AdvancedOptionsPanel title="Connection metadata JSON" summary="For diagnostics or provider-specific compatibility flags. Normal setup should not need this.">
|
||||||
<FormField label="i18n:govoplan-files.metadata_json.b0e4c283" documentation={CONNECTOR_RECOVERY_DOCUMENTATION}>
|
<FormField label="i18n:govoplan-files.metadata_json.b0e4c283" documentation={CONNECTOR_RECOVERY_DOCUMENTATION}>
|
||||||
<textarea value={draft.metadataJson} disabled={saving} onChange={(event) => patchDraft({ metadataJson: event.target.value })} rows={5} spellCheck={false} />
|
<textarea value={draft.metadataJson} disabled={saving} onChange={(event) => patchDraft({ metadataJson: event.target.value })} rows={5} spellCheck={false} />
|
||||||
@@ -1528,6 +1531,7 @@ function emptyDraft(scopeType: ConnectorScope): ConnectorProfileDraft {
|
|||||||
canBrowse: true,
|
canBrowse: true,
|
||||||
canImport: true,
|
canImport: true,
|
||||||
canSync: true,
|
canSync: true,
|
||||||
|
canWriteRemote: false,
|
||||||
policyMode: "allow-provider",
|
policyMode: "allow-provider",
|
||||||
allowedPaths: "",
|
allowedPaths: "",
|
||||||
deniedPaths: "",
|
deniedPaths: "",
|
||||||
@@ -1566,6 +1570,7 @@ function draftFromProfile(profile: FileConnectorProfile, scopeType: ConnectorSco
|
|||||||
canBrowse: profile.capabilities.includes("browse"),
|
canBrowse: profile.capabilities.includes("browse"),
|
||||||
canImport: profile.capabilities.includes("import"),
|
canImport: profile.capabilities.includes("import"),
|
||||||
canSync: profile.capabilities.includes("sync"),
|
canSync: profile.capabilities.includes("sync"),
|
||||||
|
canWriteRemote: profile.capabilities.includes("write"),
|
||||||
policyMode: denyProviders.includes("*") || denyProviders.includes(profile.provider) ? "deny-provider" : allowPaths.length > 0 ? "allowed-paths" : "allow-provider",
|
policyMode: denyProviders.includes("*") || denyProviders.includes(profile.provider) ? "deny-provider" : allowPaths.length > 0 ? "allowed-paths" : "allow-provider",
|
||||||
allowedPaths: allowPaths.join("\n"),
|
allowedPaths: allowPaths.join("\n"),
|
||||||
deniedPaths: denyPaths.join("\n"),
|
deniedPaths: denyPaths.join("\n"),
|
||||||
|
|||||||
@@ -0,0 +1,528 @@
|
|||||||
|
import { MetricGrid } from "@govoplan/core-webui";
|
||||||
|
import { useEffect, useMemo, useState } from "react";
|
||||||
|
import {
|
||||||
|
AdminPageLayout,
|
||||||
|
adminErrorMessage,
|
||||||
|
Button,
|
||||||
|
Card,
|
||||||
|
ConfirmDialog,
|
||||||
|
DataGrid,
|
||||||
|
Dialog,
|
||||||
|
DismissibleAlert,
|
||||||
|
DocumentationHelpLink,
|
||||||
|
FormField,
|
||||||
|
MetricCard,
|
||||||
|
StatusBadge,
|
||||||
|
TableActionGroup,
|
||||||
|
ToggleSwitch,
|
||||||
|
type ApiSettings,
|
||||||
|
type DataGridColumn
|
||||||
|
} from "@govoplan/core-webui";
|
||||||
|
import {
|
||||||
|
CheckCircle2,
|
||||||
|
Eye,
|
||||||
|
Play,
|
||||||
|
Plus,
|
||||||
|
RefreshCw,
|
||||||
|
RotateCw,
|
||||||
|
Trash2
|
||||||
|
} from "lucide-react";
|
||||||
|
import {
|
||||||
|
cleanupFileIntegrityFinding,
|
||||||
|
createFileIntegrityScan,
|
||||||
|
listFileIntegrityFindings,
|
||||||
|
listFileIntegrityScans,
|
||||||
|
recheckFileIntegrityFinding,
|
||||||
|
runFileIntegrityScanBatch,
|
||||||
|
type FileIntegrityActionResult,
|
||||||
|
type FileIntegrityFinding,
|
||||||
|
type FileIntegrityScan
|
||||||
|
} from "../../api/files";
|
||||||
|
|
||||||
|
type Props = {
|
||||||
|
settings: ApiSettings;
|
||||||
|
canWrite: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
|
const DOCUMENTATION = {
|
||||||
|
topicId: "files.reference.integrity-recovery-and-fail-closed-transports",
|
||||||
|
documentationType: "admin" as const
|
||||||
|
};
|
||||||
|
|
||||||
|
export default function FileIntegrityPanel({ settings, canWrite }: Props) {
|
||||||
|
const [scans, setScans] = useState<FileIntegrityScan[]>([]);
|
||||||
|
const [selectedScanId, setSelectedScanId] = useState("");
|
||||||
|
const [findings, setFindings] = useState<FileIntegrityFinding[]>([]);
|
||||||
|
const [loading, setLoading] = useState(true);
|
||||||
|
const [busy, setBusy] = useState(false);
|
||||||
|
const [error, setError] = useState("");
|
||||||
|
const [success, setSuccess] = useState("");
|
||||||
|
const [createOpen, setCreateOpen] = useState(false);
|
||||||
|
const [verifyChecksums, setVerifyChecksums] = useState(true);
|
||||||
|
const [batchSize, setBatchSize] = useState(100);
|
||||||
|
const [cleanupPreview, setCleanupPreview] = useState<FileIntegrityActionResult | null>(null);
|
||||||
|
|
||||||
|
const selectedScan = scans.find((scan) => scan.id === selectedScanId) ?? null;
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
void loadScans();
|
||||||
|
}, [settings.accessToken, settings.apiBaseUrl, settings.apiKey]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (!selectedScanId) {
|
||||||
|
setFindings([]);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
void loadFindings(selectedScanId);
|
||||||
|
}, [selectedScanId, settings.accessToken, settings.apiBaseUrl, settings.apiKey]);
|
||||||
|
|
||||||
|
async function loadScans() {
|
||||||
|
setLoading(true);
|
||||||
|
setError("");
|
||||||
|
try {
|
||||||
|
const loaded = await listFileIntegrityScans(settings);
|
||||||
|
setScans(loaded);
|
||||||
|
setSelectedScanId((current) => (
|
||||||
|
loaded.some((scan) => scan.id === current) ? current : loaded[0]?.id ?? ""
|
||||||
|
));
|
||||||
|
} catch (err) {
|
||||||
|
setError(adminErrorMessage(err));
|
||||||
|
} finally {
|
||||||
|
setLoading(false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function loadFindings(scanId: string) {
|
||||||
|
setError("");
|
||||||
|
try {
|
||||||
|
setFindings(await listFileIntegrityFindings(settings, scanId));
|
||||||
|
} catch (err) {
|
||||||
|
setError(adminErrorMessage(err));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function replaceScan(next: FileIntegrityScan) {
|
||||||
|
setScans((current) => [
|
||||||
|
next,
|
||||||
|
...current.filter((scan) => scan.id !== next.id)
|
||||||
|
].sort((left, right) => right.created_at.localeCompare(left.created_at)));
|
||||||
|
}
|
||||||
|
|
||||||
|
function replaceFinding(next: FileIntegrityFinding) {
|
||||||
|
setFindings((current) => current.map((finding) => (
|
||||||
|
finding.id === next.id ? next : finding
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
|
||||||
|
async function createScan() {
|
||||||
|
setBusy(true);
|
||||||
|
setError("");
|
||||||
|
setSuccess("");
|
||||||
|
try {
|
||||||
|
const scan = await createFileIntegrityScan(settings, {
|
||||||
|
verify_checksums: verifyChecksums,
|
||||||
|
batch_size: batchSize
|
||||||
|
});
|
||||||
|
replaceScan(scan);
|
||||||
|
setSelectedScanId(scan.id);
|
||||||
|
setCreateOpen(false);
|
||||||
|
setSuccess("Integrity scan created. Run its first bounded batch when ready.");
|
||||||
|
} catch (err) {
|
||||||
|
setError(adminErrorMessage(err));
|
||||||
|
} finally {
|
||||||
|
setBusy(false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function runBatch(scan: FileIntegrityScan) {
|
||||||
|
setBusy(true);
|
||||||
|
setError("");
|
||||||
|
setSuccess("");
|
||||||
|
try {
|
||||||
|
const next = await runFileIntegrityScanBatch(settings, scan);
|
||||||
|
replaceScan(next);
|
||||||
|
setSelectedScanId(next.id);
|
||||||
|
await loadFindings(next.id);
|
||||||
|
setSuccess(
|
||||||
|
next.status === "completed"
|
||||||
|
? "Integrity scan completed. Review and resolve every finding."
|
||||||
|
: `Completed one ${next.batch_size}-item batch; the scan can be resumed.`
|
||||||
|
);
|
||||||
|
} catch (err) {
|
||||||
|
setError(adminErrorMessage(err));
|
||||||
|
await loadScans();
|
||||||
|
} finally {
|
||||||
|
setBusy(false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function recheck(finding: FileIntegrityFinding) {
|
||||||
|
setBusy(true);
|
||||||
|
setError("");
|
||||||
|
setSuccess("");
|
||||||
|
try {
|
||||||
|
const result = await recheckFileIntegrityFinding(settings, finding);
|
||||||
|
replaceFinding(result.finding);
|
||||||
|
setSuccess(
|
||||||
|
result.inspection_valid
|
||||||
|
? "The stored object now matches its recorded integrity evidence."
|
||||||
|
: "The object still fails integrity verification and remains quarantined."
|
||||||
|
);
|
||||||
|
} catch (err) {
|
||||||
|
setError(adminErrorMessage(err));
|
||||||
|
if (selectedScanId) await loadFindings(selectedScanId);
|
||||||
|
} finally {
|
||||||
|
setBusy(false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function previewCleanup(finding: FileIntegrityFinding) {
|
||||||
|
setBusy(true);
|
||||||
|
setError("");
|
||||||
|
setSuccess("");
|
||||||
|
try {
|
||||||
|
const preview = await cleanupFileIntegrityFinding(settings, finding, true);
|
||||||
|
setCleanupPreview(preview);
|
||||||
|
if (preview.action !== "would_delete") {
|
||||||
|
replaceFinding(preview.finding);
|
||||||
|
setSuccess(cleanupActionMessage(preview.action));
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
setError(adminErrorMessage(err));
|
||||||
|
if (selectedScanId) await loadFindings(selectedScanId);
|
||||||
|
} finally {
|
||||||
|
setBusy(false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function confirmCleanup() {
|
||||||
|
if (!cleanupPreview) return;
|
||||||
|
setBusy(true);
|
||||||
|
setError("");
|
||||||
|
setSuccess("");
|
||||||
|
try {
|
||||||
|
const result = await cleanupFileIntegrityFinding(
|
||||||
|
settings,
|
||||||
|
cleanupPreview.finding,
|
||||||
|
false
|
||||||
|
);
|
||||||
|
replaceFinding(result.finding);
|
||||||
|
setCleanupPreview(null);
|
||||||
|
setSuccess(cleanupActionMessage(result.action));
|
||||||
|
} catch (err) {
|
||||||
|
setCleanupPreview(null);
|
||||||
|
setError(adminErrorMessage(err));
|
||||||
|
if (selectedScanId) await loadFindings(selectedScanId);
|
||||||
|
} finally {
|
||||||
|
setBusy(false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const scanColumns = useMemo<DataGridColumn<FileIntegrityScan>[]>(() => [
|
||||||
|
{
|
||||||
|
id: "created",
|
||||||
|
header: "Created",
|
||||||
|
width: 180,
|
||||||
|
minWidth: 150,
|
||||||
|
sortable: true,
|
||||||
|
value: (scan) => scan.created_at,
|
||||||
|
render: (scan) => formatDateTime(scan.created_at)
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "status",
|
||||||
|
header: "Status",
|
||||||
|
width: 130,
|
||||||
|
minWidth: 110,
|
||||||
|
sortable: true,
|
||||||
|
filterable: true,
|
||||||
|
filterType: "list",
|
||||||
|
value: (scan) => scan.status,
|
||||||
|
render: (scan) => <StatusBadge status={statusTone(scan.status)} label={scan.status} />
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "phase",
|
||||||
|
header: "Phase",
|
||||||
|
width: 110,
|
||||||
|
minWidth: 90,
|
||||||
|
value: (scan) => scan.phase
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "progress",
|
||||||
|
header: "Progress",
|
||||||
|
minWidth: 250,
|
||||||
|
resizable: true,
|
||||||
|
value: (scan) => `${scan.scanned_blob_count} blobs, ${scan.scanned_object_count} objects`,
|
||||||
|
render: (scan) => (
|
||||||
|
<span>
|
||||||
|
{scan.scanned_blob_count} blobs ({scan.verified_blob_count} verified, {scan.quarantined_blob_count} quarantined), {" "}
|
||||||
|
{scan.scanned_object_count} objects ({scan.orphan_object_count} orphaned)
|
||||||
|
</span>
|
||||||
|
)
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "backend",
|
||||||
|
header: "Storage",
|
||||||
|
width: 120,
|
||||||
|
minWidth: 100,
|
||||||
|
value: (scan) => scan.storage_backend
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "actions",
|
||||||
|
header: "Actions",
|
||||||
|
width: 105,
|
||||||
|
minWidth: 105,
|
||||||
|
sticky: "end",
|
||||||
|
align: "right",
|
||||||
|
render: (scan) => (
|
||||||
|
<TableActionGroup
|
||||||
|
minimumSlots={2}
|
||||||
|
actions={[
|
||||||
|
{
|
||||||
|
id: "inspect",
|
||||||
|
label: "Inspect findings",
|
||||||
|
icon: <Eye size={16} />,
|
||||||
|
onClick: () => setSelectedScanId(scan.id)
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "run",
|
||||||
|
label: scan.status === "failed" ? "Resume next batch" : "Run next batch",
|
||||||
|
icon: <Play size={16} />,
|
||||||
|
onClick: () => void runBatch(scan),
|
||||||
|
disabled: busy || ["completed", "cancelled"].includes(scan.status),
|
||||||
|
disabledReason: ["completed", "cancelled"].includes(scan.status)
|
||||||
|
? "This scan is complete."
|
||||||
|
: undefined
|
||||||
|
}
|
||||||
|
]}
|
||||||
|
/>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
], [busy]);
|
||||||
|
|
||||||
|
const findingColumns = useMemo<DataGridColumn<FileIntegrityFinding>[]>(() => [
|
||||||
|
{
|
||||||
|
id: "kind",
|
||||||
|
header: "Finding",
|
||||||
|
width: 180,
|
||||||
|
minWidth: 145,
|
||||||
|
sortable: true,
|
||||||
|
filterable: true,
|
||||||
|
value: (finding) => finding.kind.replaceAll("_", " ")
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "state",
|
||||||
|
header: "State",
|
||||||
|
width: 115,
|
||||||
|
minWidth: 95,
|
||||||
|
sortable: true,
|
||||||
|
filterable: true,
|
||||||
|
value: (finding) => finding.state,
|
||||||
|
render: (finding) => <StatusBadge status={statusTone(finding.state)} label={finding.state} />
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "object",
|
||||||
|
header: "Affected object",
|
||||||
|
minWidth: 260,
|
||||||
|
resizable: true,
|
||||||
|
filterable: true,
|
||||||
|
value: (finding) => finding.storage_key,
|
||||||
|
render: (finding) => (
|
||||||
|
<span title={finding.storage_key}>
|
||||||
|
{finding.blob_id ? `Blob ${finding.blob_id}` : finding.storage_key}
|
||||||
|
</span>
|
||||||
|
)
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "evidence",
|
||||||
|
header: "Expected / observed",
|
||||||
|
minWidth: 230,
|
||||||
|
resizable: true,
|
||||||
|
value: (finding) => evidenceLabel(finding)
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "updated",
|
||||||
|
header: "Updated",
|
||||||
|
width: 175,
|
||||||
|
minWidth: 145,
|
||||||
|
sortable: true,
|
||||||
|
value: (finding) => finding.updated_at,
|
||||||
|
render: (finding) => formatDateTime(finding.updated_at)
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "actions",
|
||||||
|
header: "Actions",
|
||||||
|
width: 70,
|
||||||
|
minWidth: 70,
|
||||||
|
sticky: "end",
|
||||||
|
align: "right",
|
||||||
|
render: (finding) => (
|
||||||
|
<TableActionGroup
|
||||||
|
minimumSlots={1}
|
||||||
|
actions={finding.kind === "orphan_object"
|
||||||
|
? [{
|
||||||
|
id: "cleanup",
|
||||||
|
label: "Preview safe cleanup",
|
||||||
|
icon: <Trash2 size={16} />,
|
||||||
|
variant: "danger",
|
||||||
|
onClick: () => void previewCleanup(finding),
|
||||||
|
disabled: busy || finding.state === "deleted",
|
||||||
|
disabledReason: finding.state === "deleted" ? "The object is already absent." : undefined
|
||||||
|
}]
|
||||||
|
: [{
|
||||||
|
id: "recheck",
|
||||||
|
label: "Recheck stored object",
|
||||||
|
icon: <RotateCw size={16} />,
|
||||||
|
onClick: () => void recheck(finding),
|
||||||
|
disabled: busy
|
||||||
|
}]}
|
||||||
|
/>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
], [busy]);
|
||||||
|
|
||||||
|
return (
|
||||||
|
<>
|
||||||
|
<AdminPageLayout
|
||||||
|
title="File integrity"
|
||||||
|
description="Run bounded storage reconciliation and resolve quarantined or unreferenced objects without acting on stale operator state."
|
||||||
|
loading={loading}
|
||||||
|
error={error}
|
||||||
|
success={success}
|
||||||
|
actions={(
|
||||||
|
<>
|
||||||
|
<Button
|
||||||
|
title="Reload scans and findings"
|
||||||
|
aria-label="Reload scans and findings"
|
||||||
|
onClick={() => void loadScans()}
|
||||||
|
disabled={loading || busy}
|
||||||
|
>
|
||||||
|
<RefreshCw size={16} />
|
||||||
|
</Button>
|
||||||
|
<Button variant="primary" onClick={() => setCreateOpen(true)} disabled={!canWrite || busy}>
|
||||||
|
<Plus size={16} /> New scan
|
||||||
|
</Button>
|
||||||
|
<DocumentationHelpLink reference={DOCUMENTATION} label="Open Files integrity documentation" />
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
>
|
||||||
|
<DismissibleAlert tone="info" dismissible={false} compact>
|
||||||
|
Cleanup is available only for objects with no managed database reference and always starts with a dry-run preview. Files does not yet implement legal-hold or hard-purge policy; such objects must remain outside cleanup until those controls exist.
|
||||||
|
</DismissibleAlert>
|
||||||
|
|
||||||
|
<Card title="Integrity scans">
|
||||||
|
<div className="admin-table-surface">
|
||||||
|
<DataGrid
|
||||||
|
id="files-integrity-scans-v1"
|
||||||
|
rows={scans}
|
||||||
|
columns={scanColumns}
|
||||||
|
getRowKey={(scan) => scan.id}
|
||||||
|
initialFit="container"
|
||||||
|
rowClassName={(scan) => scan.id === selectedScanId ? "is-selected" : undefined}
|
||||||
|
emptyText="No integrity scans have been created."
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
</Card>
|
||||||
|
|
||||||
|
{selectedScan && (
|
||||||
|
<>
|
||||||
|
<MetricGrid>
|
||||||
|
<MetricCard label="Verified blobs" value={selectedScan.verified_blob_count} tone="good" />
|
||||||
|
<MetricCard label="Quarantined blobs" value={selectedScan.quarantined_blob_count} tone={selectedScan.quarantined_blob_count ? "danger" : "neutral"} />
|
||||||
|
<MetricCard label="Orphan objects" value={selectedScan.orphan_object_count} tone={selectedScan.orphan_object_count ? "warning" : "neutral"} />
|
||||||
|
<MetricCard label="Open findings" value={findings.filter((finding) => finding.state === "open").length} tone={findings.some((finding) => finding.state === "open") ? "warning" : "good"} />
|
||||||
|
</MetricGrid>
|
||||||
|
{selectedScan.last_error && (
|
||||||
|
<DismissibleAlert tone="warning" dismissible={false} compact>
|
||||||
|
The last batch failed with {selectedScan.last_error}. Verify storage availability, then resume from the recorded cursor.
|
||||||
|
</DismissibleAlert>
|
||||||
|
)}
|
||||||
|
<Card title="Findings">
|
||||||
|
<div className="admin-table-surface">
|
||||||
|
<DataGrid
|
||||||
|
id={`files-integrity-findings-${selectedScan.id}`}
|
||||||
|
rows={findings}
|
||||||
|
columns={findingColumns}
|
||||||
|
getRowKey={(finding) => finding.id}
|
||||||
|
initialFit="container"
|
||||||
|
emptyText={selectedScan.status === "completed" ? "No findings were recorded." : "No findings in completed batches yet."}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
</Card>
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
</AdminPageLayout>
|
||||||
|
|
||||||
|
<Dialog
|
||||||
|
open={createOpen}
|
||||||
|
title="Create integrity scan"
|
||||||
|
onClose={() => setCreateOpen(false)}
|
||||||
|
closeDisabled={busy}
|
||||||
|
footer={(
|
||||||
|
<>
|
||||||
|
<Button onClick={() => setCreateOpen(false)} disabled={busy}>Cancel</Button>
|
||||||
|
<Button variant="primary" onClick={() => void createScan()} disabled={busy}>
|
||||||
|
<CheckCircle2 size={16} /> {busy ? "Creating..." : "Create scan"}
|
||||||
|
</Button>
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
>
|
||||||
|
<div className="settings-list">
|
||||||
|
<ToggleSwitch
|
||||||
|
label="Verify SHA-256 checksums"
|
||||||
|
checked={verifyChecksums}
|
||||||
|
onChange={setVerifyChecksums}
|
||||||
|
disabled={busy}
|
||||||
|
help="Checksum verification reads every selected object; disabling it verifies existence and size only."
|
||||||
|
/>
|
||||||
|
<FormField label="Items per batch" documentation={DOCUMENTATION}>
|
||||||
|
<input
|
||||||
|
type="number"
|
||||||
|
min={1}
|
||||||
|
max={1000}
|
||||||
|
value={batchSize}
|
||||||
|
onChange={(event) => setBatchSize(Math.max(1, Math.min(1000, Number(event.target.value) || 1)))}
|
||||||
|
disabled={busy}
|
||||||
|
/>
|
||||||
|
</FormField>
|
||||||
|
</div>
|
||||||
|
</Dialog>
|
||||||
|
|
||||||
|
<ConfirmDialog
|
||||||
|
open={cleanupPreview?.action === "would_delete"}
|
||||||
|
title="Delete unreferenced storage object?"
|
||||||
|
message={cleanupPreview ? `The dry run confirmed that ${cleanupPreview.finding.storage_key} has no managed Files reference. Deletion cannot be undone from GovOPlaN.` : ""}
|
||||||
|
confirmLabel="Delete object"
|
||||||
|
tone="danger"
|
||||||
|
busy={busy}
|
||||||
|
onConfirm={() => void confirmCleanup()}
|
||||||
|
onCancel={() => setCleanupPreview(null)}
|
||||||
|
/>
|
||||||
|
</>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function statusTone(value: string): string {
|
||||||
|
if (["completed", "resolved", "verified"].includes(value)) return "success";
|
||||||
|
if (["failed", "deleted", "missing", "checksum_mismatch", "size_mismatch"].includes(value)) return "danger";
|
||||||
|
if (["running", "pending", "open"].includes(value)) return "warning";
|
||||||
|
return "neutral";
|
||||||
|
}
|
||||||
|
|
||||||
|
function evidenceLabel(finding: FileIntegrityFinding): string {
|
||||||
|
const expected = finding.expected_size_bytes == null ? "-" : `${finding.expected_size_bytes} B`;
|
||||||
|
const observed = finding.observed_size_bytes == null ? "-" : `${finding.observed_size_bytes} B`;
|
||||||
|
return `${expected} / ${observed}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function cleanupActionMessage(action: string): string {
|
||||||
|
if (action === "retained_referenced") return "Cleanup was blocked because the object has a managed database reference.";
|
||||||
|
if (action === "already_deleted" || action === "already_absent") return "The object was already absent; the finding is reconciled.";
|
||||||
|
if (action === "deleted") return "The unreferenced storage object was deleted and recovery evidence was recorded.";
|
||||||
|
return `Integrity cleanup result: ${action.replaceAll("_", " ")}.`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatDateTime(value: string | null | undefined): string {
|
||||||
|
if (!value) return "-";
|
||||||
|
const parsed = new Date(value);
|
||||||
|
return Number.isNaN(parsed.getTime()) ? value : parsed.toLocaleString();
|
||||||
|
}
|
||||||
@@ -0,0 +1,97 @@
|
|||||||
|
import { FileText, UploadCloud, X } from "lucide-react";
|
||||||
|
import { useCallback } from "react";
|
||||||
|
import { Link } from "react-router";
|
||||||
|
import {
|
||||||
|
Button,
|
||||||
|
DismissibleAlert,
|
||||||
|
LoadingFrame,
|
||||||
|
SelectionList,
|
||||||
|
SelectionListItem,
|
||||||
|
SelectionListItemContent,
|
||||||
|
hasScope,
|
||||||
|
quickAccessLaunchState,
|
||||||
|
useDashboardWidgetData,
|
||||||
|
type QuickAccessToolRenderContext
|
||||||
|
} from "@govoplan/core-webui";
|
||||||
|
import { listFiles } from "../../api/files";
|
||||||
|
|
||||||
|
type Props = Pick<
|
||||||
|
QuickAccessToolRenderContext,
|
||||||
|
"settings" | "auth" | "launchContext" | "complete" | "cancel" | "close"
|
||||||
|
>;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Bounded file selection. listFiles remains the owner-side authorization
|
||||||
|
* check; the host receives only a versioned reference after explicit choice.
|
||||||
|
*/
|
||||||
|
export default function FileQuickAccess({
|
||||||
|
settings,
|
||||||
|
auth,
|
||||||
|
launchContext,
|
||||||
|
complete,
|
||||||
|
cancel,
|
||||||
|
close
|
||||||
|
}: Props) {
|
||||||
|
const load = useCallback(
|
||||||
|
async () => (await listFiles(settings, { sort: "recent", page_size: 7 })).files,
|
||||||
|
[settings]
|
||||||
|
);
|
||||||
|
const { data: files, loading, error } = useDashboardWidgetData(load, 0);
|
||||||
|
const canUpload = hasScope(auth, "files:upload");
|
||||||
|
|
||||||
|
return (
|
||||||
|
<LoadingFrame loading={loading} label="Loading recent files">
|
||||||
|
{launchContext.activeObject ? (
|
||||||
|
<p className="muted small-note">
|
||||||
|
Select a file for {launchContext.activeObject.label}. Files checks
|
||||||
|
your current access again before showing this list.
|
||||||
|
</p>
|
||||||
|
) : null}
|
||||||
|
{error ? <DismissibleAlert tone="warning" resetKey={error}>{error}</DismissibleAlert> : null}
|
||||||
|
<SelectionList variant="navigation" label="Recent authorized files">
|
||||||
|
{(files ?? []).map((file) => (
|
||||||
|
<SelectionListItem
|
||||||
|
key={file.version_id}
|
||||||
|
selected={false}
|
||||||
|
onClick={() => complete({
|
||||||
|
contractVersion: "1",
|
||||||
|
outcome: "completed",
|
||||||
|
action: "selected",
|
||||||
|
reference: {
|
||||||
|
ownerModule: "files",
|
||||||
|
kind: "file-version",
|
||||||
|
objectId: file.version_id,
|
||||||
|
tenantId: file.tenant_id,
|
||||||
|
label: file.filename,
|
||||||
|
version: file.version_id,
|
||||||
|
path: `/files?fileId=${encodeURIComponent(file.id)}&versionId=${encodeURIComponent(file.version_id)}`
|
||||||
|
}
|
||||||
|
})}
|
||||||
|
>
|
||||||
|
<SelectionListItemContent
|
||||||
|
leading={<FileText size={16} aria-hidden="true" />}
|
||||||
|
title={file.filename}
|
||||||
|
description={file.display_path}
|
||||||
|
/>
|
||||||
|
</SelectionListItem>
|
||||||
|
))}
|
||||||
|
</SelectionList>
|
||||||
|
{!loading && !error && !(files ?? []).length ? (
|
||||||
|
<p className="muted">No authorized files are available.</p>
|
||||||
|
) : null}
|
||||||
|
<div className="button-row compact-actions">
|
||||||
|
{canUpload ? (
|
||||||
|
<Link
|
||||||
|
className="btn btn-secondary"
|
||||||
|
to="/files?quickAction=upload"
|
||||||
|
state={quickAccessLaunchState(launchContext)}
|
||||||
|
onClick={close}
|
||||||
|
>
|
||||||
|
<UploadCloud size={15} aria-hidden="true" /> i18n:govoplan-files.upload.8bdf057f
|
||||||
|
</Link>
|
||||||
|
) : null}
|
||||||
|
<Button onClick={() => cancel("user")}><X size={15} aria-hidden="true" /> i18n:govoplan-files.cancel.77dfd213</Button>
|
||||||
|
</div>
|
||||||
|
</LoadingFrame>
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
import { useEffect, useMemo, useRef, useState, type DragEvent as ReactDragEvent, type KeyboardEvent as ReactKeyboardEvent, type MouseEvent as ReactMouseEvent } from "react";
|
import { useEffect, useMemo, useRef, useState, type DragEvent as ReactDragEvent, type KeyboardEvent as ReactKeyboardEvent, type MouseEvent as ReactMouseEvent } from "react";
|
||||||
import { ArrowUp, ChevronRight, Copy, Download, File, Folder, Home, KeyRound, Link2, ListFilter, MoveRight, Plus, RefreshCw, Search, Share2, Trash2, UploadCloud } from "lucide-react";
|
import { ArrowUp, ChevronRight, Copy, Download, File, Folder, Home, KeyRound, Link2, ListFilter, MoveRight, Plus, RefreshCw, Search, Share2, Trash2, UploadCloud } from "lucide-react";
|
||||||
import {
|
import { FormGrid, ActionToolbar,
|
||||||
Button,
|
Button,
|
||||||
ConfirmDialog,
|
ConfirmDialog,
|
||||||
DocumentationHelpLink,
|
DocumentationHelpLink,
|
||||||
@@ -16,6 +16,7 @@ import {
|
|||||||
type ApiSettings,
|
type ApiSettings,
|
||||||
type AuthInfo, i18nMessage } from
|
type AuthInfo, i18nMessage } from
|
||||||
"@govoplan/core-webui";
|
"@govoplan/core-webui";
|
||||||
|
import { useLocation, useNavigate } from "react-router";
|
||||||
import {
|
import {
|
||||||
bulkDeleteFiles,
|
bulkDeleteFiles,
|
||||||
bulkRenameFiles,
|
bulkRenameFiles,
|
||||||
@@ -24,9 +25,11 @@ import {
|
|||||||
createFolder,
|
createFolder,
|
||||||
confirmArchiveUpload,
|
confirmArchiveUpload,
|
||||||
deleteFolder,
|
deleteFolder,
|
||||||
|
deleteFileConnectorSpace,
|
||||||
downloadFile,
|
downloadFile,
|
||||||
downloadFilesAsZip,
|
downloadFilesAsZip,
|
||||||
fetchResourceAccessExplanation,
|
fetchResourceAccessExplanation,
|
||||||
|
fetchResourceAccessExplanationSubjects,
|
||||||
listFilesDelta,
|
listFilesDelta,
|
||||||
listFilesByProperties,
|
listFilesByProperties,
|
||||||
listFileConnectorProfiles,
|
listFileConnectorProfiles,
|
||||||
@@ -50,7 +53,8 @@ import {
|
|||||||
type FileSpace,
|
type FileSpace,
|
||||||
type ManagedFile,
|
type ManagedFile,
|
||||||
type RenameResponse,
|
type RenameResponse,
|
||||||
type ResourceAccessExplanationResponse } from
|
type ResourceAccessExplanationResponse,
|
||||||
|
type ResourceAccessExplanationSubjectsResponse } from
|
||||||
"../../api/files";
|
"../../api/files";
|
||||||
import { EMPTY_FILES, EMPTY_FOLDERS, EMPTY_SPACES, INTERNAL_DRAG_TYPE } from "./constants";
|
import { EMPTY_FILES, EMPTY_FOLDERS, EMPTY_SPACES, INTERNAL_DRAG_TYPE } from "./constants";
|
||||||
import { FileConflictDialog, FileContextMenu, FileDialog, FolderTree, RenamePreviewList, TransferFolderSelector } from "./components/FileManagerComponents";
|
import { FileConflictDialog, FileContextMenu, FileDialog, FolderTree, RenamePreviewList, TransferFolderSelector } from "./components/FileManagerComponents";
|
||||||
@@ -114,12 +118,15 @@ const FILES_WORKFLOW_DOCUMENTATION = {
|
|||||||
} as const;
|
} as const;
|
||||||
|
|
||||||
export default function FilesPage({ settings, auth }: {settings: ApiSettings;auth: AuthInfo;}) {
|
export default function FilesPage({ settings, auth }: {settings: ApiSettings;auth: AuthInfo;}) {
|
||||||
|
const location = useLocation();
|
||||||
|
const navigate = useNavigate();
|
||||||
const canDownload = hasScope(auth, "files:download");
|
const canDownload = hasScope(auth, "files:download");
|
||||||
const canUpload = hasScope(auth, "files:upload");
|
const canUpload = hasScope(auth, "files:upload");
|
||||||
const canOrganize = hasScope(auth, "files:organize");
|
const canOrganize = hasScope(auth, "files:organize");
|
||||||
const canDelete = hasScope(auth, "files:delete");
|
const canDelete = hasScope(auth, "files:delete");
|
||||||
const canShare = hasScope(auth, "files:file:share");
|
const canShare = hasScope(auth, "files:file:share");
|
||||||
const [spaces, setSpaces] = useState<FileSpace[]>(EMPTY_SPACES);
|
const [spaces, setSpaces] = useState<FileSpace[]>(EMPTY_SPACES);
|
||||||
|
const [spacesLoaded, setSpacesLoaded] = useState(false);
|
||||||
const [activeSpaceId, setActiveSpaceId] = useState("");
|
const [activeSpaceId, setActiveSpaceId] = useState("");
|
||||||
const activeSpace = spaces.find((space) => space.id === activeSpaceId) ?? spaces[0] ?? null;
|
const activeSpace = spaces.find((space) => space.id === activeSpaceId) ?? spaces[0] ?? null;
|
||||||
const activeSpaceIsConnector = activeSpace?.space_type === "connector";
|
const activeSpaceIsConnector = activeSpace?.space_type === "connector";
|
||||||
@@ -160,16 +167,20 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut
|
|||||||
const [connectorLoading, setConnectorLoading] = useState(false);
|
const [connectorLoading, setConnectorLoading] = useState(false);
|
||||||
const [connectorError, setConnectorError] = useState("");
|
const [connectorError, setConnectorError] = useState("");
|
||||||
const [connectorSpaceLabel, setConnectorSpaceLabel] = useState("");
|
const [connectorSpaceLabel, setConnectorSpaceLabel] = useState("");
|
||||||
|
const [connectorSpaceReadOnly, setConnectorSpaceReadOnly] = useState(true);
|
||||||
const [connectorSpaceOwnerSpaceId, setConnectorSpaceOwnerSpaceId] = useState("");
|
const [connectorSpaceOwnerSpaceId, setConnectorSpaceOwnerSpaceId] = useState("");
|
||||||
const [connectorSpaceItemsBySpace, setConnectorSpaceItemsBySpace] = useState<Record<string, FileConnectorBrowseItem[]>>({});
|
const [connectorSpaceItemsBySpace, setConnectorSpaceItemsBySpace] = useState<Record<string, FileConnectorBrowseItem[]>>({});
|
||||||
const [connectorSpaceLibraryBySpace, setConnectorSpaceLibraryBySpace] = useState<Record<string, string | null>>({});
|
const [connectorSpaceLibraryBySpace, setConnectorSpaceLibraryBySpace] = useState<Record<string, string | null>>({});
|
||||||
const [connectorSpaceSelectedItem, setConnectorSpaceSelectedItem] = useState<FileConnectorBrowseItem | null>(null);
|
const [connectorSpaceSelectedItem, setConnectorSpaceSelectedItem] = useState<FileConnectorBrowseItem | null>(null);
|
||||||
|
const [connectorSpaceRemovalTarget, setConnectorSpaceRemovalTarget] = useState<FileSpace | null>(null);
|
||||||
const [connectorSpaceLoading, setConnectorSpaceLoading] = useState(false);
|
const [connectorSpaceLoading, setConnectorSpaceLoading] = useState(false);
|
||||||
const [connectorSpaceError, setConnectorSpaceError] = useState("");
|
const [connectorSpaceError, setConnectorSpaceError] = useState("");
|
||||||
const [message, setMessage] = useState("");
|
const [message, setMessage] = useState("");
|
||||||
const [error, setError] = useState("");
|
const [error, setError] = useState("");
|
||||||
const [accessExplanationTarget, setAccessExplanationTarget] = useState<FileAccessExplanationTarget | null>(null);
|
const [accessExplanationTarget, setAccessExplanationTarget] = useState<FileAccessExplanationTarget | null>(null);
|
||||||
const [resourceAccessExplanation, setResourceAccessExplanation] = useState<ResourceAccessExplanationResponse | null>(null);
|
const [resourceAccessExplanation, setResourceAccessExplanation] = useState<ResourceAccessExplanationResponse | null>(null);
|
||||||
|
const [resourceAccessSubjects, setResourceAccessSubjects] = useState<ResourceAccessExplanationSubjectsResponse | null>(null);
|
||||||
|
const [resourceAccessSubjectId, setResourceAccessSubjectId] = useState("");
|
||||||
const [resourceAccessLoading, setResourceAccessLoading] = useState(false);
|
const [resourceAccessLoading, setResourceAccessLoading] = useState(false);
|
||||||
const [shareDialogFile, setShareDialogFile] = useState<ManagedFile | null>(null);
|
const [shareDialogFile, setShareDialogFile] = useState<ManagedFile | null>(null);
|
||||||
const { setDragActive, internalDrag, setInternalDrag, dropTargetKey, setDropTargetKey, clearDropState: clearDragDropState } = useFileDragDropState();
|
const { setDragActive, internalDrag, setInternalDrag, dropTargetKey, setDropTargetKey, clearDropState: clearDragDropState } = useFileDragDropState();
|
||||||
@@ -330,6 +341,7 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut
|
|||||||
}, [activeSpaceId, currentFolder, propertyFiltersActive, propertyFilterResults, searchActive, searchResults, sortColumn, sortDirection]);
|
}, [activeSpaceId, currentFolder, propertyFiltersActive, propertyFilterResults, searchActive, searchResults, sortColumn, sortDirection]);
|
||||||
|
|
||||||
async function loadSpaces() {
|
async function loadSpaces() {
|
||||||
|
setSpacesLoaded(false);
|
||||||
setBusy(true);
|
setBusy(true);
|
||||||
setError("");
|
setError("");
|
||||||
try {
|
try {
|
||||||
@@ -341,6 +353,7 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut
|
|||||||
} catch (err) {
|
} catch (err) {
|
||||||
setError(err instanceof Error ? err.message : String(err));
|
setError(err instanceof Error ? err.message : String(err));
|
||||||
} finally {
|
} finally {
|
||||||
|
setSpacesLoaded(true);
|
||||||
setBusy(false);
|
setBusy(false);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -434,6 +447,40 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut
|
|||||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||||
}, [settings.apiBaseUrl, settings.apiKey, settings.accessToken]);
|
}, [settings.apiBaseUrl, settings.apiKey, settings.accessToken]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
const parameters = new URLSearchParams(location.search);
|
||||||
|
if (parameters.get("quickAction") !== "upload" || !spacesLoaded) return;
|
||||||
|
|
||||||
|
const uploadSpace =
|
||||||
|
activeSpace && !isConnectorSpace(activeSpace)
|
||||||
|
? activeSpace
|
||||||
|
: spaces.find((space) => !isConnectorSpace(space)) ?? null;
|
||||||
|
if (canUpload && uploadSpace) {
|
||||||
|
setActiveSpaceId(uploadSpace.id);
|
||||||
|
openDialog("upload", { spaceId: uploadSpace.id, folderPath: "" });
|
||||||
|
} else if (!canUpload) {
|
||||||
|
setError("File upload permission is required.");
|
||||||
|
} else {
|
||||||
|
setError("No authorized managed file space is available for upload.");
|
||||||
|
}
|
||||||
|
|
||||||
|
parameters.delete("quickAction");
|
||||||
|
const search = parameters.toString();
|
||||||
|
navigate(
|
||||||
|
{ pathname: location.pathname, search: search ? `?${search}` : "" },
|
||||||
|
{ replace: true, state: location.state }
|
||||||
|
);
|
||||||
|
}, [
|
||||||
|
activeSpace,
|
||||||
|
canUpload,
|
||||||
|
location.pathname,
|
||||||
|
location.search,
|
||||||
|
location.state,
|
||||||
|
navigate,
|
||||||
|
spaces,
|
||||||
|
spacesLoaded
|
||||||
|
]);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (!contextMenu) return undefined;
|
if (!contextMenu) return undefined;
|
||||||
const close = () => setContextMenu(null);
|
const close = () => setContextMenu(null);
|
||||||
@@ -538,6 +585,7 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut
|
|||||||
setConnectorError("");
|
setConnectorError("");
|
||||||
setConnectorSelectedItem(null);
|
setConnectorSelectedItem(null);
|
||||||
setConnectorSpaceLabel("");
|
setConnectorSpaceLabel("");
|
||||||
|
setConnectorSpaceReadOnly(true);
|
||||||
resetArchiveUploadState();
|
resetArchiveUploadState();
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1057,6 +1105,7 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut
|
|||||||
library_id: connectorLibraryId || undefined,
|
library_id: connectorLibraryId || undefined,
|
||||||
remote_path: remotePath,
|
remote_path: remotePath,
|
||||||
sync_mode: "manual",
|
sync_mode: "manual",
|
||||||
|
read_only: connectorSpaceReadOnly,
|
||||||
metadata: {
|
metadata: {
|
||||||
created_from_ui: true,
|
created_from_ui: true,
|
||||||
source_label: activeConnectorProfile?.label,
|
source_label: activeConnectorProfile?.label,
|
||||||
@@ -1079,6 +1128,35 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function removeConnectorSpace() {
|
||||||
|
const target = connectorSpaceRemovalTarget;
|
||||||
|
if (!canOrganize || !target?.connector_space_id || !isConnectorSpace(target)) return;
|
||||||
|
setBusy(true);
|
||||||
|
setError("");
|
||||||
|
setMessage("");
|
||||||
|
try {
|
||||||
|
await deleteFileConnectorSpace(settings, target.connector_space_id);
|
||||||
|
setConnectorSpaceRemovalTarget(null);
|
||||||
|
setConnectorSpaceItemsBySpace((current) => {
|
||||||
|
const next = { ...current };
|
||||||
|
delete next[target.id];
|
||||||
|
return next;
|
||||||
|
});
|
||||||
|
setConnectorSpaceLibraryBySpace((current) => {
|
||||||
|
const next = { ...current };
|
||||||
|
delete next[target.id];
|
||||||
|
return next;
|
||||||
|
});
|
||||||
|
setConnectorSpaceSelectedItem(null);
|
||||||
|
setMessage(`Removed connector space “${target.label}”. Remote content and managed GovOPlaN files were not changed.`);
|
||||||
|
await loadSpaces();
|
||||||
|
} catch (err) {
|
||||||
|
setError(err instanceof Error ? err.message : String(err));
|
||||||
|
} finally {
|
||||||
|
setBusy(false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
function connectorMetadataString(item: FileConnectorBrowseItem, key: string): string | null {
|
function connectorMetadataString(item: FileConnectorBrowseItem, key: string): string | null {
|
||||||
const value = item.metadata[key];
|
const value = item.metadata[key];
|
||||||
if (typeof value === "string" && value.trim()) return value;
|
if (typeof value === "string" && value.trim()) return value;
|
||||||
@@ -1840,14 +1918,25 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function openAccessExplanation(target: FileAccessExplanationTarget): Promise<void> {
|
async function openAccessExplanation(target: FileAccessExplanationTarget): Promise<void> {
|
||||||
if (!auth.user?.id) return;
|
const currentUserId = auth.principal?.membership_id || auth.user?.id;
|
||||||
|
if (!currentUserId) return;
|
||||||
setAccessExplanationTarget(target);
|
setAccessExplanationTarget(target);
|
||||||
setResourceAccessExplanation(null);
|
setResourceAccessExplanation(null);
|
||||||
|
setResourceAccessSubjects(null);
|
||||||
setResourceAccessLoading(true);
|
setResourceAccessLoading(true);
|
||||||
setError("");
|
setError("");
|
||||||
try {
|
try {
|
||||||
|
const subjects = await fetchResourceAccessExplanationSubjects(settings, {
|
||||||
|
tenantId: (auth.active_tenant ?? auth.tenant)?.id
|
||||||
|
});
|
||||||
|
const initialSubjectId = subjects.users.some((user) => user.id === currentUserId)
|
||||||
|
? currentUserId
|
||||||
|
: subjects.users[0]?.id;
|
||||||
|
if (!initialSubjectId) throw new Error("No permitted access-explanation subject is available.");
|
||||||
|
setResourceAccessSubjects(subjects);
|
||||||
|
setResourceAccessSubjectId(initialSubjectId);
|
||||||
setResourceAccessExplanation(await fetchResourceAccessExplanation(settings, {
|
setResourceAccessExplanation(await fetchResourceAccessExplanation(settings, {
|
||||||
userId: auth.user.id,
|
userId: initialSubjectId,
|
||||||
resourceType: target.resourceType,
|
resourceType: target.resourceType,
|
||||||
resourceId: target.resourceId,
|
resourceId: target.resourceId,
|
||||||
action: target.action,
|
action: target.action,
|
||||||
@@ -1861,6 +1950,27 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function selectAccessExplanationSubject(userId: string): Promise<void> {
|
||||||
|
if (!accessExplanationTarget || userId === resourceAccessSubjectId) return;
|
||||||
|
setResourceAccessLoading(true);
|
||||||
|
setError("");
|
||||||
|
try {
|
||||||
|
const explanation = await fetchResourceAccessExplanation(settings, {
|
||||||
|
userId,
|
||||||
|
resourceType: accessExplanationTarget.resourceType,
|
||||||
|
resourceId: accessExplanationTarget.resourceId,
|
||||||
|
action: accessExplanationTarget.action,
|
||||||
|
tenantId: (auth.active_tenant ?? auth.tenant)?.id
|
||||||
|
});
|
||||||
|
setResourceAccessExplanation(explanation);
|
||||||
|
setResourceAccessSubjectId(userId);
|
||||||
|
} catch (err) {
|
||||||
|
setError(err instanceof Error ? err.message : String(err));
|
||||||
|
} finally {
|
||||||
|
setResourceAccessLoading(false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
function openTransferDialogForContext(menu: ContextMenuState | null, mode: TransferMode) {
|
function openTransferDialogForContext(menu: ContextMenuState | null, mode: TransferMode) {
|
||||||
const sets = selectedSetsForContext(menu);
|
const sets = selectedSetsForContext(menu);
|
||||||
const space = spaceForContext(menu);
|
const space = spaceForContext(menu);
|
||||||
@@ -2089,7 +2199,7 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut
|
|||||||
const syncBlocker = workingBlocker || (!activeSpace ? "Select a file space first." : "") || (!canUpload ? "File upload permission is required to import synchronized content." : "") || (activeSpaceIsConnector && connectorSpaceSelectedItem?.kind !== "file" ? "Select one remote file to synchronize." : "");
|
const syncBlocker = workingBlocker || (!activeSpace ? "Select a file space first." : "") || (!canUpload ? "File upload permission is required to import synchronized content." : "") || (activeSpaceIsConnector && connectorSpaceSelectedItem?.kind !== "file" ? "Select one remote file to synchronize." : "");
|
||||||
|
|
||||||
const toolbar =
|
const toolbar =
|
||||||
<div className="file-manager-toolbar" aria-label="i18n:govoplan-files.file_actions.9e1b94c5">
|
<ActionToolbar className="file-manager-toolbar" aria-label="i18n:govoplan-files.file_actions.9e1b94c5">
|
||||||
<Button variant="primary" onClick={() => openDialog("upload", toolbarTarget())} disabled={Boolean(uploadBlocker)} disabledReason={uploadBlocker}><UploadCloud size={16} aria-hidden="true" /> i18n:govoplan-files.upload.8bdf057f</Button>
|
<Button variant="primary" onClick={() => openDialog("upload", toolbarTarget())} disabled={Boolean(uploadBlocker)} disabledReason={uploadBlocker}><UploadCloud size={16} aria-hidden="true" /> i18n:govoplan-files.upload.8bdf057f</Button>
|
||||||
<Button
|
<Button
|
||||||
onClick={() => activeSpaceIsConnector ? void syncConnectorSpaceSelection() : void openConnectorSyncDialog(toolbarTarget())}
|
onClick={() => activeSpaceIsConnector ? void syncConnectorSpaceSelection() : void openConnectorSyncDialog(toolbarTarget())}
|
||||||
@@ -2110,12 +2220,21 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut
|
|||||||
<Button onClick={() => accessExplainableTarget && void openAccessExplanation(accessExplainableTarget)} disabled={Boolean(accessExplanationBlocker)} disabledReason={accessExplanationBlocker}><KeyRound size={16} aria-hidden="true" /> i18n:govoplan-files.explain_access.4d5fac37</Button>
|
<Button onClick={() => accessExplainableTarget && void openAccessExplanation(accessExplainableTarget)} disabled={Boolean(accessExplanationBlocker)} disabledReason={accessExplanationBlocker}><KeyRound size={16} aria-hidden="true" /> i18n:govoplan-files.explain_access.4d5fac37</Button>
|
||||||
<Button variant="danger" onClick={() => void deleteSelected()} disabled={Boolean(deleteBlocker)} disabledReason={deleteBlocker}><Trash2 size={16} aria-hidden="true" /> i18n:govoplan-files.delete.f6fdbe48</Button>
|
<Button variant="danger" onClick={() => void deleteSelected()} disabled={Boolean(deleteBlocker)} disabledReason={deleteBlocker}><Trash2 size={16} aria-hidden="true" /> i18n:govoplan-files.delete.f6fdbe48</Button>
|
||||||
{activeSpaceIsConnector &&
|
{activeSpaceIsConnector &&
|
||||||
|
<Button
|
||||||
|
variant="danger"
|
||||||
|
onClick={() => activeSpace && setConnectorSpaceRemovalTarget(activeSpace)}
|
||||||
|
disabled={busy || !canOrganize || !activeSpace?.connector_space_id}
|
||||||
|
disabledReason={workingBlocker || (!canOrganize ? "File organization permission is required to remove a connector space." : !activeSpace?.connector_space_id ? "This space is not a removable connector space." : "")}>
|
||||||
|
<Trash2 size={16} aria-hidden="true" /> Remove space
|
||||||
|
</Button>
|
||||||
|
}
|
||||||
|
{activeSpaceIsConnector &&
|
||||||
<Button onClick={() => activeSpace && void loadConnectorSpaceContents(activeSpace)} disabled={busy || connectorSpaceLoading || !activeSpace} disabledReason={workingBlocker || (connectorSpaceLoading ? "This connector space is already refreshing." : !activeSpace ? "Select a connector space first." : "")}>
|
<Button onClick={() => activeSpace && void loadConnectorSpaceContents(activeSpace)} disabled={busy || connectorSpaceLoading || !activeSpace} disabledReason={workingBlocker || (connectorSpaceLoading ? "This connector space is already refreshing." : !activeSpace ? "Select a connector space first." : "")}>
|
||||||
<RefreshCw size={16} aria-hidden="true" /> i18n:govoplan-files.refresh.56e3badc
|
<RefreshCw size={16} aria-hidden="true" /> i18n:govoplan-files.refresh.56e3badc
|
||||||
</Button>
|
</Button>
|
||||||
}
|
}
|
||||||
<DocumentationHelpLink reference={FILES_WORKFLOW_DOCUMENTATION} />
|
<DocumentationHelpLink reference={FILES_WORKFLOW_DOCUMENTATION} />
|
||||||
</div>;
|
</ActionToolbar>;
|
||||||
|
|
||||||
|
|
||||||
const uploadBusyLabel = uploadPhase === "unpacking" ? "i18n:govoplan-files.unpacking_zip_archive.698095f4" : "i18n:govoplan-files.uploading_files.6536791d";
|
const uploadBusyLabel = uploadPhase === "unpacking" ? "i18n:govoplan-files.unpacking_zip_archive.698095f4" : "i18n:govoplan-files.uploading_files.6536791d";
|
||||||
@@ -2209,11 +2328,11 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut
|
|||||||
return (
|
return (
|
||||||
<div className="connector-space-panel">
|
<div className="connector-space-panel">
|
||||||
<div className="connector-browser connector-space-browser">
|
<div className="connector-browser connector-space-browser">
|
||||||
<div className="connector-browser-toolbar">
|
<ActionToolbar className="connector-browser-toolbar">
|
||||||
<Button onClick={browseConnectorSpaceParent} disabled={connectorSpaceParentDisabled}><ArrowUp size={15} aria-hidden="true" /> i18n:govoplan-files.up.2038bdec</Button>
|
<Button onClick={browseConnectorSpaceParent} disabled={connectorSpaceParentDisabled}><ArrowUp size={15} aria-hidden="true" /> i18n:govoplan-files.up.2038bdec</Button>
|
||||||
<span className="connector-browser-path" title={connectorSpaceLocationLabel}>{connectorSpaceLocationLabel}</span>
|
<span className="connector-browser-path" title={connectorSpaceLocationLabel}>{connectorSpaceLocationLabel}</span>
|
||||||
<Button onClick={() => void loadConnectorSpaceContents(activeSpace)} disabled={busy || connectorSpaceLoading}><RefreshCw size={15} aria-hidden="true" /> i18n:govoplan-files.refresh.56e3badc</Button>
|
<Button onClick={() => void loadConnectorSpaceContents(activeSpace)} disabled={busy || connectorSpaceLoading}><RefreshCw size={15} aria-hidden="true" /> i18n:govoplan-files.refresh.56e3badc</Button>
|
||||||
</div>
|
</ActionToolbar>
|
||||||
{connectorSpaceError && <p className="field-error connector-browser-error">{connectorSpaceError}</p>}
|
{connectorSpaceError && <p className="field-error connector-browser-error">{connectorSpaceError}</p>}
|
||||||
<div className="connector-browser-list" role="list" aria-label="i18n:govoplan-files.connector_space_files.dbb0ab24">
|
<div className="connector-browser-list" role="list" aria-label="i18n:govoplan-files.connector_space_files.dbb0ab24">
|
||||||
{connectorSpaceLoading &&
|
{connectorSpaceLoading &&
|
||||||
@@ -2481,6 +2600,17 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut
|
|||||||
onConfirm={() => void performConfirmedDelete()}
|
onConfirm={() => void performConfirmedDelete()}
|
||||||
onCancel={() => setDeleteDialog(null)} />
|
onCancel={() => setDeleteDialog(null)} />
|
||||||
|
|
||||||
|
<ConfirmDialog
|
||||||
|
open={Boolean(connectorSpaceRemovalTarget)}
|
||||||
|
title="Remove connector space?"
|
||||||
|
message={connectorSpaceRemovalTarget ? `Remove “${connectorSpaceRemovalTarget.label}” from Files? This retires only the local connector-space link. Remote files and folders remain at the provider. Imported GovOPlaN files, metadata and shares remain managed in their owner spaces. The connector profile, credentials and remote references are not deleted and can be linked again.` : ""}
|
||||||
|
confirmLabel="Remove space"
|
||||||
|
cancelLabel="i18n:govoplan-files.cancel.77dfd213"
|
||||||
|
tone="danger"
|
||||||
|
busy={busy}
|
||||||
|
onConfirm={() => void removeConnectorSpace()}
|
||||||
|
onCancel={() => setConnectorSpaceRemovalTarget(null)} />
|
||||||
|
|
||||||
|
|
||||||
{contextMenu &&
|
{contextMenu &&
|
||||||
<FileContextMenu
|
<FileContextMenu
|
||||||
@@ -2504,13 +2634,16 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut
|
|||||||
}
|
}
|
||||||
|
|
||||||
{accessExplanationTarget &&
|
{accessExplanationTarget &&
|
||||||
<FileDialog title="i18n:govoplan-core.access_explanation.75ee7f62" onClose={() => {if (!resourceAccessLoading) {setAccessExplanationTarget(null);setResourceAccessExplanation(null);}}}>
|
<FileDialog title="i18n:govoplan-core.access_explanation.75ee7f62" onClose={() => {if (!resourceAccessLoading) {setAccessExplanationTarget(null);setResourceAccessExplanation(null);setResourceAccessSubjects(null);setResourceAccessSubjectId("");}}}>
|
||||||
<ResourceAccessExplanation
|
<ResourceAccessExplanation
|
||||||
loading={resourceAccessLoading}
|
loading={resourceAccessLoading}
|
||||||
explanation={resourceAccessExplanation}
|
explanation={resourceAccessExplanation}
|
||||||
|
subjects={resourceAccessSubjects}
|
||||||
|
selectedUserId={resourceAccessSubjectId}
|
||||||
|
onSelectedUserIdChange={(userId) => void selectAccessExplanationSubject(userId)}
|
||||||
fallbackResourceLabel={accessExplanationTarget.label} />
|
fallbackResourceLabel={accessExplanationTarget.label} />
|
||||||
<div className="button-row compact-actions align-end">
|
<div className="button-row compact-actions align-end">
|
||||||
<Button onClick={() => {setAccessExplanationTarget(null);setResourceAccessExplanation(null);}} disabled={resourceAccessLoading}>i18n:govoplan-files.close.bbfa773e</Button>
|
<Button onClick={() => {setAccessExplanationTarget(null);setResourceAccessExplanation(null);setResourceAccessSubjects(null);setResourceAccessSubjectId("");}} disabled={resourceAccessLoading}>i18n:govoplan-files.close.bbfa773e</Button>
|
||||||
</div>
|
</div>
|
||||||
</FileDialog>
|
</FileDialog>
|
||||||
}
|
}
|
||||||
@@ -2595,7 +2728,7 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut
|
|||||||
</FormField>
|
</FormField>
|
||||||
}
|
}
|
||||||
|
|
||||||
<div className="archive-selection-toolbar">
|
<ActionToolbar justify="between" className="archive-selection-toolbar">
|
||||||
<label>
|
<label>
|
||||||
<input
|
<input
|
||||||
type="checkbox"
|
type="checkbox"
|
||||||
@@ -2606,7 +2739,7 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut
|
|||||||
<span>Select all files</span>
|
<span>Select all files</span>
|
||||||
</label>
|
</label>
|
||||||
<span>{selectedArchivePaths.size} of {archivePreview.file_count} selected</span>
|
<span>{selectedArchivePaths.size} of {archivePreview.file_count} selected</span>
|
||||||
</div>
|
</ActionToolbar>
|
||||||
|
|
||||||
<div className="archive-entry-list" role="list" aria-label="Archive contents">
|
<div className="archive-entry-list" role="list" aria-label="Archive contents">
|
||||||
{archivePreview.entries.map((entry) => {
|
{archivePreview.entries.map((entry) => {
|
||||||
@@ -2696,11 +2829,11 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut
|
|||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div className="connector-browser">
|
<div className="connector-browser">
|
||||||
<div className="connector-browser-toolbar">
|
<ActionToolbar className="connector-browser-toolbar">
|
||||||
<Button onClick={browseConnectorParent} disabled={connectorParentDisabled}><ArrowUp size={15} aria-hidden="true" /> i18n:govoplan-files.up.2038bdec</Button>
|
<Button onClick={browseConnectorParent} disabled={connectorParentDisabled}><ArrowUp size={15} aria-hidden="true" /> i18n:govoplan-files.up.2038bdec</Button>
|
||||||
<span className="connector-browser-path" title={connectorLocationLabel}>{connectorLocationLabel}</span>
|
<span className="connector-browser-path" title={connectorLocationLabel}>{connectorLocationLabel}</span>
|
||||||
<Button onClick={() => connectorProfileId && void browseConnector(connectorProfileId, connectorPath, connectorLibraryId)} disabled={busy || connectorLoading || !connectorProfileId}><RefreshCw size={15} aria-hidden="true" /> i18n:govoplan-files.refresh.56e3badc</Button>
|
<Button onClick={() => connectorProfileId && void browseConnector(connectorProfileId, connectorPath, connectorLibraryId)} disabled={busy || connectorLoading || !connectorProfileId}><RefreshCw size={15} aria-hidden="true" /> i18n:govoplan-files.refresh.56e3badc</Button>
|
||||||
</div>
|
</ActionToolbar>
|
||||||
{connectorError && <p className="field-error connector-browser-error">{connectorError}</p>}
|
{connectorError && <p className="field-error connector-browser-error">{connectorError}</p>}
|
||||||
<div className="connector-browser-list" role="list" aria-label="i18n:govoplan-files.connector_files.8f351f5d">
|
<div className="connector-browser-list" role="list" aria-label="i18n:govoplan-files.connector_files.8f351f5d">
|
||||||
{connectorLoading &&
|
{connectorLoading &&
|
||||||
@@ -2773,14 +2906,27 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut
|
|||||||
<FormField label="i18n:govoplan-files.space_label.ec892726">
|
<FormField label="i18n:govoplan-files.space_label.ec892726">
|
||||||
<input value={connectorSpaceLabel} onChange={(event) => setConnectorSpaceLabel(event.target.value)} disabled={busy} placeholder={connectorSpaceSuggestedLabel()} />
|
<input value={connectorSpaceLabel} onChange={(event) => setConnectorSpaceLabel(event.target.value)} disabled={busy} placeholder={connectorSpaceSuggestedLabel()} />
|
||||||
</FormField>
|
</FormField>
|
||||||
|
<FormField label="Connector mode" help="Two-way mode permits explicit writes only. Remote deletion, rename, and move propagation remain disabled.">
|
||||||
|
<select
|
||||||
|
value={connectorSpaceReadOnly ? "read-only" : "two-way"}
|
||||||
|
onChange={(event) => setConnectorSpaceReadOnly(event.target.value !== "two-way")}
|
||||||
|
disabled={busy}>
|
||||||
|
<option value="read-only">Read-only</option>
|
||||||
|
<option
|
||||||
|
value="two-way"
|
||||||
|
disabled={activeConnectorProfile?.provider !== "s3" || !activeConnectorProfile.capabilities.includes("write")}>
|
||||||
|
Two-way (S3, explicit writes)
|
||||||
|
</option>
|
||||||
|
</select>
|
||||||
|
</FormField>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div className="connector-browser">
|
<div className="connector-browser">
|
||||||
<div className="connector-browser-toolbar">
|
<ActionToolbar className="connector-browser-toolbar">
|
||||||
<Button onClick={browseConnectorParent} disabled={connectorParentDisabled}><ArrowUp size={15} aria-hidden="true" /> i18n:govoplan-files.up.2038bdec</Button>
|
<Button onClick={browseConnectorParent} disabled={connectorParentDisabled}><ArrowUp size={15} aria-hidden="true" /> i18n:govoplan-files.up.2038bdec</Button>
|
||||||
<span className="connector-browser-path" title={connectorLocationLabel}>{connectorLocationLabel}</span>
|
<span className="connector-browser-path" title={connectorLocationLabel}>{connectorLocationLabel}</span>
|
||||||
<Button onClick={() => connectorProfileId && void browseConnector(connectorProfileId, connectorPath, connectorLibraryId)} disabled={busy || connectorLoading || !connectorProfileId}><RefreshCw size={15} aria-hidden="true" /> i18n:govoplan-files.refresh.56e3badc</Button>
|
<Button onClick={() => connectorProfileId && void browseConnector(connectorProfileId, connectorPath, connectorLibraryId)} disabled={busy || connectorLoading || !connectorProfileId}><RefreshCw size={15} aria-hidden="true" /> i18n:govoplan-files.refresh.56e3badc</Button>
|
||||||
</div>
|
</ActionToolbar>
|
||||||
{connectorError && <p className="field-error connector-browser-error">{connectorError}</p>}
|
{connectorError && <p className="field-error connector-browser-error">{connectorError}</p>}
|
||||||
<div className="connector-browser-list" role="list" aria-label="i18n:govoplan-files.connector_folders.960cbb03">
|
<div className="connector-browser-list" role="list" aria-label="i18n:govoplan-files.connector_folders.960cbb03">
|
||||||
{connectorLoading &&
|
{connectorLoading &&
|
||||||
@@ -2854,7 +3000,7 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut
|
|||||||
{dialog === "transfer" && transferDialogState &&
|
{dialog === "transfer" && transferDialogState &&
|
||||||
<FileDialog title={`${transferMode === "copy" ? "i18n:govoplan-files.copy.af74f7c5" : "i18n:govoplan-files.move.76cdb950"} selection`} onClose={closeDialog}>
|
<FileDialog title={`${transferMode === "copy" ? "i18n:govoplan-files.copy.af74f7c5" : "i18n:govoplan-files.move.76cdb950"} selection`} onClose={closeDialog}>
|
||||||
<p className="muted">i18n:govoplan-files.choose_a_destination_space_and_folder_folders_ar.45158643</p>
|
<p className="muted">i18n:govoplan-files.choose_a_destination_space_and_folder_folders_ar.45158643</p>
|
||||||
<div className="form-grid two">
|
<FormGrid columns={2} collapseAt="standard" className="">
|
||||||
<FormField label="i18n:govoplan-files.destination_space.92b63970" help="i18n:govoplan-files.select_the_space_that_will_receive_the_selected_.0a8bea8f">
|
<FormField label="i18n:govoplan-files.destination_space.92b63970" help="i18n:govoplan-files.select_the_space_that_will_receive_the_selected_.0a8bea8f">
|
||||||
<select value={transferDialogState.targetSpaceId} onChange={(event) => setTransferDialogState((current) => current ? { ...current, targetSpaceId: event.target.value, targetFolder: "" } : current)}>
|
<select value={transferDialogState.targetSpaceId} onChange={(event) => setTransferDialogState((current) => current ? { ...current, targetSpaceId: event.target.value, targetFolder: "" } : current)}>
|
||||||
{spaces.filter((space) => !isConnectorSpace(space)).map((space) => <option key={space.id} value={space.id}>{space.label}</option>)}
|
{spaces.filter((space) => !isConnectorSpace(space)).map((space) => <option key={space.id} value={space.id}>{space.label}</option>)}
|
||||||
@@ -2870,7 +3016,7 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut
|
|||||||
onSelect={(folderPath) => setTransferDialogState((current) => current ? { ...current, targetFolder: folderPath } : current)} />
|
onSelect={(folderPath) => setTransferDialogState((current) => current ? { ...current, targetFolder: folderPath } : current)} />
|
||||||
|
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</FormGrid>
|
||||||
<div className="button-row compact-actions align-end">
|
<div className="button-row compact-actions align-end">
|
||||||
<Button onClick={closeDialog} disabled={busy}>i18n:govoplan-files.cancel.77dfd213</Button>
|
<Button onClick={closeDialog} disabled={busy}>i18n:govoplan-files.cancel.77dfd213</Button>
|
||||||
<Button variant="primary" onClick={() => void handleTransferDialogSubmit()} disabled={busy}>{transferMode === "copy" ? "i18n:govoplan-files.copy.af74f7c5" : "i18n:govoplan-files.move.76cdb950"}</Button>
|
<Button variant="primary" onClick={() => void handleTransferDialogSubmit()} disabled={busy}>{transferMode === "copy" ? "i18n:govoplan-files.copy.af74f7c5" : "i18n:govoplan-files.move.76cdb950"}</Button>
|
||||||
@@ -2893,7 +3039,7 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut
|
|||||||
{dialog === "rename" &&
|
{dialog === "rename" &&
|
||||||
<FileDialog title="i18n:govoplan-files.bulk_rename_selected_items.5e79d694" onClose={closeDialog}>
|
<FileDialog title="i18n:govoplan-files.bulk_rename_selected_items.5e79d694" onClose={closeDialog}>
|
||||||
<p className="muted">i18n:govoplan-files.bulk_rename_changes_managed_display_paths_only_i.8cf34a6b</p>
|
<p className="muted">i18n:govoplan-files.bulk_rename_changes_managed_display_paths_only_i.8cf34a6b</p>
|
||||||
<div className="form-grid two">
|
<FormGrid columns={2} collapseAt="standard" className="">
|
||||||
<FormField label="i18n:govoplan-files.mode.a7b93d21" help="i18n:govoplan-files.choose_how_the_selected_names_should_be_changed.0231854f">
|
<FormField label="i18n:govoplan-files.mode.a7b93d21" help="i18n:govoplan-files.choose_how_the_selected_names_should_be_changed.0231854f">
|
||||||
<select value={renameMode} onChange={(event) => setRenameMode(event.target.value as RenameMode)}>
|
<select value={renameMode} onChange={(event) => setRenameMode(event.target.value as RenameMode)}>
|
||||||
<option value="prefix">i18n:govoplan-files.add_prefix.672452bc</option>
|
<option value="prefix">i18n:govoplan-files.add_prefix.672452bc</option>
|
||||||
@@ -2904,7 +3050,7 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut
|
|||||||
{renameMode === "prefix" && <FormField label="i18n:govoplan-files.prefix.90eceb01"><input value={renamePrefix} onChange={(event) => setRenamePrefix(event.target.value)} /></FormField>}
|
{renameMode === "prefix" && <FormField label="i18n:govoplan-files.prefix.90eceb01"><input value={renamePrefix} onChange={(event) => setRenamePrefix(event.target.value)} /></FormField>}
|
||||||
{renameMode === "suffix" && <FormField label="i18n:govoplan-files.suffix.885db975"><input value={renameSuffix} onChange={(event) => setRenameSuffix(event.target.value)} /></FormField>}
|
{renameMode === "suffix" && <FormField label="i18n:govoplan-files.suffix.885db975"><input value={renameSuffix} onChange={(event) => setRenameSuffix(event.target.value)} /></FormField>}
|
||||||
{renameMode === "replace" && <><FormField label="i18n:govoplan-files.find.df251b06"><input value={renameFind} onChange={(event) => setRenameFind(event.target.value)} /></FormField><FormField label="i18n:govoplan-files.replacement.c385e347"><input value={renameReplacement} onChange={(event) => setRenameReplacement(event.target.value)} /></FormField></>}
|
{renameMode === "replace" && <><FormField label="i18n:govoplan-files.find.df251b06"><input value={renameFind} onChange={(event) => setRenameFind(event.target.value)} /></FormField><FormField label="i18n:govoplan-files.replacement.c385e347"><input value={renameReplacement} onChange={(event) => setRenameReplacement(event.target.value)} /></FormField></>}
|
||||||
</div>
|
</FormGrid>
|
||||||
{selectedFolderPaths.size > 0 &&
|
{selectedFolderPaths.size > 0 &&
|
||||||
<ToggleSwitch label="i18n:govoplan-files.apply_recursively_to_folder_contents.c7076984" checked={renameRecursive} onChange={setRenameRecursive} disabled={busy} />
|
<ToggleSwitch label="i18n:govoplan-files.apply_recursively_to_folder_contents.c7076984" checked={renameRecursive} onChange={setRenameRecursive} disabled={busy} />
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import { useEffect, useMemo, useState } from "react";
|
import { useEffect, useMemo, useState } from "react";
|
||||||
import { Trash2 } from "lucide-react";
|
import { Trash2 } from "lucide-react";
|
||||||
import {
|
import { DialogSection,
|
||||||
Button,
|
Button,
|
||||||
ConfirmDialog,
|
ConfirmDialog,
|
||||||
DataGrid,
|
DataGrid,
|
||||||
@@ -193,7 +193,7 @@ export default function FileShareDialog({
|
|||||||
return (
|
return (
|
||||||
<>
|
<>
|
||||||
<FileDialog title={`Manage shares - ${file.filename}`} onClose={() => !busy && onClose()}>
|
<FileDialog title={`Manage shares - ${file.filename}`} onClose={() => !busy && onClose()}>
|
||||||
<div className="file-share-dialog-content">
|
<DialogSection className="file-share-dialog-content">
|
||||||
{error ? (
|
{error ? (
|
||||||
<DismissibleAlert tone="danger" resetKey={error}>
|
<DismissibleAlert tone="danger" resetKey={error}>
|
||||||
{error}
|
{error}
|
||||||
@@ -268,7 +268,7 @@ export default function FileShareDialog({
|
|||||||
<div className="button-row compact-actions align-end">
|
<div className="button-row compact-actions align-end">
|
||||||
<Button onClick={onClose} disabled={busy}>Close</Button>
|
<Button onClick={onClose} disabled={busy}>Close</Button>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</DialogSection>
|
||||||
</FileDialog>
|
</FileDialog>
|
||||||
<ConfirmDialog
|
<ConfirmDialog
|
||||||
open={Boolean(revokeTarget)}
|
open={Boolean(revokeTarget)}
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ import {
|
|||||||
type FileSpace,
|
type FileSpace,
|
||||||
type ManagedFile } from
|
type ManagedFile } from
|
||||||
"../../../api/files";
|
"../../../api/files";
|
||||||
import {
|
import { ActionToolbar,
|
||||||
Button,
|
Button,
|
||||||
ConfirmDialog,
|
ConfirmDialog,
|
||||||
Dialog,
|
Dialog,
|
||||||
@@ -407,7 +407,7 @@ export default function ManagedFileChooser({
|
|||||||
</aside>
|
</aside>
|
||||||
|
|
||||||
<section className={`managed-file-chooser-browser ${mode === "folder" ? "folder-mode" : "attachment-mode"}`}>
|
<section className={`managed-file-chooser-browser ${mode === "folder" ? "folder-mode" : "attachment-mode"}`}>
|
||||||
<div className="managed-file-chooser-toolbar">
|
<ActionToolbar className="managed-file-chooser-toolbar">
|
||||||
<div className="managed-file-breadcrumb" aria-label="i18n:govoplan-files.current_folder.5aeab2f0">
|
<div className="managed-file-breadcrumb" aria-label="i18n:govoplan-files.current_folder.5aeab2f0">
|
||||||
<button type="button" onClick={() => openFolder(effectiveRoot)} disabled={loading}>
|
<button type="button" onClick={() => openFolder(effectiveRoot)} disabled={loading}>
|
||||||
<Home size={14} aria-hidden="true" /> {selectedSpace?.label || "i18n:govoplan-files.files.6ce6c512"}
|
<Home size={14} aria-hidden="true" /> {selectedSpace?.label || "i18n:govoplan-files.files.6ce6c512"}
|
||||||
@@ -419,7 +419,7 @@ export default function ManagedFileChooser({
|
|||||||
</span>
|
</span>
|
||||||
)}
|
)}
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</ActionToolbar>
|
||||||
|
|
||||||
{mode === "attachment" &&
|
{mode === "attachment" &&
|
||||||
<ManagedPatternEditor
|
<ManagedPatternEditor
|
||||||
|
|||||||
@@ -157,6 +157,7 @@ export const generatedTranslations: PlatformTranslations = {
|
|||||||
"i18n:govoplan-files.file.2c3cafa4": "File",
|
"i18n:govoplan-files.file.2c3cafa4": "File",
|
||||||
"i18n:govoplan-files.files_with_the_same_visible_name_already_exist_i.1bb487b0": "Files with the same visible name already exist in this folder.",
|
"i18n:govoplan-files.files_with_the_same_visible_name_already_exist_i.1bb487b0": "Files with the same visible name already exist in this folder.",
|
||||||
"i18n:govoplan-files.files.6ce6c512": "Files",
|
"i18n:govoplan-files.files.6ce6c512": "Files",
|
||||||
|
"i18n:govoplan-files.quick_access_description": "Recent and contextual files available to you.",
|
||||||
"i18n:govoplan-files.finalizing_upload.bcce936d": "Finalizing upload…",
|
"i18n:govoplan-files.finalizing_upload.bcce936d": "Finalizing upload…",
|
||||||
"i18n:govoplan-files.find_and_replace.2c6b404b": "Find and replace",
|
"i18n:govoplan-files.find_and_replace.2c6b404b": "Find and replace",
|
||||||
"i18n:govoplan-files.find.df251b06": "Find",
|
"i18n:govoplan-files.find.df251b06": "Find",
|
||||||
@@ -512,6 +513,7 @@ export const generatedTranslations: PlatformTranslations = {
|
|||||||
"i18n:govoplan-files.file.2c3cafa4": "Datei",
|
"i18n:govoplan-files.file.2c3cafa4": "Datei",
|
||||||
"i18n:govoplan-files.files_with_the_same_visible_name_already_exist_i.1bb487b0": "Files with the same visible name already exist in this folder.",
|
"i18n:govoplan-files.files_with_the_same_visible_name_already_exist_i.1bb487b0": "Files with the same visible name already exist in this folder.",
|
||||||
"i18n:govoplan-files.files.6ce6c512": "Dateien",
|
"i18n:govoplan-files.files.6ce6c512": "Dateien",
|
||||||
|
"i18n:govoplan-files.quick_access_description": "Für Sie verfügbare aktuelle und kontextbezogene Dateien.",
|
||||||
"i18n:govoplan-files.finalizing_upload.bcce936d": "Finalizing upload…",
|
"i18n:govoplan-files.finalizing_upload.bcce936d": "Finalizing upload…",
|
||||||
"i18n:govoplan-files.find_and_replace.2c6b404b": "Find and replace",
|
"i18n:govoplan-files.find_and_replace.2c6b404b": "Find and replace",
|
||||||
"i18n:govoplan-files.find.df251b06": "Find",
|
"i18n:govoplan-files.find.df251b06": "Find",
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
export { default } from "./module";
|
export { default } from "./module";
|
||||||
export * from "./module";
|
export * from "./module";
|
||||||
export { default as FilesPage } from "./features/files/FilesPage";
|
export { default as FilesPage } from "./features/files/FilesPage";
|
||||||
|
export { default as FileIntegrityPanel } from "./features/files/FileIntegrityPanel";
|
||||||
export * from "./api/files";
|
export * from "./api/files";
|
||||||
export type { PlatformWebModule, PlatformNavItem, PlatformRouteContribution, PlatformRouteContext } from "@govoplan/core-webui";
|
export type { PlatformWebModule, PlatformNavItem, PlatformRouteContribution, PlatformRouteContext } from "@govoplan/core-webui";
|
||||||
export { FolderTree } from "./features/files/components/FileManagerComponents";
|
export { FolderTree } from "./features/files/components/FileManagerComponents";
|
||||||
|
|||||||
+31
-3
@@ -5,17 +5,20 @@ import {
|
|||||||
type DashboardWidgetsUiCapability,
|
type DashboardWidgetsUiCapability,
|
||||||
type FilesConnectorsUiCapability,
|
type FilesConnectorsUiCapability,
|
||||||
type FilesFileExplorerUiCapability,
|
type FilesFileExplorerUiCapability,
|
||||||
type PlatformWebModule
|
type PlatformWebModule,
|
||||||
|
type QuickAccessToolsUiCapability
|
||||||
} from "@govoplan/core-webui";
|
} from "@govoplan/core-webui";
|
||||||
import { FolderTree } from "./features/files/components/FileManagerComponents";
|
import { FolderTree } from "./features/files/components/FileManagerComponents";
|
||||||
import FileConnectorSettingsPanel from "./features/files/FileConnectorSettingsPanel";
|
import FileConnectorSettingsPanel from "./features/files/FileConnectorSettingsPanel";
|
||||||
import ManagedFileChooser from "./features/files/components/ManagedFileChooser";
|
import ManagedFileChooser from "./features/files/components/ManagedFileChooser";
|
||||||
import FileSpacesWidget from "./features/files/FileSpacesWidget";
|
import FileSpacesWidget from "./features/files/FileSpacesWidget";
|
||||||
|
import FileQuickAccess from "./features/files/FileQuickAccess";
|
||||||
import { listFileSpaces, listFiles, listFilesDelta, listFolders, resolveFilePatterns, shareFilesWithTarget } from "./api/files";
|
import { listFileSpaces, listFiles, listFilesDelta, listFolders, resolveFilePatterns, shareFilesWithTarget } from "./api/files";
|
||||||
import { generatedTranslations } from "./i18n/generatedTranslations";
|
import { generatedTranslations } from "./i18n/generatedTranslations";
|
||||||
import "./styles/file-manager.css";
|
import "./styles/file-manager.css";
|
||||||
|
|
||||||
const FilesPage = lazy(() => import("./features/files/FilesPage"));
|
const FilesPage = lazy(() => import("./features/files/FilesPage"));
|
||||||
|
const FileIntegrityPanel = lazy(() => import("./features/files/FileIntegrityPanel"));
|
||||||
|
|
||||||
const fileRead = ["files:file:read"];
|
const fileRead = ["files:file:read"];
|
||||||
const translations = {
|
const translations = {
|
||||||
@@ -66,6 +69,14 @@ const fileDashboardWidgets: DashboardWidgetsUiCapability = {
|
|||||||
}
|
}
|
||||||
]
|
]
|
||||||
};
|
};
|
||||||
|
const fileQuickAccessTools: QuickAccessToolsUiCapability = {
|
||||||
|
tools: [
|
||||||
|
{
|
||||||
|
id: "files.recent",
|
||||||
|
render: (context) => createElement(FileQuickAccess, context)
|
||||||
|
}
|
||||||
|
]
|
||||||
|
};
|
||||||
const fileConnectorAdminSections: AdminSectionsUiCapability = {
|
const fileConnectorAdminSections: AdminSectionsUiCapability = {
|
||||||
sections: [
|
sections: [
|
||||||
{
|
{
|
||||||
@@ -97,6 +108,20 @@ const fileConnectorAdminSections: AdminSectionsUiCapability = {
|
|||||||
scopeType: "tenant",
|
scopeType: "tenant",
|
||||||
canWrite: hasScope(auth, "admin:settings:write") || hasScope(auth, "files:file:admin")
|
canWrite: hasScope(auth, "admin:settings:write") || hasScope(auth, "files:file:admin")
|
||||||
})
|
})
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "tenant-file-integrity",
|
||||||
|
moduleId: "files",
|
||||||
|
kind: "operations",
|
||||||
|
surfaceId: "files.admin.tenant-integrity",
|
||||||
|
label: "File integrity",
|
||||||
|
group: "TENANT",
|
||||||
|
order: 66,
|
||||||
|
allOf: ["files:file:admin"],
|
||||||
|
render: ({ settings, auth }) => createElement(FileIntegrityPanel, {
|
||||||
|
settings,
|
||||||
|
canWrite: hasScope(auth, "files:file:admin")
|
||||||
|
})
|
||||||
}]
|
}]
|
||||||
|
|
||||||
};
|
};
|
||||||
@@ -110,10 +135,12 @@ export const filesModule: PlatformWebModule = {
|
|||||||
viewSurfaces: [
|
viewSurfaces: [
|
||||||
{ id: "files.admin.system-connectors", moduleId: "files", kind: "section", label: "System file connections", order: 75 },
|
{ id: "files.admin.system-connectors", moduleId: "files", kind: "section", label: "System file connections", order: 75 },
|
||||||
{ id: "files.admin.tenant-connectors", moduleId: "files", kind: "section", label: "Tenant file connections", order: 65 },
|
{ id: "files.admin.tenant-connectors", moduleId: "files", kind: "section", label: "Tenant file connections", order: 65 },
|
||||||
|
{ id: "files.admin.tenant-integrity", moduleId: "files", kind: "section", label: "File integrity", order: 66 },
|
||||||
{ id: "files.admin.group-connectors", moduleId: "files", kind: "section", label: "Group file connections", order: 65 },
|
{ id: "files.admin.group-connectors", moduleId: "files", kind: "section", label: "Group file connections", order: 65 },
|
||||||
{ id: "files.admin.user-connectors", moduleId: "files", kind: "section", label: "User file connections", order: 65 },
|
{ id: "files.admin.user-connectors", moduleId: "files", kind: "section", label: "User file connections", order: 65 },
|
||||||
{ id: "files.settings.connectors", moduleId: "files", kind: "section", label: "Personal file connections", order: 20 },
|
{ id: "files.settings.connectors", moduleId: "files", kind: "section", label: "Personal file connections", order: 20 },
|
||||||
{ id: "files.widget.spaces", moduleId: "files", kind: "section", label: "File spaces widget", order: 35 }
|
{ id: "files.widget.spaces", moduleId: "files", kind: "section", label: "File spaces widget", order: 35 },
|
||||||
|
{ id: "files.quick_access.files", moduleId: "files", kind: "quick_access", label: "Files Quick Access", order: 40 }
|
||||||
],
|
],
|
||||||
navItems: [{ to: "/files", label: "i18n:govoplan-files.files.6ce6c512", iconName: "folder", anyOf: fileRead, order: 40 }],
|
navItems: [{ to: "/files", label: "i18n:govoplan-files.files.6ce6c512", iconName: "folder", anyOf: fileRead, order: 40 }],
|
||||||
routes: [
|
routes: [
|
||||||
@@ -134,7 +161,8 @@ export const filesModule: PlatformWebModule = {
|
|||||||
FileConnectorScopeManager: FileConnectorSettingsPanel
|
FileConnectorScopeManager: FileConnectorSettingsPanel
|
||||||
} satisfies FilesConnectorsUiCapability,
|
} satisfies FilesConnectorsUiCapability,
|
||||||
"admin.sections": fileConnectorAdminSections,
|
"admin.sections": fileConnectorAdminSections,
|
||||||
"dashboard.widgets": fileDashboardWidgets
|
"dashboard.widgets": fileDashboardWidgets,
|
||||||
|
"quickAccess.tools": fileQuickAccessTools
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
@@ -113,7 +113,7 @@
|
|||||||
min-width: 190px;
|
min-width: 190px;
|
||||||
overflow: hidden;
|
overflow: hidden;
|
||||||
border: var(--border-line-dark);
|
border: var(--border-line-dark);
|
||||||
border-radius: 12px;
|
border-radius: var(--radius-lg);
|
||||||
background: var(--panel);
|
background: var(--panel);
|
||||||
box-shadow: var(--shadow-popover);
|
box-shadow: var(--shadow-popover);
|
||||||
padding: 6px;
|
padding: 6px;
|
||||||
@@ -125,7 +125,7 @@
|
|||||||
gap: 9px;
|
gap: 9px;
|
||||||
width: 100%;
|
width: 100%;
|
||||||
border: 0;
|
border: 0;
|
||||||
border-radius: 8px;
|
border-radius: var(--radius-md);
|
||||||
background: transparent;
|
background: transparent;
|
||||||
color: var(--text);
|
color: var(--text);
|
||||||
cursor: pointer;
|
cursor: pointer;
|
||||||
@@ -161,7 +161,7 @@
|
|||||||
max-height: min(720px, calc(100vh - 44px));
|
max-height: min(720px, calc(100vh - 44px));
|
||||||
overflow: auto;
|
overflow: auto;
|
||||||
border: var(--border-line-dark);
|
border: var(--border-line-dark);
|
||||||
border-radius: 16px;
|
border-radius: var(--radius-xl);
|
||||||
background: var(--panel);
|
background: var(--panel);
|
||||||
box-shadow: var(--shadow-popover);
|
box-shadow: var(--shadow-popover);
|
||||||
}
|
}
|
||||||
@@ -236,7 +236,7 @@
|
|||||||
scrollbar-gutter: stable;
|
scrollbar-gutter: stable;
|
||||||
padding: 8px;
|
padding: 8px;
|
||||||
border: var(--border-line);
|
border: var(--border-line);
|
||||||
border-radius: 12px;
|
border-radius: var(--radius-lg);
|
||||||
background: var(--panel-soft);
|
background: var(--panel-soft);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -247,7 +247,7 @@
|
|||||||
width: 100%;
|
width: 100%;
|
||||||
min-width: 0;
|
min-width: 0;
|
||||||
border: 0;
|
border: 0;
|
||||||
border-radius: 9px;
|
border-radius: var(--radius-md);
|
||||||
background: transparent;
|
background: transparent;
|
||||||
color: var(--text);
|
color: var(--text);
|
||||||
cursor: pointer;
|
cursor: pointer;
|
||||||
@@ -308,7 +308,7 @@
|
|||||||
gap: 1px;
|
gap: 1px;
|
||||||
overflow: hidden;
|
overflow: hidden;
|
||||||
border: var(--border-line);
|
border: var(--border-line);
|
||||||
border-radius: 6px;
|
border-radius: var(--radius-compact);
|
||||||
background: var(--line);
|
background: var(--line);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -332,13 +332,6 @@
|
|||||||
font-size: var(--font-size-sm);
|
font-size: var(--font-size-sm);
|
||||||
}
|
}
|
||||||
|
|
||||||
.archive-selection-toolbar {
|
|
||||||
display: flex;
|
|
||||||
align-items: center;
|
|
||||||
justify-content: space-between;
|
|
||||||
gap: 12px;
|
|
||||||
}
|
|
||||||
|
|
||||||
.archive-selection-toolbar label,
|
.archive-selection-toolbar label,
|
||||||
.archive-entry-row {
|
.archive-entry-row {
|
||||||
display: flex;
|
display: flex;
|
||||||
@@ -363,7 +356,7 @@
|
|||||||
overflow: auto;
|
overflow: auto;
|
||||||
scrollbar-gutter: stable;
|
scrollbar-gutter: stable;
|
||||||
border: var(--border-line);
|
border: var(--border-line);
|
||||||
border-radius: 6px;
|
border-radius: var(--radius-compact);
|
||||||
background: var(--surface);
|
background: var(--surface);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -437,7 +430,7 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@media (max-width: 620px) {
|
@media (max-width: 680px) {
|
||||||
.archive-preview-summary {
|
.archive-preview-summary {
|
||||||
grid-template-columns: minmax(0, 1fr);
|
grid-template-columns: minmax(0, 1fr);
|
||||||
}
|
}
|
||||||
@@ -460,7 +453,7 @@
|
|||||||
min-height: 320px;
|
min-height: 320px;
|
||||||
overflow: hidden;
|
overflow: hidden;
|
||||||
border: var(--border-line);
|
border: var(--border-line);
|
||||||
border-radius: 12px;
|
border-radius: var(--radius-lg);
|
||||||
background: var(--panel);
|
background: var(--panel);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -505,7 +498,7 @@
|
|||||||
min-height: 48px;
|
min-height: 48px;
|
||||||
width: 100%;
|
width: 100%;
|
||||||
border: 0;
|
border: 0;
|
||||||
border-radius: 8px;
|
border-radius: var(--radius-md);
|
||||||
background: transparent;
|
background: transparent;
|
||||||
color: var(--text);
|
color: var(--text);
|
||||||
cursor: pointer;
|
cursor: pointer;
|
||||||
@@ -575,7 +568,7 @@
|
|||||||
min-width: 0;
|
min-width: 0;
|
||||||
padding: 10px 12px;
|
padding: 10px 12px;
|
||||||
border: var(--border-line);
|
border: var(--border-line);
|
||||||
border-radius: 10px;
|
border-radius: var(--radius-lg);
|
||||||
background: var(--panel-soft);
|
background: var(--panel-soft);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -623,7 +616,7 @@
|
|||||||
align-items: center;
|
align-items: center;
|
||||||
padding: 10px 12px;
|
padding: 10px 12px;
|
||||||
border: var(--border-line);
|
border: var(--border-line);
|
||||||
border-radius: 8px;
|
border-radius: var(--radius-md);
|
||||||
background: var(--panel-soft);
|
background: var(--panel-soft);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -720,7 +713,7 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@media (max-width: 1050px) {
|
@media (max-width: 1100px) {
|
||||||
.files-page .file-manager-shell {
|
.files-page .file-manager-shell {
|
||||||
grid-template-columns: 1fr;
|
grid-template-columns: 1fr;
|
||||||
}
|
}
|
||||||
@@ -765,7 +758,7 @@
|
|||||||
gap: 3px;
|
gap: 3px;
|
||||||
padding: 12px 14px;
|
padding: 12px 14px;
|
||||||
border: var(--border-line);
|
border: var(--border-line);
|
||||||
border-radius: 12px;
|
border-radius: var(--radius-lg);
|
||||||
background: var(--panel-soft);
|
background: var(--panel-soft);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -788,7 +781,7 @@
|
|||||||
align-items: center;
|
align-items: center;
|
||||||
padding: 10px;
|
padding: 10px;
|
||||||
border: var(--border-line);
|
border: var(--border-line);
|
||||||
border-radius: 12px;
|
border-radius: var(--radius-lg);
|
||||||
background: var(--panel-soft);
|
background: var(--panel-soft);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -826,7 +819,7 @@
|
|||||||
display: block;
|
display: block;
|
||||||
width: 100%;
|
width: 100%;
|
||||||
border: 1px dashed var(--line-dark);
|
border: 1px dashed var(--line-dark);
|
||||||
border-radius: 6px;
|
border-radius: var(--radius-compact);
|
||||||
background: var(--panel-soft);
|
background: var(--panel-soft);
|
||||||
color: var(--muted);
|
color: var(--muted);
|
||||||
cursor: pointer;
|
cursor: pointer;
|
||||||
@@ -949,7 +942,7 @@
|
|||||||
.managed-file-entry {
|
.managed-file-entry {
|
||||||
width: 100%;
|
width: 100%;
|
||||||
border: 1px solid transparent;
|
border: 1px solid transparent;
|
||||||
border-radius: 9px;
|
border-radius: var(--radius-md);
|
||||||
background: transparent;
|
background: transparent;
|
||||||
color: var(--text-strong);
|
color: var(--text-strong);
|
||||||
text-align: left;
|
text-align: left;
|
||||||
@@ -1027,7 +1020,7 @@
|
|||||||
gap: 5px;
|
gap: 5px;
|
||||||
align-items: center;
|
align-items: center;
|
||||||
border: 0;
|
border: 0;
|
||||||
border-radius: 6px;
|
border-radius: var(--radius-compact);
|
||||||
background: transparent;
|
background: transparent;
|
||||||
color: var(--text-strong);
|
color: var(--text-strong);
|
||||||
padding: 5px 6px;
|
padding: 5px 6px;
|
||||||
@@ -1176,7 +1169,7 @@
|
|||||||
gap: 4px;
|
gap: 4px;
|
||||||
align-items: center;
|
align-items: center;
|
||||||
border: 1px solid var(--success-line, var(--line));
|
border: 1px solid var(--success-line, var(--line));
|
||||||
border-radius: 999px;
|
border-radius: var(--radius-pill);
|
||||||
padding: 3px 7px;
|
padding: 3px 7px;
|
||||||
color: var(--success, var(--muted));
|
color: var(--success, var(--muted));
|
||||||
font-size: 11px;
|
font-size: 11px;
|
||||||
@@ -1263,7 +1256,7 @@
|
|||||||
color: var(--muted);
|
color: var(--muted);
|
||||||
}
|
}
|
||||||
|
|
||||||
@media (max-width: 850px) {
|
@media (max-width: 900px) {
|
||||||
.managed-file-entry-head,
|
.managed-file-entry-head,
|
||||||
.managed-file-entry {
|
.managed-file-entry {
|
||||||
grid-template-columns: 22px minmax(0, 1fr) 100px;
|
grid-template-columns: 22px minmax(0, 1fr) 100px;
|
||||||
|
|||||||
Reference in New Issue
Block a user