Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
378f4d6ac5 | ||
|
|
a8c3192c3a | ||
|
|
ce4eaefbfd | ||
|
|
6c3cf1c55e | ||
|
|
95aef18955 | ||
|
|
a9c7a7a40c | ||
|
|
e1361427b8 | ||
|
|
ad55d47645 | ||
|
|
21122e058e | ||
|
|
25140fb0b6 | ||
|
|
e9561a87e1 | ||
|
|
9d7d3f96ec | ||
|
|
44ba0ec9c1 | ||
|
|
1e511e4c15 | ||
|
|
cb37079465 | ||
|
|
04e2bae190 | ||
|
|
aa9f90c71e | ||
|
|
60c3d14f08 | ||
|
|
3a3360650f | ||
|
|
c9f8e9262e | ||
|
|
edee29af51 | ||
|
|
0293e49def | ||
|
|
aaa364b8d4 | ||
|
|
4bb17c0f4c | ||
|
|
92e649477f | ||
|
|
0c68e904cf | ||
|
|
8ec31b16ee | ||
|
|
7e6be4b017 | ||
|
|
04882f1628 | ||
|
|
d8ae506ff8 | ||
|
|
6baf2a421b | ||
|
|
b993d8e31a | ||
|
|
359c4e9570 | ||
|
|
233ce40983 | ||
|
|
f084a0f3a9 | ||
|
|
b752dea610 | ||
|
|
159a012833 | ||
|
|
835eacfc5d | ||
|
|
85606d5580 | ||
|
|
632cc6cf7d | ||
|
|
86a905a3a7 | ||
|
|
5b868272b9 | ||
|
|
93eea839c8 | ||
|
|
b667e7ff0e | ||
|
|
0fd8f6e02a |
@@ -0,0 +1,270 @@
|
|||||||
|
name: Module Package Release
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
tags:
|
||||||
|
- "v*"
|
||||||
|
workflow_dispatch:
|
||||||
|
inputs:
|
||||||
|
release_tag:
|
||||||
|
description: Existing protected version tag to publish
|
||||||
|
required: true
|
||||||
|
type: string
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
publish-packages:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
env:
|
||||||
|
GITEA_REPOSITORY: ${{ gitea.repository }}
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
|
||||||
|
with:
|
||||||
|
fetch-depth: 0
|
||||||
|
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065
|
||||||
|
with:
|
||||||
|
python-version: "3.12"
|
||||||
|
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020
|
||||||
|
with:
|
||||||
|
node-version: "22"
|
||||||
|
- name: Select and validate protected release tag
|
||||||
|
shell: bash
|
||||||
|
env:
|
||||||
|
REQUESTED_TAG: ${{ inputs.release_tag }}
|
||||||
|
TRIGGER_TAG: ${{ gitea.ref_name }}
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
tag="${REQUESTED_TAG:-$TRIGGER_TAG}"
|
||||||
|
case "$tag" in
|
||||||
|
v[0-9]*.[0-9]*.[0-9]*) ;;
|
||||||
|
*) echo "Release tag must start with a SemVer-shaped vX.Y.Z value" >&2; exit 1 ;;
|
||||||
|
esac
|
||||||
|
git fetch --force origin "refs/tags/$tag:refs/tags/$tag" refs/heads/main:refs/remotes/origin/main
|
||||||
|
tag_commit="$(git rev-list -n 1 "$tag")"
|
||||||
|
git merge-base --is-ancestor "$tag_commit" refs/remotes/origin/main || {
|
||||||
|
echo "Release tag is not contained in main" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
git checkout --detach "$tag"
|
||||||
|
printf 'RELEASE_TAG=%s\n' "$tag" >> "$GITEA_ENV"
|
||||||
|
printf 'SOURCE_DATE_EPOCH=%s\n' "$(git show -s --format=%ct HEAD)" >> "$GITEA_ENV"
|
||||||
|
- name: Validate package versions
|
||||||
|
run: |
|
||||||
|
python - <<'PY'
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import tomllib
|
||||||
|
|
||||||
|
tag = os.environ["RELEASE_TAG"]
|
||||||
|
expected = tag.removeprefix("v")
|
||||||
|
project = tomllib.loads(Path("pyproject.toml").read_text(encoding="utf-8"))["project"]
|
||||||
|
if project.get("version") != expected:
|
||||||
|
raise SystemExit(f"pyproject version {project.get('version')!r} does not match {tag}")
|
||||||
|
if re.fullmatch(r"govoplan-[a-z0-9-]+", str(project.get("name", ""))) is None:
|
||||||
|
raise SystemExit("Python distribution name must use the govoplan-* namespace")
|
||||||
|
webui = Path("webui/package.json")
|
||||||
|
if webui.is_file():
|
||||||
|
package = json.loads(webui.read_text(encoding="utf-8"))
|
||||||
|
if package.get("version") != expected:
|
||||||
|
raise SystemExit(f"WebUI version {package.get('version')!r} does not match {tag}")
|
||||||
|
if re.fullmatch(r"@govoplan/[a-z0-9-]+-webui", str(package.get("name", ""))) is None:
|
||||||
|
raise SystemExit("WebUI package name must use the @govoplan/*-webui namespace")
|
||||||
|
release = Path("webui/package.release.json")
|
||||||
|
if release.is_file():
|
||||||
|
release_package = json.loads(release.read_text(encoding="utf-8"))
|
||||||
|
if (
|
||||||
|
release_package.get("name") != package.get("name")
|
||||||
|
or release_package.get("version") != expected
|
||||||
|
):
|
||||||
|
raise SystemExit("WebUI release package identity does not match package.json and the release tag")
|
||||||
|
PY
|
||||||
|
- name: Build immutable package artifacts
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
python -m pip install --disable-pip-version-check build==1.5.0 twine==7.0.0
|
||||||
|
rm -rf dist .package-webui
|
||||||
|
python -m build --wheel --outdir dist
|
||||||
|
python -m twine check dist/*.whl
|
||||||
|
if [[ -f webui/package.json ]]; then
|
||||||
|
mkdir .package-webui
|
||||||
|
cp -a webui/. .package-webui/
|
||||||
|
rm -rf .package-webui/node_modules .package-webui/dist
|
||||||
|
if [[ -f .package-webui/package.release.json ]]; then
|
||||||
|
cp .package-webui/package.release.json .package-webui/package.json
|
||||||
|
fi
|
||||||
|
node <<'NODE'
|
||||||
|
const fs = require("node:fs");
|
||||||
|
const path = ".package-webui/package.json";
|
||||||
|
const packageJson = JSON.parse(fs.readFileSync(path, "utf8"));
|
||||||
|
const groups = ["dependencies", "optionalDependencies", "peerDependencies"];
|
||||||
|
for (const group of groups) {
|
||||||
|
for (const [name, specifier] of Object.entries(packageJson[group] || {})) {
|
||||||
|
if (!name.startsWith("@govoplan/")) continue;
|
||||||
|
if (typeof specifier !== "string") {
|
||||||
|
throw new Error(`${group}.${name} must use a string version`);
|
||||||
|
}
|
||||||
|
const packageSlug = name.slice("@govoplan/".length);
|
||||||
|
if (!packageSlug.endsWith("-webui")) {
|
||||||
|
throw new Error(`${group}.${name} is outside the WebUI package namespace`);
|
||||||
|
}
|
||||||
|
const repository = `govoplan-${packageSlug.slice(0, -"-webui".length)}`;
|
||||||
|
const escapedRepository = repository.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||||
|
const gitTag = specifier.match(
|
||||||
|
new RegExp(
|
||||||
|
`^git\\+(?:ssh://git@|https://)git\\.add-ideas\\.de/(?:GovOPlaN|add-ideas)/${escapedRepository}\\.git#v([0-9]+\\.[0-9]+\\.[0-9]+)$`,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
if (gitTag) {
|
||||||
|
packageJson[group][name] = gitTag[1];
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (specifier.startsWith("file:") || specifier.startsWith("git+")) {
|
||||||
|
throw new Error(
|
||||||
|
`${group}.${name} must resolve to an exact registry version for publication`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
delete packageJson.private;
|
||||||
|
fs.writeFileSync(path, `${JSON.stringify(packageJson, null, 2)}\n`);
|
||||||
|
NODE
|
||||||
|
npm pkg delete private --prefix .package-webui
|
||||||
|
(cd .package-webui && npm pack --ignore-scripts --pack-destination ../dist)
|
||||||
|
fi
|
||||||
|
python - <<'PY'
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
artifacts = []
|
||||||
|
for path in sorted(Path("dist").iterdir()):
|
||||||
|
if path.suffix not in {".whl", ".tgz"}:
|
||||||
|
continue
|
||||||
|
digest = hashlib.sha256(path.read_bytes()).hexdigest()
|
||||||
|
artifacts.append({"filename": path.name, "sha256": digest, "size": path.stat().st_size})
|
||||||
|
payload = {
|
||||||
|
"schema_version": "1",
|
||||||
|
"repository": os.environ["GITEA_REPOSITORY"],
|
||||||
|
"tag": os.environ["RELEASE_TAG"],
|
||||||
|
"commit": subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip(),
|
||||||
|
"artifacts": artifacts,
|
||||||
|
}
|
||||||
|
Path("dist/package-artifacts.json").write_text(
|
||||||
|
json.dumps(payload, indent=2, sort_keys=True) + "\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
PY
|
||||||
|
- name: Retain package hash evidence
|
||||||
|
uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32
|
||||||
|
with:
|
||||||
|
name: module-packages-${{ gitea.ref_name }}
|
||||||
|
path: dist/package-artifacts.json
|
||||||
|
- name: Check immutable registry state
|
||||||
|
shell: bash
|
||||||
|
env:
|
||||||
|
PACKAGE_TOKEN: ${{ secrets.GOVOPLAN_PACKAGE_TOKEN }}
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
test -n "$PACKAGE_TOKEN"
|
||||||
|
python - <<'PY'
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import tomllib
|
||||||
|
from urllib.error import HTTPError
|
||||||
|
from urllib.parse import quote
|
||||||
|
from urllib.request import Request, urlopen
|
||||||
|
|
||||||
|
api_root = "https://git.add-ideas.de/api/v1/packages/GovOPlaN"
|
||||||
|
token = os.environ["PACKAGE_TOKEN"]
|
||||||
|
|
||||||
|
def should_publish(kind, name, version, path):
|
||||||
|
package_url = "/".join(
|
||||||
|
(api_root, kind, quote(name, safe=""), quote(version, safe=""), "files")
|
||||||
|
)
|
||||||
|
request = Request(
|
||||||
|
package_url,
|
||||||
|
headers={"Accept": "application/json", "Authorization": f"token {token}"},
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
with urlopen(request, timeout=30) as response:
|
||||||
|
files = json.load(response)
|
||||||
|
except HTTPError as exc:
|
||||||
|
if exc.code == 404:
|
||||||
|
print(f"{kind} package {name}=={version} is not published yet")
|
||||||
|
return True
|
||||||
|
raise
|
||||||
|
if not isinstance(files, list) or len(files) != 1:
|
||||||
|
raise SystemExit(
|
||||||
|
f"immutable {kind} package {name}=={version} has an unexpected file set"
|
||||||
|
)
|
||||||
|
expected_sha256 = hashlib.sha256(path.read_bytes()).hexdigest()
|
||||||
|
if files[0].get("sha256") != expected_sha256:
|
||||||
|
raise SystemExit(
|
||||||
|
f"immutable {kind} package {name}=={version} already exists with a different SHA-256"
|
||||||
|
)
|
||||||
|
print(f"verified existing {kind} package {name}=={version} ({expected_sha256})")
|
||||||
|
return False
|
||||||
|
|
||||||
|
project = tomllib.loads(Path("pyproject.toml").read_text(encoding="utf-8"))["project"]
|
||||||
|
wheels = tuple(Path("dist").glob("*.whl"))
|
||||||
|
if len(wheels) != 1:
|
||||||
|
raise SystemExit("release build must contain exactly one wheel")
|
||||||
|
publish_pypi = should_publish(
|
||||||
|
"pypi", str(project["name"]), str(project["version"]), wheels[0]
|
||||||
|
)
|
||||||
|
|
||||||
|
tarballs = tuple(Path("dist").glob("*.tgz"))
|
||||||
|
if len(tarballs) > 1:
|
||||||
|
raise SystemExit("release build must contain at most one npm package")
|
||||||
|
publish_npm = False
|
||||||
|
if tarballs:
|
||||||
|
webui = json.loads(
|
||||||
|
Path(".package-webui/package.json").read_text(encoding="utf-8")
|
||||||
|
)
|
||||||
|
publish_npm = should_publish(
|
||||||
|
"npm", str(webui["name"]), str(webui["version"]), tarballs[0]
|
||||||
|
)
|
||||||
|
|
||||||
|
with Path(os.environ["GITEA_ENV"]).open("a", encoding="utf-8") as env_file:
|
||||||
|
env_file.write(f"PUBLISH_PYPI={int(publish_pypi)}\n")
|
||||||
|
env_file.write(f"PUBLISH_NPM={int(publish_npm)}\n")
|
||||||
|
PY
|
||||||
|
- name: Publish wheel and WebUI package
|
||||||
|
shell: bash
|
||||||
|
env:
|
||||||
|
PACKAGE_USERNAME: ${{ secrets.GOVOPLAN_PACKAGE_USERNAME }}
|
||||||
|
PACKAGE_TOKEN: ${{ secrets.GOVOPLAN_PACKAGE_TOKEN }}
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
test -n "$PACKAGE_USERNAME"
|
||||||
|
test -n "$PACKAGE_TOKEN"
|
||||||
|
if [[ "$PUBLISH_PYPI" == 1 ]]; then
|
||||||
|
TWINE_USERNAME="$PACKAGE_USERNAME" TWINE_PASSWORD="$PACKAGE_TOKEN" \
|
||||||
|
python -m twine upload --non-interactive \
|
||||||
|
--repository-url https://git.add-ideas.de/api/packages/GovOPlaN/pypi \
|
||||||
|
dist/*.whl
|
||||||
|
else
|
||||||
|
echo "Exact wheel is already present; skipping immutable retry."
|
||||||
|
fi
|
||||||
|
shopt -s nullglob
|
||||||
|
webui_packages=(dist/*.tgz)
|
||||||
|
if (( ${#webui_packages[@]} )) && [[ "$PUBLISH_NPM" == 1 ]]; then
|
||||||
|
npmrc="$(mktemp)"
|
||||||
|
trap 'rm -f "$npmrc"' EXIT
|
||||||
|
chmod 600 "$npmrc"
|
||||||
|
printf '%s\n' \
|
||||||
|
'@govoplan:registry=https://git.add-ideas.de/api/packages/GovOPlaN/npm/' \
|
||||||
|
"//git.add-ideas.de/api/packages/GovOPlaN/npm/:_authToken=$PACKAGE_TOKEN" \
|
||||||
|
> "$npmrc"
|
||||||
|
NPM_CONFIG_USERCONFIG="$npmrc" npm publish "./${webui_packages[0]}" \
|
||||||
|
--ignore-scripts --access public \
|
||||||
|
--registry https://git.add-ideas.de/api/packages/GovOPlaN/npm/
|
||||||
|
elif (( ${#webui_packages[@]} )); then
|
||||||
|
echo "Exact WebUI package is already present; skipping immutable retry."
|
||||||
|
fi
|
||||||
@@ -1,5 +1,11 @@
|
|||||||
# GovOPlaN Files Codex Guide
|
# GovOPlaN Files Codex Guide
|
||||||
|
|
||||||
|
## Documentation Contract
|
||||||
|
|
||||||
|
- Treat documentation as part of every behavior change. Update this module's manifest-driven `DocumentationTopic` contributions for affected user and administrator behavior.
|
||||||
|
- Keep feature content here; `govoplan-docs` projects it without importing Files internals.
|
||||||
|
- Maintain a static user/admin baseline and run `/mnt/DATA/git/govoplan/tools/checks/check-manifest-shapes.py` after behavior or manifest changes.
|
||||||
|
|
||||||
## Scope
|
## Scope
|
||||||
|
|
||||||
This repository owns the `files` module: managed file storage APIs, file metadata, shares, uploads/downloads, folder and pattern helpers, backend module manifest, and `@govoplan/files-webui`.
|
This repository owns the `files` module: managed file storage APIs, file metadata, shares, uploads/downloads, folder and pattern helpers, backend module manifest, and `@govoplan/files-webui`.
|
||||||
|
|||||||
@@ -55,6 +55,15 @@ for example for managed attachment selection and campaign file sharing. Files
|
|||||||
does not import campaign internals; campaign share/existence checks use the core
|
does not import campaign internals; campaign share/existence checks use the core
|
||||||
`campaigns.access` capability registered by the campaign module.
|
`campaigns.access` capability registered by the campaign module.
|
||||||
|
|
||||||
|
Files also publishes the optional `privacy.dsar.files` capability. The Core
|
||||||
|
data-subject workflow can use it to collect bounded, tenant-scoped file,
|
||||||
|
version, share, folder, evidence, and non-secret configuration metadata for a
|
||||||
|
direct membership subject. The provider may revoke a subject-targeted share or
|
||||||
|
detach mutable actor references idempotently, but it never exports credential
|
||||||
|
material or raw bytes and never bypasses Files retention, legal hold, evidence,
|
||||||
|
purge approval, audit, or recovery controls. See the handbook's data-subject
|
||||||
|
request coverage section for the review and erasure boundary.
|
||||||
|
|
||||||
Platform RBAC and governance rules are documented in `govoplan-core/docs/`.
|
Platform RBAC and governance rules are documented in `govoplan-core/docs/`.
|
||||||
|
|
||||||
Managed files can carry source provenance for connector and import workflows.
|
Managed files can carry source provenance for connector and import workflows.
|
||||||
@@ -105,7 +114,8 @@ run profile policy with the `browse` operation, and support Seafile,
|
|||||||
WebDAV/Nextcloud, and SMB when the optional `smb` extra is installed. Seafile
|
WebDAV/Nextcloud, and SMB when the optional `smb` extra is installed. Seafile
|
||||||
profiles browse libraries and directories via Seafile's read-only API; profiles
|
profiles browse libraries and directories via Seafile's read-only API; profiles
|
||||||
can still opt into WebDAV browsing by setting `metadata.webdav_endpoint_url` or
|
can still opt into WebDAV browsing by setting `metadata.webdav_endpoint_url` or
|
||||||
`metadata.browse_protocol` to `webdav`.
|
`metadata.browse_protocol` to `webdav`. S3-compatible profiles support bucket
|
||||||
|
and prefix browsing when the optional `s3` extra is installed.
|
||||||
`POST /api/v1/files/connectors/profiles/{profile_id}/import` imports a Seafile
|
`POST /api/v1/files/connectors/profiles/{profile_id}/import` imports a Seafile
|
||||||
or WebDAV/Nextcloud/SMB file into managed storage through the same governance,
|
or WebDAV/Nextcloud/SMB file into managed storage through the same governance,
|
||||||
conflict handling, source provenance, and connector audit path as direct
|
conflict handling, source provenance, and connector audit path as direct
|
||||||
@@ -113,17 +123,38 @@ uploads. The Seafile provider uses account-token auth and the native file
|
|||||||
download-link API; Nextcloud and generic WebDAV profiles use authenticated `GET`
|
download-link API; Nextcloud and generic WebDAV profiles use authenticated `GET`
|
||||||
requests against the configured WebDAV endpoint. SMB profiles use
|
requests against the configured WebDAV endpoint. SMB profiles use
|
||||||
`smb://server[:port]/share[/path]` endpoints and deployment-owned or encrypted
|
`smb://server[:port]/share[/path]` endpoints and deployment-owned or encrypted
|
||||||
stored credentials through `smbprotocol`. Because that SDK cannot accept a
|
stored credentials through `smbprotocol`. Files installs a pinned transport for
|
||||||
preconnected socket and may follow DFS referrals to additional hosts, live SMB
|
every initial session, reconnect, alias, and DFS referral target. The deployment
|
||||||
access fails closed in both public-only and private-network deployments. It will
|
private-network policy is re-evaluated immediately before each socket opens.
|
||||||
remain disabled until every initial connection and referral target can be
|
|
||||||
policy-validated and pinned.
|
|
||||||
|
|
||||||
The S3 browse/import implementation and S3 managed-storage backend currently
|
S3 connector browse/import binds botocore HTTP and HTTPS pools to the same pinned
|
||||||
fail closed before creating a `boto3` client. Botocore does not yet use the
|
socket policy. Retries, redirects, endpoint discovery, bucket aliases, and new
|
||||||
GovOPlaN pinned HTTP transport and may manage redirects itself, so neither an
|
connections are therefore revalidated while TLS keeps the configured hostname
|
||||||
explicit endpoint nor SDK endpoint discovery is allowed until both peer pinning
|
for SNI and certificate checks. Connector clients do not use outbound proxies or
|
||||||
and redirect revalidation are enforced.
|
ambient AWS credential discovery; configure credentials on the governed profile,
|
||||||
|
or explicitly use an anonymous profile for public objects. Keep connector spaces
|
||||||
|
read-only unless a remote write is explicitly needed.
|
||||||
|
An S3 profile with the `write` capability can back a two-way connector space.
|
||||||
|
`POST /api/v1/files/connector-spaces/{space_id}/write-back` then conditionally
|
||||||
|
creates or replaces one remote object from a managed file, records durable
|
||||||
|
recovery intent before the provider effect, and verifies request/content markers
|
||||||
|
afterward. Automatic remote delete, rename, move, and ACL propagation remain
|
||||||
|
disabled; other connector providers remain read-only.
|
||||||
|
An incompatible SDK upgrade fails closed before a usable client/session is
|
||||||
|
returned.
|
||||||
|
|
||||||
|
Durable platform storage has a separate deployment boundary: installer-owned
|
||||||
|
Garage is accepted only at its exact generated endpoint, while an
|
||||||
|
operator-selected external backend requires a clean HTTPS origin and
|
||||||
|
`FILE_STORAGE_S3_ENDPOINT_TRUSTED=true`. That flag is deployment configuration,
|
||||||
|
cannot be supplied through a Files connector profile, and does not replace
|
||||||
|
operator responsibility for DNS, certificates, egress, bucket policy,
|
||||||
|
versioning, and recovery.
|
||||||
|
|
||||||
|
The actual local/S3 backend implementation is owned by Core so Files, Campaign,
|
||||||
|
and workers resolve the same object namespace. Files owns file metadata and key
|
||||||
|
layout. Node-local storage is supported only for `local` or one-host
|
||||||
|
`host-shared` profiles; multi-host `shared` deployments require S3.
|
||||||
|
|
||||||
Destructive Files-module retirement applies the same credential lifecycle before
|
Destructive Files-module retirement applies the same credential lifecycle before
|
||||||
dropping tables. Every remaining Files-owned encrypted connector secret is
|
dropping tables. Every remaining Files-owned encrypted connector secret is
|
||||||
@@ -139,17 +170,46 @@ Connector and collaboration ownership boundaries are documented in
|
|||||||
`docs/CONNECTOR_BOUNDARY.md` and `docs/DOCUMENT_COLLABORATION_BOUNDARY.md`.
|
`docs/CONNECTOR_BOUNDARY.md` and `docs/DOCUMENT_COLLABORATION_BOUNDARY.md`.
|
||||||
The role-adaptive user, administration, integration, and operator guide is the
|
The role-adaptive user, administration, integration, and operator guide is the
|
||||||
[Files handbook](docs/FILES_HANDBOOK.md).
|
[Files handbook](docs/FILES_HANDBOOK.md).
|
||||||
|
The Files route, connector surfaces, consequence classes, and pattern-language
|
||||||
|
verification are recorded in
|
||||||
|
[Files interface pattern migration](docs/INTERFACE_PATTERN_MIGRATION.md).
|
||||||
|
|
||||||
ZIP uploads are processed without buffering the whole archive in memory. The API
|
Archive imports use a two-phase preview and confirmation flow for ZIP, TAR,
|
||||||
spools incoming ZIP request bodies to a bounded temporary file, then extracts
|
TAR.GZ, TAR.BZ2, and TAR.XZ. Requests are spooled to bounded temporary files;
|
||||||
members with per-file and total extracted-size limits before storing managed
|
the server validates paths, entry count, expanded size, and expansion ratio,
|
||||||
files.
|
then returns a 30-minute tenant/user-bound preview token. Confirmation reuploads
|
||||||
|
the original archive and stores only the selected members. Password-protected
|
||||||
|
ZIP passwords remain request-only and are never included in the preview token.
|
||||||
|
|
||||||
|
Managed blob writes and applied orphan cleanup use Core's durable recovery
|
||||||
|
ledger. On PostgreSQL, intent, request digests, recovery mode, and a distributed
|
||||||
|
lease are committed before physical storage effects; the Files session commit
|
||||||
|
verifies database and streamed object evidence, while rollback compensates only
|
||||||
|
a newly reserved unreferenced key. Development SQLite records blob intent in
|
||||||
|
the caller transaction to avoid its second-writer deadlock, then verifies on
|
||||||
|
commit or reconstructs compensation evidence after handled rollback. Because a
|
||||||
|
hard loss before that commit can leave an unrecorded object, SQLite requires a
|
||||||
|
complete integrity scan after a crash and is not a production recovery profile.
|
||||||
|
New object keys are opaque and do not retain the uploaded filename. Uncertain
|
||||||
|
or mismatched effects remain visible through Ops.
|
||||||
|
|
||||||
|
Operators with `files:file:admin` can run bounded, resumable integrity scans in
|
||||||
|
Administration. Scan batches and finding actions carry monotonic revisions;
|
||||||
|
stale resume, recheck, or cleanup requests fail before touching object storage.
|
||||||
|
Orphan cleanup always requires a dry-run preview followed by separate
|
||||||
|
confirmation and records recovery-ledger evidence.
|
||||||
|
|
||||||
Bulk rename and transfer APIs are owner-scoped: callers must provide the active
|
Bulk rename and transfer APIs are owner-scoped: callers must provide the active
|
||||||
user or group file space with `owner_type` and `owner_id`. The storage layer
|
user or group file space with `owner_type` and `owner_id`. The storage layer
|
||||||
keeps a named legacy file-only helper for historical callers that lack owner
|
keeps a named legacy file-only helper for historical callers that lack owner
|
||||||
context, and regression tests cover its write-access checks.
|
context, and regression tests cover its write-access checks.
|
||||||
|
|
||||||
|
Optional producer modules can persist generated output through the provider-
|
||||||
|
neutral `files.artifact_store` capability. Files remains responsible for upload
|
||||||
|
authorization, ownership, path normalization, versioning, and blob storage;
|
||||||
|
producers receive stable file/version references without importing Files
|
||||||
|
internals. See [Generated Artifact Store](docs/GENERATED_ARTIFACT_STORE.md).
|
||||||
|
|
||||||
## Release packaging
|
## Release packaging
|
||||||
|
|
||||||
The repository root includes a `package.json` for git-based WebUI installs. It exports the package `@govoplan/files-webui` from `webui/src` so release builds can depend on tagged git refs instead of local `file:` paths.
|
The repository root includes a `package.json` for git-based WebUI installs. It exports the package `@govoplan/files-webui` from `webui/src` so release builds can depend on tagged git refs instead of local `file:` paths.
|
||||||
|
|||||||
@@ -19,6 +19,7 @@ normal product workflows and can share the same governance model:
|
|||||||
- Nextcloud through WebDAV
|
- Nextcloud through WebDAV
|
||||||
- generic WebDAV
|
- generic WebDAV
|
||||||
- SMB through the optional `smb` extra
|
- SMB through the optional `smb` extra
|
||||||
|
- S3-compatible stores through the optional `s3` extra
|
||||||
|
|
||||||
These providers are surfaced through connector descriptors at
|
These providers are surfaced through connector descriptors at
|
||||||
`GET /api/v1/files/connectors/providers`. Provider descriptors declare whether
|
`GET /api/v1/files/connectors/providers`. Provider descriptors declare whether
|
||||||
@@ -60,11 +61,14 @@ Every provider must:
|
|||||||
- use a transport that pins every connection to a policy-validated DNS/IP answer
|
- use a transport that pins every connection to a policy-validated DNS/IP answer
|
||||||
and revalidates redirects; SDK transports without that guarantee fail closed
|
and revalidates redirects; SDK transports without that guarantee fail closed
|
||||||
|
|
||||||
Live SMB access is disabled in all modes until `smbprotocol` initial connections
|
SMB initial connections, reconnects, aliases, and DFS referral targets are
|
||||||
and DFS referral targets can be pinned and policy-validated. An explicit IP is
|
created through a Files-owned pinned `smbprotocol` transport. S3 HTTP/HTTPS pools
|
||||||
not sufficient because the server may still issue a referral to another peer.
|
use the equivalent botocore adapter for every connection selected by retries,
|
||||||
Live S3 access is likewise disabled until `boto3`/botocore can be bound to the
|
redirects, endpoint discovery, and virtual-host addressing. Both adapters apply
|
||||||
pinned transport, including SDK-managed redirects and endpoint discovery.
|
the deployment-wide private-network policy at socket creation. S3 keeps the
|
||||||
|
configured hostname for TLS SNI and certificate verification, but does not use
|
||||||
|
outbound proxies or ambient AWS credential discovery. An incompatible optional
|
||||||
|
SDK release fails closed before a usable session or client is returned.
|
||||||
|
|
||||||
## Non-Goals For Files
|
## Non-Goals For Files
|
||||||
|
|
||||||
|
|||||||
@@ -61,11 +61,10 @@ Linked connector space:
|
|||||||
- read-only flag from provider/policy
|
- read-only flag from provider/policy
|
||||||
- active/deleted state
|
- active/deleted state
|
||||||
|
|
||||||
The linked space should be addressable as a normal file space in the files UI.
|
The linked space is addressable as a normal file space in the Files UI. It can
|
||||||
For the first implementation slice, actions may use the existing connector
|
browse the remote folder, synchronize one selected file, or run a bounded manual
|
||||||
browse/sync APIs and managed file storage. A linked space can therefore browse
|
folder synchronization into managed storage. Background sync jobs remain a
|
||||||
the remote folder and sync selected files into managed storage. Later slices can
|
separate scheduling concern.
|
||||||
add a native remote listing view or background sync jobs.
|
|
||||||
|
|
||||||
## Policy Semantics
|
## Policy Semantics
|
||||||
|
|
||||||
@@ -105,6 +104,9 @@ Implemented:
|
|||||||
- Files UI sync dialog for choosing a profile, browsing a remote folder, and
|
- Files UI sync dialog for choosing a profile, browsing a remote folder, and
|
||||||
syncing a selected file into a managed destination folder
|
syncing a selected file into a managed destination folder
|
||||||
- Files UI connector-space view with provider/read-only/manual-sync state
|
- Files UI connector-space view with provider/read-only/manual-sync state
|
||||||
|
- bounded folder-level manual sync with recursive traversal, pagination,
|
||||||
|
per-item review outcomes, source-provenance matching, and one summary audit
|
||||||
|
event
|
||||||
- Docker dev stack smoke checks for WebDAV, Nextcloud, and SMB
|
- Docker dev stack smoke checks for WebDAV, Nextcloud, and SMB
|
||||||
|
|
||||||
Missing:
|
Missing:
|
||||||
@@ -137,9 +139,14 @@ Missing:
|
|||||||
- Remaining: edit/manage actions for existing linked spaces.
|
- Remaining: edit/manage actions for existing linked spaces.
|
||||||
|
|
||||||
4. Add sync orchestration.
|
4. Add sync orchestration.
|
||||||
- Manual sync selected file is already available.
|
- Manual selected-file and folder-level synchronization are available.
|
||||||
- Add folder-level manual sync.
|
- Folder sync preserves remote relative paths, bounds file count and depth,
|
||||||
- Add optional scheduled/background sync with conflict reporting.
|
and reports created, updated, unchanged, skipped, conflict, policy-denied,
|
||||||
|
and failed items without rolling back successful siblings.
|
||||||
|
- The default conflict strategy is `skip`, so unrelated managed files are
|
||||||
|
left untouched. `rename`, `reject`, and `overwrite` remain explicit choices.
|
||||||
|
- Add optional scheduled/background sync separately; it must reuse the same
|
||||||
|
policy, provenance, result, and audit contracts.
|
||||||
|
|
||||||
5. Add provider-specific expansion.
|
5. Add provider-specific expansion.
|
||||||
- OAuth/secret-store credentials.
|
- OAuth/secret-store credentials.
|
||||||
|
|||||||
+446
-97
@@ -1,6 +1,6 @@
|
|||||||
# GovOPlaN Files Handbook
|
# GovOPlaN Files Handbook
|
||||||
|
|
||||||
This handbook describes the Files module as implemented in version `0.1.9`.
|
This handbook describes the Files module as implemented in version `0.1.18`.
|
||||||
It is the operational source of truth for users, process owners, administrators,
|
It is the operational source of truth for users, process owners, administrators,
|
||||||
operators, auditors, and module integrators. Statements about future behavior
|
operators, auditors, and module integrators. Statements about future behavior
|
||||||
are marked **planned**; an unmarked statement describes the current code.
|
are marked **planned**; an unmarked statement describes the current code.
|
||||||
@@ -8,7 +8,8 @@ are marked **planned**; an unmarked statement describes the current code.
|
|||||||
Files is a governed snapshot store. It owns managed file content, versions,
|
Files is a governed snapshot store. It owns managed file content, versions,
|
||||||
logical folders, shares, source provenance, and the evidence that another
|
logical folders, shares, source provenance, and the evidence that another
|
||||||
GovOPlaN module used a particular file version. It can browse selected external
|
GovOPlaN module used a particular file version. It can browse selected external
|
||||||
stores read-only and import a frozen copy. It is not a general remote filesystem,
|
stores and import a frozen copy. Explicit, conditional S3 write-back is available
|
||||||
|
only through an administrator-enabled two-way connector space. Files is not a general remote filesystem,
|
||||||
a document collaboration engine, or a records-management system.
|
a document collaboration engine, or a records-management system.
|
||||||
|
|
||||||
## Choose a reading path
|
## Choose a reading path
|
||||||
@@ -24,32 +25,65 @@ a document collaboration engine, or a records-management system.
|
|||||||
| Verify a release or scenario | [Acceptance scenarios](#acceptance-scenarios) |
|
| Verify a release or scenario | [Acceptance scenarios](#acceptance-scenarios) |
|
||||||
| Check whether an idea exists today | [Implemented and planned boundary](#implemented-and-planned-boundary) |
|
| Check whether an idea exists today | [Implemented and planned boundary](#implemented-and-planned-boundary) |
|
||||||
|
|
||||||
|
The Files-owned interface archetypes, consequence classes, disabled-state
|
||||||
|
wording, and verification evidence are recorded in
|
||||||
|
[Files Interface Pattern Migration](INTERFACE_PATTERN_MIGRATION.md).
|
||||||
|
|
||||||
## The service contract
|
## The service contract
|
||||||
|
|
||||||
A managed file is a tenant-scoped logical asset with exactly one user or group
|
A managed file is a tenant-scoped logical asset with exactly one user or group
|
||||||
owner, a normalized path, and a current version. The current version points to a
|
owner, a normalized path, and a current version. The current version points to a
|
||||||
blob record containing the storage location, SHA-256 checksum, byte size, and
|
blob record containing the storage location, SHA-256 checksum, byte size, and
|
||||||
content type. Current service paths append a version when connector sync finds
|
content type. A protected blob also records its Encryption envelope, protection
|
||||||
|
discriminator, stored-ciphertext checksum, and stored-ciphertext size. Current
|
||||||
|
service paths append a version when connector sync finds
|
||||||
changed content; they do not mutate the previous version record.
|
changed content; they do not mutate the previous version record.
|
||||||
|
|
||||||
Content with the same tenant, SHA-256 checksum, and size can reuse one blob.
|
Unprotected content with the same tenant, plaintext SHA-256 checksum, size, and
|
||||||
This is storage deduplication and integrity evidence, not proof of authorship or
|
protection discriminator can reuse one blob. Protected content is deduplicated
|
||||||
source authenticity.
|
only inside the same vault/profile discriminator; ciphertext is never silently
|
||||||
|
reused across protection boundaries. Plaintext checksums remain semantic
|
||||||
|
version evidence, while download and integrity scans verify stored ciphertext
|
||||||
|
before asking Encryption to open it. Neither digest proves authorship or source
|
||||||
|
authenticity.
|
||||||
|
|
||||||
The main domain objects are:
|
The main domain objects are:
|
||||||
|
|
||||||
| Object | Meaning | Lifecycle today |
|
| Object | Meaning | Lifecycle today |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| File asset | The user-facing file identity, owner, logical path, description, metadata, and current version | Created, organized, shared, and soft-deleted |
|
| File asset | The user-facing file identity, owner, logical path, description, metadata, current version, retention deadline, legal hold, and lifecycle revision | Created, organized, shared, soft-deleted, restored, or governed-purged |
|
||||||
| File version | A numbered snapshot of one asset and its blob | Appended by connector sync when bytes change; retained |
|
| File version | A numbered snapshot of one asset and its blob | Appended by connector sync when bytes change; retained until its asset is eligible for purge |
|
||||||
| File blob | Stored bytes plus checksum, size, backend, key, reference count, and optional retention timestamp | Reused within a tenant; no automated garbage collection |
|
| File blob | Stored plaintext semantics plus stored-byte integrity, backend key, optional Encryption envelope, reference count, and retention timestamp | Reused only within a tenant and matching protection boundary; collected only after a fenced reference recheck |
|
||||||
| Folder | An explicit logical path in a user or group space | Created, moved/renamed through organize operations, and soft-deleted |
|
| Folder | An explicit logical path in a user or group space | Created, moved/renamed, soft-deleted, and restored when its original path is free |
|
||||||
| Share | A grant from an asset to a user, group, tenant, or campaign with `read`, `write`, or `manage` permission | Created or updated; no public revocation endpoint yet |
|
| Share | A grant from an asset to a user, group, tenant, or campaign with `read`, `write`, or `manage` permission | Created or updated; no public revocation endpoint yet |
|
||||||
| Connector profile | A governed external endpoint, scope, optional credential link, policy, and descriptive capabilities | Created, updated, disabled, or credential-scrubbed on deletion |
|
| Connector profile | A governed external endpoint, scope, optional credential link, policy, and descriptive capabilities | Created, updated, disabled, or credential-scrubbed on deletion |
|
||||||
| Connector credential | Reusable authentication material, optionally limited to a provider and scope | Encrypted when database-managed; immediately scrubbed on deletion |
|
| Connector credential | Reusable authentication material, optionally limited to a provider and scope | Encrypted when database-managed; immediately scrubbed on deletion |
|
||||||
| Connector policy | Allow and deny rules inherited from system through tenant to one leaf scope | Evaluated before configuration and connector use |
|
| Connector policy | Allow and deny rules inherited from system through tenant to one leaf scope | Evaluated before configuration and connector use |
|
||||||
| Connector space | A read-only, manually synchronized remote folder/library linked to a user or group space | Created, updated, disabled, and soft-deleted |
|
| Connector space | A manually synchronized remote folder/library linked to a user or group space; read-only by default, with explicit S3 two-way mode available | Created, updated, disabled, soft-deleted, and restored |
|
||||||
| Campaign attachment use | Evidence connecting a campaign job or entry to an exact asset, version, blob, checksum, and stage | Retained for campaign execution evidence |
|
| Campaign attachment use | Evidence connecting a campaign job or entry to an exact asset, version, blob, checksum, and stage | Retained for campaign execution evidence |
|
||||||
|
| Form evidence upload grant | A one-use, hash-only bearer grant tied to an exact Form instance/revision, purpose, custodian, size, and media-type policy | Issued for at most 15 minutes, consumed by one managed upload, then retained as evidence provenance |
|
||||||
|
|
||||||
|
## Deployment configuration packages
|
||||||
|
|
||||||
|
Files registers the `files.configuration` capability for `managed_storage`
|
||||||
|
fragments. Managed content storage is deployment-owned: the installer selects
|
||||||
|
local persistent storage, managed Garage, or an external S3-compatible service
|
||||||
|
and mounts a validated non-secret infrastructure capability receipt. Files
|
||||||
|
does not copy that endpoint or its credentials into module-owned tables.
|
||||||
|
|
||||||
|
Preflight compares the receipt's `files.storage` capability with the effective
|
||||||
|
runtime backend. It validates backend kind, sanitized S3 endpoint, bucket,
|
||||||
|
Garage management marker or external trust marker, an absolute persistent path
|
||||||
|
for local storage, and the presence of Files-owned `env:` secret references.
|
||||||
|
Secret values are never read into a plan, diagnostic, export, or fragment.
|
||||||
|
|
||||||
|
When runtime and receipt agree, the plan reports `skip`: the desired binding is
|
||||||
|
already effective, and repeated apply is a no-op. A mismatch blocks import and
|
||||||
|
explains which deployment setting must be reconciled. The provider deliberately
|
||||||
|
does not mutate process environment, migrate stored objects, probe remote
|
||||||
|
storage, or reinterpret an infrastructure replacement as safe. Use the Files
|
||||||
|
integrity and Ops checks after deployment and complete migration/recovery review
|
||||||
|
before changing an active backend.
|
||||||
|
|
||||||
## User tasks
|
## User tasks
|
||||||
|
|
||||||
@@ -72,9 +106,15 @@ Files administrator; governed API operations can administer other tenant-owned
|
|||||||
Files resources when their owner is specified. Linked connector spaces appear
|
Files resources when their owner is specified. Linked connector spaces appear
|
||||||
beside managed spaces when they are active and visible to the user.
|
beside managed spaces when they are active and visible to the user.
|
||||||
|
|
||||||
A connector space is intentionally read-only. It is a view of an approved
|
A connector space is read-only by default. It is a view of an approved remote
|
||||||
remote location and a starting point for importing or synchronizing selected
|
location and a starting point for importing a selected file or synchronizing a
|
||||||
files into managed storage; it is not a mounted write-through filesystem.
|
selected file or bounded folder tree into managed storage. Folder sync preserves
|
||||||
|
relative remote paths in the linked owner's managed space and returns a
|
||||||
|
reviewable result for every discovered or deliberately skipped item. An
|
||||||
|
administrator may opt an S3 space into two-way mode
|
||||||
|
only after enabling the profile's `write` capability. Even then, Files exposes
|
||||||
|
explicit, conditional file write-back—not a mounted filesystem. Automatic
|
||||||
|
remote delete, rename, move, and ACL propagation stay disabled.
|
||||||
|
|
||||||
### Upload files
|
### Upload files
|
||||||
|
|
||||||
@@ -94,21 +134,29 @@ An ordinary upload does not append a new version to an existing asset. The
|
|||||||
`overwrite` strategy retires the old asset at that path. Version-preserving
|
`overwrite` strategy retires the old asset at that path. Version-preserving
|
||||||
updates currently belong to connector sync.
|
updates currently belong to connector sync.
|
||||||
|
|
||||||
### Upload and unpack ZIP files
|
### Preview and unpack archives
|
||||||
|
|
||||||
The UI can unpack a ZIP upload. The request body is spooled to a bounded
|
The UI previews ZIP, TAR, TAR.GZ, TAR.BZ2, and TAR.XZ before writing managed
|
||||||
temporary file rather than buffered wholly in memory. The defaults are:
|
files. Users may select individual files or complete folders. The request is
|
||||||
|
spooled to a bounded temporary file rather than buffered wholly in memory. The
|
||||||
|
defaults are:
|
||||||
|
|
||||||
- 250 MiB compressed request and total extracted data;
|
- 250 MiB compressed request;
|
||||||
- 50 MiB per extracted member;
|
- 50 MiB per extracted member;
|
||||||
- 1,000 non-directory members;
|
- 2 GiB total expanded data;
|
||||||
- encrypted archives are rejected;
|
- 10,000 declared entries;
|
||||||
- member paths are normalized and `..` traversal is rejected;
|
- 100:1 maximum expansion ratio;
|
||||||
- the actual bytes read are counted, not only the ZIP header declarations.
|
- a 30-minute preview token bound to the tenant, user, archive digest, and
|
||||||
|
destination;
|
||||||
|
- password-protected ZIP support with request-only password handling;
|
||||||
|
- traversal, duplicate paths, links, devices, and other special entries are
|
||||||
|
rejected;
|
||||||
|
- actual bytes read are counted, not only archive header declarations.
|
||||||
|
|
||||||
The byte limits use `FILE_UPLOAD_ZIP_MAX_BYTES` and
|
The browser retains the selected archive and password until confirmation.
|
||||||
`FILE_UPLOAD_MAX_BYTES`. The 1,000-member limit is currently fixed in the
|
Confirmation reuploads the archive, verifies the token and digest, repeats all
|
||||||
extraction service.
|
safety checks, and commits only the selected files. No preview archive or
|
||||||
|
password is retained server-side.
|
||||||
|
|
||||||
### Organize files and folders
|
### Organize files and folders
|
||||||
|
|
||||||
@@ -129,6 +177,73 @@ Recursive folder deletion is the default. It soft-deletes the selected folder,
|
|||||||
its child folders, and files below it. A non-recursive delete fails when the
|
its child folders, and files below it. A non-recursive delete fails when the
|
||||||
folder is not empty.
|
folder is not empty.
|
||||||
|
|
||||||
|
### Restore, retain, hold, and purge
|
||||||
|
|
||||||
|
Soft deletion is reversible for callers with `files:file:restore`. Restoring a
|
||||||
|
file preserves its asset identity, complete version history, blob references,
|
||||||
|
and source provenance. Recursive folder restore reactivates the folder tree and
|
||||||
|
its deleted files. Connector-space restore reactivates only the local link.
|
||||||
|
Each restore fails if an active resource already occupies the original path or
|
||||||
|
label.
|
||||||
|
|
||||||
|
Retention and legal hold are independent lifecycle controls. A caller with
|
||||||
|
`files:file:retention` supplies the current lifecycle revision, a reason, an
|
||||||
|
optional retained-until time, and legal-hold state. Stale revisions fail rather
|
||||||
|
than overwriting a concurrent decision.
|
||||||
|
|
||||||
|
Hard purge is deliberately separate from ordinary delete:
|
||||||
|
|
||||||
|
1. A caller with `files:file:purge` previews 1–100 soft-deleted assets.
|
||||||
|
2. The preview reports retention, legal-hold, active-share, Campaign-evidence,
|
||||||
|
and Form-evidence blockers and returns a SHA-256 over current lifecycle and
|
||||||
|
blob-reference state.
|
||||||
|
3. Execution requires that exact hash, a stable idempotency key, an approval
|
||||||
|
reference, and the literal `PURGE` confirmation.
|
||||||
|
4. The recovery-ledger operation and tenant purge fence are durable before the
|
||||||
|
irreversible database transaction. A stale preview or blocker rejects the
|
||||||
|
operation without erasure.
|
||||||
|
5. Purge removes eligible asset, version, and inactive-share rows and
|
||||||
|
recalculates blob reference counts. It does not delete bytes inline.
|
||||||
|
6. A separately authorized bounded blob-GC call takes the same per-blob lease
|
||||||
|
used by uploads, rechecks all `FileVersion` references, deletes the exact
|
||||||
|
object, verifies absence, and only then deletes `FileBlob` metadata.
|
||||||
|
|
||||||
|
Automatic time-based purge scheduling is not implemented. Operators initiate
|
||||||
|
preview, execute, and garbage collection under their local retention process.
|
||||||
|
|
||||||
|
### Data-subject request coverage
|
||||||
|
|
||||||
|
Files registers `privacy.dsar.files` when the module is active. The provider
|
||||||
|
requires a corroborated tenant membership identifier (or a namespaced Files
|
||||||
|
user reference), searches only that tenant, and fails explicitly if its bounded
|
||||||
|
result limit would be exceeded. It reports managed assets, exact versions,
|
||||||
|
folders, user-targeted shares, Form and Campaign evidence, connector
|
||||||
|
configuration actor references, and integrity-operation evidence.
|
||||||
|
|
||||||
|
The export includes only governed metadata. It never embeds file bytes, blob
|
||||||
|
storage keys, passwords, tokens, environment-variable names, secret-provider
|
||||||
|
references, or encrypted connector values. A reviewer follows the authorized
|
||||||
|
version download route when the file itself must be inspected.
|
||||||
|
|
||||||
|
The erasure plan deliberately separates four outcomes:
|
||||||
|
|
||||||
|
- an active share aimed at the subject can be revoked idempotently;
|
||||||
|
- mutable creator/updater references can be detached after tenant, subject, and
|
||||||
|
current-value revalidation;
|
||||||
|
- legal hold, active retention, submitted Form evidence, Campaign delivery
|
||||||
|
evidence, connector configuration history, and integrity evidence are
|
||||||
|
retained with a reason; and
|
||||||
|
- unstructured file content, ownership, filenames, and paths require manual
|
||||||
|
review.
|
||||||
|
|
||||||
|
DSAR execution never invokes physical byte deletion. If the privacy decision
|
||||||
|
authorizes erasure, the operator must use the separate Files soft-delete, purge
|
||||||
|
preview, approval, execution, and blob-GC sequence. This preserves its distinct
|
||||||
|
authority, evidence blockers, audit trail, distributed fencing, and recovery
|
||||||
|
ledger semantics. Email, account, or identity selectors alone are insufficient
|
||||||
|
because Files does not import the Access directory; the Access search supplies
|
||||||
|
the corroborated membership reference for Files coverage.
|
||||||
|
|
||||||
### Find and download files
|
### Find and download files
|
||||||
|
|
||||||
Files can list by owner and path, use cursor pagination, and consume incremental
|
Files can list by owner and path, use cursor pagination, and consume incremental
|
||||||
@@ -169,7 +284,35 @@ inside the chosen owner space:
|
|||||||
- identical checksum and size updates provenance and returns `unchanged`;
|
- identical checksum and size updates provenance and returns `unchanged`;
|
||||||
- changed bytes append a version and return `updated`.
|
- changed bytes append a version and return `updated`.
|
||||||
|
|
||||||
Browse, import, and sync never write, rename, or delete the remote source.
|
Folder-level manual sync uses
|
||||||
|
`POST /api/v1/files/connector-spaces/{space_id}/sync`. The interactive request
|
||||||
|
is bounded by file count and traversal depth, follows provider pagination, and
|
||||||
|
can include or exclude subfolders. Every file is isolated in a database
|
||||||
|
savepoint, so one conflict, policy denial, invalid provider result, or transport
|
||||||
|
failure remains reviewable without discarding successful siblings. Results are
|
||||||
|
classified as `created`, `updated`, `unchanged`, `skipped`, `conflict`,
|
||||||
|
`policy_denied`, or `failed`. The UI defaults unrelated target collisions to
|
||||||
|
`skip`; `rename`, `reject`, and `overwrite` require an explicit operator choice.
|
||||||
|
The response marks a run as truncated when its bound is reached so the operator
|
||||||
|
can continue with a narrower remote folder. This is interactive orchestration,
|
||||||
|
not background scheduling.
|
||||||
|
|
||||||
|
Browse, import, and inbound sync never mutate the remote source. An S3 space in
|
||||||
|
explicit two-way mode can write a selected managed file to one remote object
|
||||||
|
path through `POST /api/v1/files/connector-spaces/{space_id}/write-back`.
|
||||||
|
Creating a path uses `If-None-Match`; overwriting requires the currently
|
||||||
|
observed ETag or version and uses a conditional request. The provider object is
|
||||||
|
then re-read and must contain both the expected content digest and the recovery
|
||||||
|
operation marker. Uncertain or mismatching outcomes remain fenced and visible
|
||||||
|
in Ops. Remote deletion, rename, move, and permission propagation are disabled.
|
||||||
|
|
||||||
|
Connector administration separates endpoint profiles, reusable credentials,
|
||||||
|
and inherited policy. Ordinary setup uses typed fields and provider discovery;
|
||||||
|
provider metadata JSON is available only under advanced compatibility options.
|
||||||
|
Read-only deployment entries explain where they must be changed, and disabled
|
||||||
|
actions identify the missing permission, target, input, or running operation.
|
||||||
|
The contextual help icon opens the configured Help Center topic when Docs is
|
||||||
|
enabled and the hosted GovOPlaN documentation otherwise.
|
||||||
|
|
||||||
## Process perspective
|
## Process perspective
|
||||||
|
|
||||||
@@ -187,9 +330,10 @@ The managed-file flow is:
|
|||||||
8. Connector-originated operations also emit their connector audit evidence.
|
8. Connector-originated operations also emit their connector audit evidence.
|
||||||
|
|
||||||
Blob storage is not part of the database transaction. An object may therefore
|
Blob storage is not part of the database transaction. An object may therefore
|
||||||
be left without committed metadata after a process or database failure. No
|
be left without committed metadata after a process or database failure. The
|
||||||
automatic orphan reconciliation exists yet; operators must account for this in
|
operator integrity API scans database blobs and the tenant storage prefix in
|
||||||
integrity checks and retention plans.
|
bounded, resumable phases. It reports orphan objects before any cleanup and
|
||||||
|
never deletes them as part of a scan.
|
||||||
|
|
||||||
### Governed connector import
|
### Governed connector import
|
||||||
|
|
||||||
@@ -266,10 +410,14 @@ those relationships first.
|
|||||||
| --- | --- |
|
| --- | --- |
|
||||||
| `files:file:read` | List and inspect accessible files, folders, spaces, and visible connectors |
|
| `files:file:read` | List and inspect accessible files, folders, spaces, and visible connectors |
|
||||||
| `files:file:download` | Download an accessible current version or ZIP archive |
|
| `files:file:download` | Download an accessible current version or ZIP archive |
|
||||||
| `files:file:upload` | Upload managed assets and import/sync selected connector files |
|
| `files:file:upload` | Upload managed assets and import/sync selected connector files or bounded connector folders |
|
||||||
| `files:file:organize` | Create folders, rename, move/copy, and manage linked connector spaces |
|
| `files:file:organize` | Create folders, rename, move/copy, and manage linked connector spaces |
|
||||||
| `files:file:share` | Create or update file shares |
|
| `files:file:share` | Create or update file shares |
|
||||||
| `files:file:delete` | Soft-delete accessible writable files and folders |
|
| `files:file:delete` | Soft-delete accessible writable files and folders |
|
||||||
|
| `files:file:restore` | Restore owned or administered soft-deleted files, folders, and connector-space links |
|
||||||
|
| `files:file:retention` | Set retention deadlines and legal holds with optimistic revision checks |
|
||||||
|
| `files:file:purge` | Preview and execute irreversible metadata purge and collect unreferenced blobs |
|
||||||
|
| `files:connector:write` | Write an accessible managed file to an explicitly writable connector space |
|
||||||
| `files:file:admin` | Administer all Files spaces and connector settings in the active tenant |
|
| `files:file:admin` | Administer all Files spaces and connector settings in the active tenant |
|
||||||
|
|
||||||
The `file_manager` role template grants all normal file operations except
|
The `file_manager` role template grants all normal file operations except
|
||||||
@@ -289,12 +437,14 @@ Keep these definitions separate:
|
|||||||
local policy, and descriptive operation capabilities;
|
local policy, and descriptive operation capabilities;
|
||||||
- a **policy** restricts what a scope may configure or use;
|
- a **policy** restricts what a scope may configure or use;
|
||||||
- a **connector space** links one approved profile/library/path to one user or
|
- a **connector space** links one approved profile/library/path to one user or
|
||||||
group and always uses manual, read-only synchronization today.
|
group, uses manual synchronization, and is read-only unless explicitly
|
||||||
|
configured for the supported two-way S3 write boundary.
|
||||||
|
|
||||||
Profile capability values such as `browse`, `import`, and `sync` are stored and
|
Profile capability values such as `browse`, `import`, `sync`, and `write` are
|
||||||
returned, but they are descriptive today. Provider implementation and policy
|
stored and returned. `write` is additionally enforced for S3 write-back, but
|
||||||
checks enforce actual availability; do not use the capability list as the sole
|
provider implementation, connector-space mode, operation permission, resource
|
||||||
security control.
|
access, and inherited policy all remain mandatory; never use the capability
|
||||||
|
list as the sole security control.
|
||||||
|
|
||||||
Profiles, credentials, and policies support `system`, `tenant`, `user`,
|
Profiles, credentials, and policies support `system`, `tenant`, `user`,
|
||||||
`group`, and `campaign` scopes. A normal user sees system and active-tenant
|
`group`, and `campaign` scopes. A normal user sees system and active-tenant
|
||||||
@@ -378,15 +528,15 @@ in a development/test runtime.
|
|||||||
| Seafile | Read-only native API browse/download-link import and manual sync implemented using the pinned HTTP transport; WebDAV opt-in supported |
|
| Seafile | Read-only native API browse/download-link import and manual sync implemented using the pinned HTTP transport; WebDAV opt-in supported |
|
||||||
| Nextcloud | Read-only WebDAV browse/import/manual sync implemented using the pinned HTTP transport |
|
| Nextcloud | Read-only WebDAV browse/import/manual sync implemented using the pinned HTTP transport |
|
||||||
| Generic WebDAV | Read-only browse/import/manual sync implemented using the pinned HTTP transport |
|
| Generic WebDAV | Read-only browse/import/manual sync implemented using the pinned HTTP transport |
|
||||||
| SMB | Browse/import code and descriptor implemented, but every live connection fails closed until initial connections and DFS referrals can be policy-validated and pinned |
|
| SMB | Read-only browse/import/manual sync implemented through a pinned smbprotocol transport for initial peers, reconnects, aliases, and DFS referral targets |
|
||||||
| S3 connector | Browse/import code and descriptor implemented, but every live SDK connection fails closed until botocore connections, redirects, and endpoint discovery can be validated and pinned |
|
| S3 connector | Bucket/prefix browse, import, manual inbound sync, and explicit conditional write-back implemented through pinned botocore pools covering retries, redirects, endpoint discovery, and provider aliases; automatic remote delete/rename/move/ACL propagation disabled |
|
||||||
| SharePoint and OneDrive | Provider keys/descriptors reserved; live Microsoft Graph browse/import is planned |
|
| SharePoint and OneDrive | Provider keys/descriptors reserved; live Microsoft Graph browse/import is planned |
|
||||||
| NFS and local connector | Described as optional future providers; the local managed-storage backend is a different feature |
|
| NFS and local connector | Described as optional future providers; the local managed-storage backend is a different feature |
|
||||||
|
|
||||||
Provider descriptors are available from
|
Provider descriptors are available from
|
||||||
`GET /api/v1/files/connectors/providers`. Use their `implemented`, `installed`,
|
`GET /api/v1/files/connectors/providers`. Use their `implemented`, `installed`,
|
||||||
and support fields for display, but expect a fail-closed transport error where
|
and support fields for display. An incompatible optional SDK release fails closed
|
||||||
the table above says live access is disabled.
|
before it can return a usable client or session.
|
||||||
|
|
||||||
## Operator runbook
|
## Operator runbook
|
||||||
|
|
||||||
@@ -398,8 +548,15 @@ the table above says live access is disabled.
|
|||||||
| `FILE_STORAGE_LOCAL_ROOT` | `runtime/files` | Primary local write/read root |
|
| `FILE_STORAGE_LOCAL_ROOT` | `runtime/files` | Primary local write/read root |
|
||||||
| `FILE_STORAGE_LOCAL_FALLBACK_ROOTS` | empty | Comma-separated older read-only roots checked after the primary root |
|
| `FILE_STORAGE_LOCAL_FALLBACK_ROOTS` | empty | Comma-separated older read-only roots checked after the primary root |
|
||||||
| `FILE_STORAGE_S3_ENDPOINT_URL` and related `FILE_STORAGE_S3_*` values | deployment-specific | S3-compatible endpoint, region, credentials, and bucket |
|
| `FILE_STORAGE_S3_ENDPOINT_URL` and related `FILE_STORAGE_S3_*` values | deployment-specific | S3-compatible endpoint, region, credentials, and bucket |
|
||||||
| `FILE_UPLOAD_MAX_BYTES` | 50 MiB | Direct-upload and ZIP-member maximum |
|
| `FILE_STORAGE_S3_DEPLOYMENT_MANAGED` | `false` | Installer-only trust marker for the exact `http://garage:3900` service; never use it for another endpoint |
|
||||||
| `FILE_UPLOAD_ZIP_MAX_BYTES` | 250 MiB | ZIP request and extracted-total maximum |
|
| `FILE_STORAGE_S3_ENDPOINT_TRUSTED` | `false` | Deployment-owner acknowledgement for one clean HTTPS external S3 origin; never expose this through connector configuration |
|
||||||
|
| `GOVOPLAN_STATE_PROFILE` | `local` | Selects `local`, one-host `host-shared`, or multi-host `shared` state validation |
|
||||||
|
| `FILE_UPLOAD_MAX_BYTES` | 50 MiB | Direct-upload and extracted archive-member maximum |
|
||||||
|
| `FILE_UPLOAD_ZIP_MAX_BYTES` | 250 MiB | Compressed archive request maximum (legacy name retained for compatibility) |
|
||||||
|
| `FILE_ARCHIVE_MAX_ENTRIES` | 10,000 | Maximum declared archive entries |
|
||||||
|
| `FILE_ARCHIVE_MAX_EXPANDED_BYTES` | 2 GiB | Maximum expanded archive bytes |
|
||||||
|
| `FILE_ARCHIVE_MAX_EXPANSION_RATIO` | 100 | Maximum expanded-to-compressed ratio |
|
||||||
|
| `FILE_ARCHIVE_PREVIEW_TTL_SECONDS` | 1,800 | Lifetime of the sealed archive preview token |
|
||||||
| `MASTER_KEY_B64` | development fallback only | Encrypts database-managed connector secrets |
|
| `MASTER_KEY_B64` | development fallback only | Encrypts database-managed connector secrets |
|
||||||
|
|
||||||
The local backend is the operational baseline. It resolves every storage key
|
The local backend is the operational baseline. It resolves every storage key
|
||||||
@@ -407,16 +564,19 @@ under the configured root and rejects escape attempts. Fallback roots support a
|
|||||||
controlled storage-root migration: new writes go to the primary root while
|
controlled storage-root migration: new writes go to the primary root while
|
||||||
reads can still find older objects.
|
reads can still find older objects.
|
||||||
|
|
||||||
The S3 managed-storage adapter currently fails closed before creating a boto3
|
The supported installer may provision a deployment-owned Garage service at the
|
||||||
client because the SDK cannot yet guarantee connection-time DNS/IP pinning and
|
exact `http://garage:3900` endpoint and set
|
||||||
redirect revalidation. Do not select `FILE_STORAGE_BACKEND=s3` for a live Files
|
`FILE_STORAGE_S3_DEPLOYMENT_MANAGED=true`. An operator-selected external S3
|
||||||
deployment until that boundary is implemented and the status above changes.
|
backend instead requires a clean HTTPS origin and
|
||||||
|
`FILE_STORAGE_S3_ENDPOINT_TRUSTED=true`. Both are deployment authority, not a
|
||||||
|
general connector or private-network bypass. Core owns the shared backend
|
||||||
|
implementation; Files owns metadata and the Files key namespace.
|
||||||
|
|
||||||
Multiple API replicas require the same durable blob namespace. Separate local
|
Multiple API replicas require the same durable blob namespace. Separate local
|
||||||
container filesystems will produce incomplete reads. Until a pinned shared
|
container filesystems will produce incomplete reads. Use `host-shared` with one
|
||||||
object-storage transport is available, use one durable shared mount or constrain
|
durable shared mount only for same-host replicas. Independent hosts require the
|
||||||
Files traffic to a deployment topology that preserves one consistent local
|
`shared` profile with external S3, PostgreSQL, Redis, a stable installation id,
|
||||||
root.
|
and one immutable module composition.
|
||||||
|
|
||||||
### Connector egress
|
### Connector egress
|
||||||
|
|
||||||
@@ -441,15 +601,29 @@ The built-in HTTP transport:
|
|||||||
- bounds structured responses to 16 MiB and file transfers to 512 MiB by
|
- bounds structured responses to 16 MiB and file transfers to 512 MiB by
|
||||||
default.
|
default.
|
||||||
|
|
||||||
|
The S3 SDK adapter applies the same socket rule to every botocore pool selected
|
||||||
|
for a retry, redirect, discovered endpoint, or virtual-host bucket alias. The
|
||||||
|
original authority remains in the request and TLS SNI/certificate check. S3
|
||||||
|
connector clients use no outbound proxy and never discover ambient AWS
|
||||||
|
credentials: configure both access and secret keys on the governed profile, or
|
||||||
|
use an anonymous profile for a public source.
|
||||||
|
|
||||||
|
The SMB adapter owns a separate connection cache and replaces smbprotocol's TCP
|
||||||
|
factory process-wide with the stricter pinned socket. Initial peers, reconnects,
|
||||||
|
server aliases, domain-controller connections, and DFS referral targets therefore
|
||||||
|
pass the same policy at connection time. Signing is required by default; enable
|
||||||
|
SMB encryption on the profile where the server supports it.
|
||||||
|
|
||||||
Override the connector response limits with
|
Override the connector response limits with
|
||||||
`GOVOPLAN_CONNECTOR_MAX_STRUCTURED_RESPONSE_BYTES` and
|
`GOVOPLAN_CONNECTOR_MAX_STRUCTURED_RESPONSE_BYTES` and
|
||||||
`GOVOPLAN_CONNECTOR_MAX_FILE_TRANSFER_BYTES`. The smaller applicable limit wins
|
`GOVOPLAN_CONNECTOR_MAX_FILE_TRANSFER_BYTES`. The smaller applicable limit wins
|
||||||
when an import is also subject to `FILE_UPLOAD_MAX_BYTES`.
|
when an import is also subject to `FILE_UPLOAD_MAX_BYTES`.
|
||||||
|
|
||||||
Never work around a pinning failure by adding a raw IP, disabling TLS, or
|
Never work around a connector pinning failure by adding a raw IP, disabling TLS,
|
||||||
enabling private networks. SMB may redirect through DFS, and S3 SDKs may perform
|
or enabling private networks. A failure means the peer policy rejected an actual
|
||||||
their own redirects or endpoint discovery; both remain disabled even for an IP
|
connection destination or the installed SDK no longer exposes the verified
|
||||||
literal until every connection peer can be governed.
|
transport seam. The separately configured platform S3 backend is trusted only by
|
||||||
|
the deployment owner and is not selectable by a user or connector profile.
|
||||||
|
|
||||||
### Backup and restore
|
### Backup and restore
|
||||||
|
|
||||||
@@ -459,13 +633,113 @@ include:
|
|||||||
- Files database rows, including asset/version/blob relationships, shares,
|
- Files database rows, including asset/version/blob relationships, shares,
|
||||||
connector settings, and campaign attachment-use evidence;
|
connector settings, and campaign attachment-use evidence;
|
||||||
- every object below `FILE_STORAGE_LOCAL_ROOT` and any still-used fallback root;
|
- every object below `FILE_STORAGE_LOCAL_ROOT` and any still-used fallback root;
|
||||||
|
or the complete S3 bucket/prefix and version/lifecycle evidence for an S3
|
||||||
|
backend;
|
||||||
- the exact `MASTER_KEY_B64` needed to decrypt retained connector credentials;
|
- the exact `MASTER_KEY_B64` needed to decrypt retained connector credentials;
|
||||||
- deployment-owned connector profile files, referenced CA bundles, and secret
|
- deployment-owned connector profile files, referenced CA bundles, and secret
|
||||||
environment configuration where those definitions are in use.
|
environment configuration where those definitions are in use.
|
||||||
|
|
||||||
There is no Files backup/restore API and no automated blob integrity or orphan
|
There is no Files backup/restore API. Use a write quiesce or coordinated
|
||||||
reconciliation job. Use a write quiesce or coordinated snapshots so database
|
snapshots so database references and objects represent the same recovery point.
|
||||||
references and objects represent the same recovery point.
|
The integrity API verifies a restored set, but it does not replace a coordinated
|
||||||
|
backup.
|
||||||
|
|
||||||
|
Operators normally use **Administration > File integrity**. The equivalent API
|
||||||
|
creates a scan with `POST /api/v1/files/integrity/scans`, then calls
|
||||||
|
`POST /api/v1/files/integrity/scans/{scan_id}/run` with the scan's current
|
||||||
|
`expected_revision` until it reports `completed`. Each call advances at most
|
||||||
|
the persisted batch size, so a stopped operator or worker can resume from the
|
||||||
|
committed blob/object cursors. Concurrent or stale actions receive `409` before
|
||||||
|
the storage backend is invoked; reload the scan and inspect the newer state.
|
||||||
|
|
||||||
|
Findings distinguish:
|
||||||
|
|
||||||
|
- `missing`: metadata references an absent object;
|
||||||
|
- `size_mismatch` or `checksum_mismatch`: bytes do not match immutable blob
|
||||||
|
metadata and the blob is quarantined;
|
||||||
|
- `orphan_object`: an object exists in the tenant Files prefix without a
|
||||||
|
corresponding blob row.
|
||||||
|
|
||||||
|
Missing or corrupt blobs fail closed for ordinary downloads and Campaign
|
||||||
|
attachment materialization. After restoring the expected bytes, use the finding
|
||||||
|
`recheck` action with its current `expected_revision`. Orphan cleanup starts
|
||||||
|
with a dry-run preview and requires separate destructive confirmation. The
|
||||||
|
confirmation reuses the finding revision from that preview, rechecks that no
|
||||||
|
database reference exists, remains scoped to the scanned tenant prefix, and is
|
||||||
|
idempotent. Both applied and dry-run actions emit audit evidence. A shared
|
||||||
|
reference blocks orphan cleanup. Managed-asset purge separately enforces asset
|
||||||
|
retention, legal hold, active shares, and retained Campaign/Form evidence before
|
||||||
|
releasing a blob for fenced garbage collection.
|
||||||
|
|
||||||
|
### Recovery ledger for object effects
|
||||||
|
|
||||||
|
On PostgreSQL, every managed blob creation or integrity repair starts a Core
|
||||||
|
recovery operation in an independent committed transaction before Files
|
||||||
|
protects or writes bytes. The operation records tenant/blob identifiers, an opaque object
|
||||||
|
locator or locator digest, semantic SHA-256/size evidence, the recovery mode,
|
||||||
|
and a distributed lease fence. It never records file contents, ZIP passwords,
|
||||||
|
connector credentials, or a newly uploaded filename. New object keys are opaque;
|
||||||
|
legacy filename-bearing keys remain readable but repair operations record only
|
||||||
|
their digest and recover through the blob ID.
|
||||||
|
|
||||||
|
SQLite is a supported local-development database but permits only one writer.
|
||||||
|
Files therefore uses an explicit reduced-durability mode there: recovery intent
|
||||||
|
and its lease are written in the caller transaction, while a process-local
|
||||||
|
fence prevents competing effects in the same runtime. Commit makes the intent
|
||||||
|
durable before independent verification. A handled rollback reconstructs a
|
||||||
|
durable recovery operation and verifies compensation or forward completion.
|
||||||
|
A hard process loss before commit can leave an object without a surviving
|
||||||
|
ledger row, so SQLite is not a production recovery profile; after such a loss,
|
||||||
|
run a complete Files integrity scan and reconcile every reported orphan before
|
||||||
|
resuming writes. PostgreSQL retains the independent pre-effect durability
|
||||||
|
guarantee.
|
||||||
|
|
||||||
|
The Files business transaction then creates or updates the blob, version, and
|
||||||
|
asset rows. Its actual SQLAlchemy commit or rollback settles every pending
|
||||||
|
operation:
|
||||||
|
|
||||||
|
- commit reloads the blob through an independent session and streams the object
|
||||||
|
to verify its stored-byte SHA-256 and size before recording success;
|
||||||
|
- rollback deletes only a newly reserved object after independently proving
|
||||||
|
that no `FileBlob` references it, then records verified compensation;
|
||||||
|
- a repaired existing object is forward-completed only when its identity,
|
||||||
|
envelope, semantic evidence, and stored bytes all match;
|
||||||
|
- missing or mismatched bytes quarantine a committed blob and leave the
|
||||||
|
operation `recovery_required`; an unavailable probe remains
|
||||||
|
`outcome_unknown` rather than becoming an ordinary upload failure.
|
||||||
|
|
||||||
|
Applied orphan cleanup has its own forward-recovery operation. The database
|
||||||
|
reference check and tenant-prefix check happen before deletion; object absence
|
||||||
|
and the durable finding state are verified afterward. If the caller transaction
|
||||||
|
rolls back after deletion, Files may forward-complete only that existing
|
||||||
|
finding after rechecking that the key is still unreferenced.
|
||||||
|
|
||||||
|
Hard purge uses an irreversible Core recovery plan with an approval reference
|
||||||
|
and a tenant-wide purge lease. The preview hash binds target IDs, deletion
|
||||||
|
state, lifecycle revisions, holds, deadlines, blockers, and blob IDs. Asset and
|
||||||
|
version removal plus audit evidence commit together. Released objects are not
|
||||||
|
deleted by that transaction. Blob GC later takes `files:blob:<tenant>:<blob>`,
|
||||||
|
the same distributed resource fence as upload/repair, locks and rechecks the
|
||||||
|
blob, and verifies object absence before removing metadata. Provider or database
|
||||||
|
ambiguity remains recovery-required or outcome-unknown in Ops.
|
||||||
|
|
||||||
|
Explicit S3 connector writes use forward recovery because the provider cannot
|
||||||
|
join the database transaction. The durable request contains only tenant,
|
||||||
|
profile, opaque target digest, content SHA-256/size, and expected revision. A
|
||||||
|
conditional create or overwrite is followed by a metadata probe for the exact
|
||||||
|
content and operation markers. Never retry an unresolved target with another
|
||||||
|
request key; reconcile the owning Files operation from provider evidence first.
|
||||||
|
|
||||||
|
Archive preview and confirmation use bounded process-local temporary staging.
|
||||||
|
Staging is not authoritative and is removed on every handled exit; extracted
|
||||||
|
members enter the same per-blob recovery boundary as direct uploads. A hard
|
||||||
|
process loss may leave a temporary OS file for normal host temporary-file
|
||||||
|
cleanup, but cannot make that staging path a managed Files object.
|
||||||
|
|
||||||
|
Use the Ops recovery-operation view to inspect `files` operations. Do not retry
|
||||||
|
a busy or unresolved blob or connector path blindly: first verify the FileBlob
|
||||||
|
row and object hash, or the remote request/content markers and revision, plus
|
||||||
|
any Encryption envelope named by a managed blob.
|
||||||
|
|
||||||
After restore:
|
After restore:
|
||||||
|
|
||||||
@@ -493,9 +767,10 @@ Legacy external secret references are detached and identified as non-owned;
|
|||||||
Files never calls a provider delete operation for them.
|
Files never calls a provider delete operation for them.
|
||||||
|
|
||||||
The retirement executor drops database tables but does not delete corresponding
|
The retirement executor drops database tables but does not delete corresponding
|
||||||
objects from the configured blob backend. Operators must include those orphaned
|
objects from the configured blob backend. Operators must include those objects
|
||||||
objects in the approved retention/destruction plan. Validate the installer
|
in the approved retention/destruction plan, report them with an integrity scan,
|
||||||
snapshot and independent blob backup before retirement.
|
and explicitly approve cleanup. Validate the installer snapshot and independent
|
||||||
|
blob backup before retirement.
|
||||||
|
|
||||||
### Operational signals
|
### Operational signals
|
||||||
|
|
||||||
@@ -506,7 +781,7 @@ Use these symptoms as routing hints:
|
|||||||
| `Stored object does not exist` | Database/blob restore mismatch, wrong root, or missing shared storage |
|
| `Stored object does not exist` | Database/blob restore mismatch, wrong root, or missing shared storage |
|
||||||
| `Stored secret cannot be decrypted` | Wrong or rotated `MASTER_KEY_B64` |
|
| `Stored secret cannot be decrypted` | Wrong or rotated `MASTER_KEY_B64` |
|
||||||
| Private/non-public endpoint blocked | Deployment-wide egress policy is public-only or DNS returned a forbidden answer |
|
| Private/non-public endpoint blocked | Deployment-wide egress policy is public-only or DNS returned a forbidden answer |
|
||||||
| SDK cannot pin redirects/referrals | Expected fail-closed S3/SMB boundary, not a transient connector outage |
|
| SDK peer-pinning seam is unavailable | Optional S3/SMB SDK is incompatible; keep access fail-closed and validate the supported dependency range before upgrade |
|
||||||
| Connector response exceeds limit | Remote payload exceeds connector or upload limit |
|
| Connector response exceeds limit | Remote payload exceeds connector or upload limit |
|
||||||
| Profile is not visible | Scope, disabled state, campaign access, or policy mismatch |
|
| Profile is not visible | Scope, disabled state, campaign access, or policy mismatch |
|
||||||
| Group removal is vetoed | The group still owns a file/folder/connector space or is a share target |
|
| Group removal is vetoed | The group still owns a file/folder/connector space or is a share target |
|
||||||
@@ -522,11 +797,26 @@ public HTTP API. They must not import Files ORM models or storage helpers.
|
|||||||
| --- | --- |
|
| --- | --- |
|
||||||
| `files.access` (`0.1.6`) | Explain resource access provenance for managed files, explicit folders, and virtual folders |
|
| `files.access` (`0.1.6`) | Explain resource access provenance for managed files, explicit folders, and virtual folders |
|
||||||
| `files.campaign_attachments` (`0.1.6`) | Resolve managed attachment matches, prepare frozen campaign snapshots, annotate built messages, share assets with a campaign, and record/mark exact attachment use |
|
| `files.campaign_attachments` (`0.1.6`) | Resolve managed attachment matches, prepare frozen campaign snapshots, annotate built messages, share assets with a campaign, and record/mark exact attachment use |
|
||||||
|
| `records.source.files` (`1.0.0`) | Recheck current Files access and resolve one exact, integrity-approved managed file version for Records filing |
|
||||||
|
| `forms_runtime.evidence.files` (`1.0.0`) | Issue a one-time managed attachment grant and re-verify the exact file/version/checksum at Form submission |
|
||||||
|
|
||||||
Files requires Core principal resolution and permission evaluation. Campaign is
|
Files requires Core principal resolution and permission evaluation. Campaign is
|
||||||
an optional dependency; when installed, Files consumes the optional
|
an optional dependency; when installed, Files consumes the optional
|
||||||
`campaigns.access` interface to verify campaign existence and access. Missing
|
`campaigns.access` interface to verify campaign existence and access. Missing
|
||||||
optional Campaign support fails explicitly rather than bypassing the check.
|
optional Campaign support fails explicitly rather than bypassing the check.
|
||||||
|
Records is also optional. When enabled, the source capability returns the
|
||||||
|
requested `FileVersion` identity, path snapshot, content metadata, SHA-256,
|
||||||
|
integrity/protection state, and launch link. It rejects mutable aliases,
|
||||||
|
cross-tenant requests, missing access, and quarantined or failed blobs. Records
|
||||||
|
stores the filing decision; Files continues to own the version and bytes.
|
||||||
|
Forms Runtime is optional as well. Its public or authenticated participant never
|
||||||
|
receives general Files access through this integration. Forms Runtime asks for a
|
||||||
|
purpose-bound grant, Files verifies an active same-tenant user custodian, stores
|
||||||
|
only the token digest, accepts one bounded upload, and returns an immutable
|
||||||
|
`EvidenceReference`. Draft save and final submit independently recheck the exact
|
||||||
|
Form instance/revision, grant, asset, version, checksum, deletion state, and
|
||||||
|
integrity state. An idempotent replay returns the existing grant without
|
||||||
|
reproducing its bearer secret.
|
||||||
|
|
||||||
### API families
|
### API families
|
||||||
|
|
||||||
@@ -535,17 +825,32 @@ All routes below are under `/api/v1/files`.
|
|||||||
| Area | Routes |
|
| Area | Routes |
|
||||||
| --- | --- |
|
| --- | --- |
|
||||||
| Spaces and content | `GET /spaces`, `GET /`, `GET /folders`, `GET /delta` |
|
| Spaces and content | `GET /spaces`, `GET /`, `GET /folders`, `GET /delta` |
|
||||||
| Upload and folders | `POST /upload`, `POST /upload-zip`, `POST /folders`, `POST /folders/delete` |
|
| Upload and folders | `POST /upload`, `POST /upload-zip` (compatibility), `POST /archive-preview`, `POST /archive-confirm`, `POST /folders`, `POST /folders/delete` |
|
||||||
| File access | `GET /{file_id}`, `GET /{file_id}/download`, `DELETE /{file_id}`, `POST /bulk-delete` |
|
| File access | `GET /{file_id}`, `GET /{file_id}/download`, `DELETE /{file_id}`, `POST /bulk-delete` |
|
||||||
| Organization | `POST /bulk-rename`, `POST /transfer`, `POST /archive.zip`, `POST /resolve-patterns` |
|
| Organization | `POST /bulk-rename`, `POST /transfer`, `POST /archive.zip`, `POST /resolve-patterns` |
|
||||||
| Sharing | `POST /{file_id}/shares`, `POST /bulk-shares` |
|
| Sharing | `POST /{file_id}/shares`, `POST /bulk-shares` |
|
||||||
| Connector spaces | `GET/POST /connector-spaces`, `PATCH/DELETE /connector-spaces/{space_id}` |
|
| Connector spaces | `GET/POST /connector-spaces`, `PATCH/DELETE /connector-spaces/{space_id}`, `POST /connector-spaces/{space_id}/restore`, `POST /connector-spaces/{space_id}/sync`, `POST /connector-spaces/{space_id}/write-back` |
|
||||||
| Connector catalog/discovery | `GET /connectors/providers`, `POST /connectors/discover` |
|
| Connector catalog/discovery | `GET /connectors/providers`, `POST /connectors/discover` |
|
||||||
| Connector profiles | `GET/POST /connectors/profiles`, `GET/PATCH/DELETE /connectors/profiles/{profile_id}` |
|
| Connector profiles | `GET/POST /connectors/profiles`, `GET/PATCH/DELETE /connectors/profiles/{profile_id}` |
|
||||||
| Browse/import/sync | `GET /connectors/profiles/{profile_id}/browse`, `POST /connectors/profiles/{profile_id}/import`, `POST /connectors/profiles/{profile_id}/sync` |
|
| Browse/import/sync | `GET /connectors/profiles/{profile_id}/browse`, `POST /connectors/profiles/{profile_id}/import`, `POST /connectors/profiles/{profile_id}/sync` |
|
||||||
|
| Restore and lifecycle | `POST /assets/{file_id}/restore`, `POST /folders/restore`, `PATCH /{file_id}/lifecycle` |
|
||||||
|
| Governed erasure | `POST /purge/preview`, `POST /purge/execute`, `POST /purge/blobs` |
|
||||||
| Credentials | `GET/POST /connectors/credentials`, `GET/PATCH/DELETE /connectors/credentials/{credential_id}` |
|
| Credentials | `GET/POST /connectors/credentials`, `GET/PATCH/DELETE /connectors/credentials/{credential_id}` |
|
||||||
| Policy | `GET/PUT /connectors/policies/{scope_type}`, `POST /connector-policy/evaluate` |
|
| Policy | `GET/PUT /connectors/policies/{scope_type}`, `POST /connector-policy/evaluate` |
|
||||||
| Incremental connector settings | `GET /connectors/settings/delta` |
|
| Incremental connector settings | `GET /connectors/settings/delta` |
|
||||||
|
| Form evidence | `POST /form-evidence/upload` with a short-lived `X-Form-Evidence-Token` issued by Forms Runtime |
|
||||||
|
|
||||||
|
The Files workspace exposes **Remove space** only for connector spaces and only
|
||||||
|
to actors with file-organization authority over the owning user
|
||||||
|
or group space. Confirmation explains the exact boundary: removal soft-deletes
|
||||||
|
the local connector-space definition and makes that virtual view disappear. It
|
||||||
|
does not mutate or delete remote provider content, previously imported managed
|
||||||
|
files, their metadata or shares, the connector profile, credentials, or remote
|
||||||
|
object references. Those retained objects therefore do not block removal and
|
||||||
|
the connector location can be linked again later. User and group managed spaces
|
||||||
|
are intrinsic ownership scopes rather than removable records, so they never
|
||||||
|
offer this action. A missing, already removed, cross-tenant, or inaccessible
|
||||||
|
connector space fails through the backend lookup and owner-access checks.
|
||||||
|
|
||||||
Consumers should use cursor/watermark contracts instead of assuming an
|
Consumers should use cursor/watermark contracts instead of assuming an
|
||||||
unbounded complete list. The default full-list page size is 500 and public page
|
unbounded complete list. The default full-list page size is 500 and public page
|
||||||
@@ -580,21 +885,32 @@ Files baseline indiscriminately.
|
|||||||
- Owner and share checks protect individual resources after operation-scope
|
- Owner and share checks protect individual resources after operation-scope
|
||||||
checks.
|
checks.
|
||||||
- Logical paths reject traversal and storage keys cannot escape the local root.
|
- Logical paths reject traversal and storage keys cannot escape the local root.
|
||||||
- Upload, ZIP extraction, connector response, and S3 stream code use bounded
|
- Upload, archive extraction, connector response, and S3 stream code use bounded
|
||||||
reads; encrypted ZIP archives are rejected.
|
reads. Archive previews are sealed and short-lived; ZIP passwords are
|
||||||
|
request-only.
|
||||||
|
- Public Form evidence uploads require a custom-header bearer grant that is
|
||||||
|
stored only as SHA-256, expires after at most 15 minutes, is bound to one
|
||||||
|
exact submission and user custodian, and can create only one managed file.
|
||||||
- Connector HTTP sockets use connection-time DNS/IP validation and pinning,
|
- Connector HTTP sockets use connection-time DNS/IP validation and pinning,
|
||||||
redirects are refused, and unsafe SDK transports fail before client creation.
|
redirects are refused, and unsafe SDK transports fail before client creation.
|
||||||
- Database-managed connector passwords/tokens are encrypted; responses redact
|
- Database-managed connector passwords/tokens are encrypted; responses redact
|
||||||
secrets and deployment references.
|
secrets and deployment references.
|
||||||
- API metadata is recursively checked for secret-like values.
|
- API metadata is recursively checked for secret-like values.
|
||||||
- Downloads use sanitized attachment filenames.
|
- Downloads use sanitized attachment filenames.
|
||||||
- SHA-256 and byte size are recorded for every blob/version.
|
- Plaintext semantic and stored-byte SHA-256/size evidence are recorded for
|
||||||
|
every protected blob; they are identical for unprotected blobs.
|
||||||
|
- Upload and archive-confirm APIs can select an Encryption vault. Protected
|
||||||
|
writes and reads fail closed if the optional Encryption capability is absent.
|
||||||
|
- Managed-object writes and applied orphan cleanup start lease-fenced Core
|
||||||
|
recovery operations before their physical effects; terminal success and
|
||||||
|
compensation require independent database and object checks.
|
||||||
|
|
||||||
The module does **not** currently provide malware scanning, content disarm and
|
The module does **not** currently provide malware scanning, content disarm and
|
||||||
reconstruction, a file-type allowlist, per-user quota, at-rest encryption for
|
reconstruction, a file-type allowlist, per-user quota, automatic encryption
|
||||||
local blob bytes, or a dedicated preview sandbox. Deployments that require
|
policy assignment, client E2EE, or a dedicated preview sandbox. The optional
|
||||||
these controls must supply them outside Files until explicit module contracts
|
server-envelope profile protects selected managed blob bytes at rest but remains
|
||||||
exist.
|
server-decryptable. Deployments that require the other controls must supply
|
||||||
|
them outside Files until explicit module contracts exist.
|
||||||
|
|
||||||
### Provenance
|
### Provenance
|
||||||
|
|
||||||
@@ -616,7 +932,7 @@ events. External metadata is provider/user input and is not a digital signature.
|
|||||||
Files records canonical audit events for:
|
Files records canonical audit events for:
|
||||||
|
|
||||||
- connector discovery attempts, before the attempted external I/O;
|
- connector discovery attempts, before the attempted external I/O;
|
||||||
- connector imports and manual syncs;
|
- connector imports, selected-file syncs, and bounded folder sync summaries;
|
||||||
- download/archive access to connector-originated managed files;
|
- download/archive access to connector-originated managed files;
|
||||||
- immediate connector profile and credential deletion/scrubbing;
|
- immediate connector profile and credential deletion/scrubbing;
|
||||||
- credential scrubbing during destructive module retirement.
|
- credential scrubbing during destructive module retirement.
|
||||||
@@ -624,6 +940,10 @@ Files records canonical audit events for:
|
|||||||
Connector audit details include the managed asset/version/blob, checksum, size,
|
Connector audit details include the managed asset/version/blob, checksum, size,
|
||||||
operation, source revision, and provenance where applicable. Deletion audit
|
operation, source revision, and provenance where applicable. Deletion audit
|
||||||
details name secret/reference kinds but never the secret values.
|
details name secret/reference kinds but never the secret values.
|
||||||
|
`files.connector.folder_synced` additionally records the connector profile,
|
||||||
|
provider, library/root, managed target, traversal mode, truncation state, typed
|
||||||
|
counts, and bounded per-item source/target/action references. It does not embed
|
||||||
|
downloaded bytes, credentials, or transport content.
|
||||||
|
|
||||||
Assets, folders, shares, profiles, credentials, policies, and connector spaces
|
Assets, folders, shares, profiles, credentials, policies, and connector spaces
|
||||||
also feed Core's incremental change sequence for UI synchronization. A change
|
also feed Core's incremental change sequence for UI synchronization. A change
|
||||||
@@ -647,6 +967,9 @@ The word "delete" has different meanings by object type:
|
|||||||
| Connector profile | Immediately disables the tombstone and clears credential links/material/references and private metadata |
|
| Connector profile | Immediately disables the tombstone and clears credential links/material/references and private metadata |
|
||||||
| Legacy `secret_ref` | Detached and audited as an unowned external reference; no provider deletion is attempted or claimed |
|
| Legacy `secret_ref` | Detached and audited as an unowned external reference; no provider deletion is attempted or claimed |
|
||||||
| Module retirement | Scrubs/audits credential material, then drops Files database tables; blob-backend cleanup is an operator responsibility |
|
| Module retirement | Scrubs/audits credential material, then drops Files database tables; blob-backend cleanup is an operator responsibility |
|
||||||
|
| Governed hard purge | Removes only preview-matched, soft-deleted asset/version rows without active retention, legal hold, active shares, Campaign evidence, or Form evidence |
|
||||||
|
| Blob garbage collection | Deletes an exact managed object only after a fresh zero-reference check under the shared blob fence; metadata follows verified object absence |
|
||||||
|
| Connector write-back | Explicit conditional S3 create/overwrite only; automatic provider delete, rename, move, and ACL propagation remain disabled |
|
||||||
|
|
||||||
Credential/profile scrubbing and its audit event use the same database
|
Credential/profile scrubbing and its audit event use the same database
|
||||||
transaction. If audit creation fails, the deletion rolls back. Repeating a
|
transaction. If audit creation fails, the deletion rolls back. Repeating a
|
||||||
@@ -654,15 +977,19 @@ delete against an already scrubbed tombstone does not recreate secret evidence.
|
|||||||
|
|
||||||
### Retention boundary
|
### Retention boundary
|
||||||
|
|
||||||
File versions and blobs are effectively retained indefinitely today. Although a
|
Each asset has an enforceable retained-until value, legal-hold flag, reason, and
|
||||||
blob has `ref_count` and `retained_until` fields, no complete retention-policy,
|
optimistic lifecycle revision. These controls block hard purge; they do not
|
||||||
legal-hold, hard-purge, or garbage-collection service enforces them. There is
|
automatically schedule it. Restore, purge preview/execute, and blob GC are
|
||||||
also no supported user restore endpoint for soft-deleted assets/folders and no
|
explicit authorized operations. Blob `retained_until` remains an additional
|
||||||
automated orphan-object reconciliation.
|
storage-level safeguard and must also have expired before automated collection
|
||||||
|
is introduced. Campaign and Form evidence are hard blockers rather than
|
||||||
|
silently cascaded references.
|
||||||
|
|
||||||
Do not promise erasure, timed retention, legal hold, or self-service recovery
|
Do not equate soft deletion with erasure. Erasure is complete only after the
|
||||||
from the current soft-delete behavior. Those require an explicit, auditable
|
approved purge removes asset/version metadata, bounded GC verifies that no
|
||||||
retention/purge design that preserves campaign and other evidence references.
|
version references the blob and removes its bytes/metadata, and any owning
|
||||||
|
Encryption retention/key-custody consequence has been handled under that
|
||||||
|
module's policy.
|
||||||
|
|
||||||
## Acceptance scenarios
|
## Acceptance scenarios
|
||||||
|
|
||||||
@@ -675,12 +1002,14 @@ her personal or Finance space, then the asset has the selected owner and tenant.
|
|||||||
Given Bob is outside Finance and has no share or Files admin permission, the
|
Given Bob is outside Finance and has no share or Files admin permission, the
|
||||||
same asset ID must not make the asset readable to Bob.
|
same asset ID must not make the asset readable to Bob.
|
||||||
|
|
||||||
### Safe ZIP import
|
### Safe archive import
|
||||||
|
|
||||||
Given a ZIP member contains `../../secret.txt`, is encrypted, exceeds 50 MiB,
|
Given an archive member contains `../../secret.txt`, is a link or special
|
||||||
or pushes actual extracted bytes above 250 MiB, unpacking must fail without a
|
filesystem object, exceeds 50 MiB, pushes actual expanded bytes above 2 GiB, or
|
||||||
committed managed asset. A valid archive must preserve normalized relative paths
|
exceeds the 100:1 expansion ratio, preview or confirmation must fail without a
|
||||||
below the chosen logical folder.
|
committed managed asset. A password-protected ZIP requires the correct
|
||||||
|
request-only password. A valid confirmation must match its unexpired preview
|
||||||
|
token and preserve normalized relative paths below the chosen logical folder.
|
||||||
|
|
||||||
### Explicit conflicts
|
### Explicit conflicts
|
||||||
|
|
||||||
@@ -699,9 +1028,11 @@ bypass the same rule.
|
|||||||
### Pinned connector transport
|
### Pinned connector transport
|
||||||
|
|
||||||
Given public-only mode and DNS returns any private address, the connection must
|
Given public-only mode and DNS returns any private address, the connection must
|
||||||
be rejected before a socket opens. Given private mode, the HTTP connection must
|
be rejected before a socket opens. Given private mode, every HTTP, S3, and SMB
|
||||||
still use the validated address and reject redirects. SMB and S3 must fail
|
connection must still use an address from the answer validated for that exact
|
||||||
before their SDK clients connect until all SDK-managed peers can be pinned.
|
attempt. Botocore retries, redirects, endpoint discovery, and aliases, plus SMB
|
||||||
|
reconnects and DFS referrals, must pass through the pinned factories. A changed
|
||||||
|
or unsupported SDK seam must fail before a usable client/session is returned.
|
||||||
|
|
||||||
### Imported evidence and sync
|
### Imported evidence and sync
|
||||||
|
|
||||||
@@ -710,6 +1041,13 @@ must carry source identity, revision when available, current version, checksum,
|
|||||||
and size. Re-syncing identical bytes must return `unchanged`; changed bytes must
|
and size. Re-syncing identical bytes must return `unchanged`; changed bytes must
|
||||||
create a higher version while preserving the previous version.
|
create a higher version while preserving the previous version.
|
||||||
|
|
||||||
|
Given a permitted linked connector folder contains changed, unchanged,
|
||||||
|
colliding, and policy-denied files, a manual folder sync must preserve remote
|
||||||
|
relative paths, commit the allowed siblings, return one typed outcome for every
|
||||||
|
discovered/skipped item, and emit a matching summary audit event. Reaching the
|
||||||
|
configured file bound must mark the run truncated rather than imply complete
|
||||||
|
coverage.
|
||||||
|
|
||||||
### Campaign freeze
|
### Campaign freeze
|
||||||
|
|
||||||
Given a campaign snapshot selected version V1, when the managed asset later
|
Given a campaign snapshot selected version V1, when the managed asset later
|
||||||
@@ -735,20 +1073,22 @@ returning different content or credentials.
|
|||||||
|
|
||||||
| Area | Implemented now | Planned or explicitly outside the current boundary |
|
| Area | Implemented now | Planned or explicitly outside the current boundary |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| Managed storage | Local durable-root backend, fallback read roots, tenant blob deduplication, checksums | Operational S3 after pinned SDK transport ([#34](https://git.add-ideas.de/add-ideas/govoplan-files/issues/34)); automated integrity/orphan reconciliation ([#36](https://git.add-ideas.de/add-ideas/govoplan-files/issues/36)) |
|
| Managed storage | Core local/S3 backend, exact managed-Garage or explicitly trusted HTTPS external S3, state-profile validation, fallback local read roots, tenant blob deduplication, checksums, bounded resumable integrity scans, quarantine, dry-run-first orphan cleanup, and Core-ledger verification/forward recovery | Scheduled scan execution and deployment-specific S3 HA/backup automation |
|
||||||
| Upload | Bounded direct upload, drag-and-drop UI, ZIP spool/extract limits, explicit conflicts | Malware scanning, quotas, type policy, resumable/chunked upload |
|
| Upload | Bounded direct upload, drag-and-drop UI, archive preview/selective extraction, password-protected ZIP support, explicit conflicts, opaque new object keys, and rollback compensation | Malware scanning, quotas, type policy, resumable/chunked upload |
|
||||||
| Organization | Folders, bulk rename preview/apply, move/copy, drag-and-drop, ZIP download, pattern resolution | General file-history UI and user-driven append-version/restore |
|
| Organization | Folders, bulk rename preview/apply, move/copy, drag-and-drop, ZIP download, pattern resolution, and API restoration preserving versions/provenance | General file-history UI and user-driven append-version UI |
|
||||||
| Sharing | User/group/tenant/campaign grant or permission update through API; campaign linkage display | Share revocation/expiry and complete general share-management UI ([#37](https://git.add-ideas.de/add-ideas/govoplan-files/issues/37)) |
|
| Sharing | User/group/tenant/campaign grants, expiry, idempotent revocation, searchable share-management UI, and campaign linkage display | Richer policy-driven share lifecycles |
|
||||||
| Deletion/retention | Soft-delete assets/folders/spaces; immediate audited connector-secret scrubbing | File restore API, hard purge, retention policy, legal hold, and blob GC ([#38](https://git.add-ideas.de/add-ideas/govoplan-files/issues/38)) |
|
| Deletion/retention | Soft-delete and restore assets/folders/spaces; optimistic retention and legal-hold controls; preview-bound, approval-referenced hard purge; reference-checked blob GC; immediate audited connector-secret scrubbing | Automatic time-based purge scheduling and richer lifecycle administration UI |
|
||||||
|
| Privacy requests | Tenant-scoped bounded DSAR metadata search; retained/manual/revoke/detach planning; idempotent share revocation and mutable actor-reference detachment; explicit separation from byte purge | Content-specific automated redaction and policy-specific approval remain manual or belong to the owning process |
|
||||||
| Connector governance | Scoped profiles/credentials/policies, effective source explanation, separate credentials, linked user/group spaces | Provider-owned external secret lifecycle; API `secret_ref` remains rejected |
|
| Connector governance | Scoped profiles/credentials/policies, effective source explanation, separate credentials, linked user/group spaces | Provider-owned external secret lifecycle; API `secret_ref` remains rejected |
|
||||||
| HTTP connectors | Pinned, bounded, no-redirect Seafile and WebDAV/Nextcloud browse/import/manual sync | Background/folder sync, remote mutation, long-running transfer workers |
|
| HTTP connectors | Pinned, bounded, no-redirect Seafile and WebDAV/Nextcloud browse/import, selected-file sync, and folder sync | Background sync, remote mutation, long-running transfer workers |
|
||||||
| SMB and S3 connectors | Provider descriptors and browse/import logic | Live access only after SDK connections plus DFS referrals/redirects are validated and pinned ([SMB #35](https://git.add-ideas.de/add-ideas/govoplan-files/issues/35), [S3 #34](https://git.add-ideas.de/add-ideas/govoplan-files/issues/34)) |
|
| SMB and S3 connectors | Provider descriptors, browse/import/manual sync, pinned SDK transports, redirect/retry/referral transport-contract tests, and explicit conditional S3 write-back with Core-ledger recovery | Live topology smoke evidence, provider-specific OAuth, additional provider writes, and background indexing remain separate deployment or connector-module concerns |
|
||||||
| Other providers | Reserved SharePoint/OneDrive keys and NFS/local descriptors | Graph/OAuth/provider paging, NFS deployment integration, DMS connectors |
|
| Other providers | Reserved SharePoint/OneDrive keys and NFS/local descriptors | Graph/OAuth/provider paging, NFS deployment integration, DMS connectors |
|
||||||
| Connector spaces | User/group link, browse, manual selected-file sync, edit/disable/delete | Background sync, remote writes/deletes, full conflict-reporting jobs |
|
| Connector spaces | User/group link, browse, bounded manual file/folder sync, edit/disable/delete/restore, read-only default, and opt-in S3 two-way mode | Background sync and automatic remote delete/rename/move/ACL propagation |
|
||||||
| Profile capabilities | Stored and displayed | Enforce capability flags as an independent operation gate |
|
| Profile capabilities | Stored, displayed, and enforced for explicit connector write-back | Broader provider-specific capability negotiation |
|
||||||
| Audit | Connector discovery/import/sync/access and connector deletion; campaign exact-use evidence | Dedicated canonical audit events for every ordinary Files mutation |
|
| Audit | Connector discovery/import/sync/access and connector deletion; campaign exact-use evidence | Dedicated canonical audit events for every ordinary Files mutation |
|
||||||
| Preview | File metadata and attachment download | Dedicated safe content-preview service |
|
| Preview | File metadata and attachment download | Dedicated safe content-preview service |
|
||||||
| Campaign | Stable capability-based frozen attachments and sent-use evidence | Campaign-specific process state remains in Campaign |
|
| Campaign | Stable capability-based frozen attachments and sent-use evidence | Campaign-specific process state remains in Campaign |
|
||||||
|
| Forms Runtime | One-time managed attachment grants plus exact-version, checksum, deletion, tenant, submission, and integrity verification | Malware scanning and advanced/qualified signature providers remain separate assurance depth |
|
||||||
| Collaboration | Governed input/output snapshots | Co-editing, comments, review, presence, locks, and semantic document versions belong to Documents/workflow/provider modules |
|
| Collaboration | Governed input/output snapshots | Co-editing, comments, review, presence, locks, and semantic document versions belong to Documents/workflow/provider modules |
|
||||||
|
|
||||||
## Release and change checklist
|
## Release and change checklist
|
||||||
@@ -764,11 +1104,20 @@ Before releasing Files:
|
|||||||
5. Exercise an allowed and denied owner/share path.
|
5. Exercise an allowed and denied owner/share path.
|
||||||
6. Exercise upload, ZIP bounds, conflict handling, download, and soft deletion.
|
6. Exercise upload, ZIP bounds, conflict handling, download, and soft deletion.
|
||||||
7. Exercise connector policy explanation and one pinned HTTP provider where
|
7. Exercise connector policy explanation and one pinned HTTP provider where
|
||||||
configured; verify S3/SMB still fail closed.
|
configured; verify the deployment-managed or trusted external S3 backend if
|
||||||
|
selected, and verify one configured S3 and SMB connector while recording the
|
||||||
|
actual target topology and private-network policy.
|
||||||
8. Verify credential deletion scrubs dependents and produces audit evidence.
|
8. Verify credential deletion scrubs dependents and produces audit evidence.
|
||||||
9. Verify a campaign attachment snapshot still identifies its exact version and
|
9. Verify a campaign attachment snapshot still identifies its exact version and
|
||||||
checksum after the current file changes.
|
checksum after the current file changes.
|
||||||
10. Update the implemented/planned table whenever a boundary changes.
|
10. Exercise a committed upload, a rolled-back upload, object tamper detection,
|
||||||
|
and applied orphan cleanup; inspect their `files` operations in Ops.
|
||||||
|
11. Exercise a Form evidence grant, token replay, wrong-submission reference,
|
||||||
|
expired token, unsupported media type, and quarantined-file rejection.
|
||||||
|
12. Exercise a Files DSAR search/plan, repeat an approved reversible action, and
|
||||||
|
confirm retained file content can only be erased through the separate purge
|
||||||
|
authority and recovery path.
|
||||||
|
13. Update the implemented/planned table whenever a boundary changes.
|
||||||
|
|
||||||
## Related documents
|
## Related documents
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,23 @@
|
|||||||
|
# Generated Artifact Store
|
||||||
|
|
||||||
|
Files implements Core's optional `files.artifact_store` capability for modules
|
||||||
|
that generate deterministic output without owning file storage.
|
||||||
|
|
||||||
|
The producer supplies bytes, filename, content type, destination folder,
|
||||||
|
optional idempotency key, and bounded non-secret provenance. Files applies the
|
||||||
|
actor's `files:file:upload` permission, tenant/user ownership, path rules,
|
||||||
|
versioning, configured blob backend, and conflict behavior. An idempotency key
|
||||||
|
is represented as source provenance so an unchanged retry does not create an
|
||||||
|
unrelated file version.
|
||||||
|
|
||||||
|
The shared Files session owns finalization. Before a new managed object is
|
||||||
|
written, Files commits a lease-fenced Core recovery operation containing only
|
||||||
|
identifiers and digests. The caller's eventual session commit independently
|
||||||
|
verifies both `FileBlob` metadata and stored bytes; rollback compensates only an
|
||||||
|
unreferenced key. Producers must therefore complete the supplied transaction
|
||||||
|
normally and must not bypass or replace Files session lifecycle handling.
|
||||||
|
|
||||||
|
The response contains only file/version identifiers, display path, media type,
|
||||||
|
size, digest, and storage provenance. Producers must not put credentials,
|
||||||
|
tokens, or rendered plaintext into metadata. Storing an artifact proves Files
|
||||||
|
accepted it; it does not prove printing, mailing, or any other external effect.
|
||||||
@@ -0,0 +1,55 @@
|
|||||||
|
# Files Interface Pattern Migration
|
||||||
|
|
||||||
|
This inventory records the Files-owned part of the GovOPlaN interface pattern
|
||||||
|
language. Core owns the shell and shared components; Files owns the composition
|
||||||
|
and consequences described here.
|
||||||
|
|
||||||
|
## Surface inventory
|
||||||
|
|
||||||
|
| Surface | Primary task | Archetype | Consequence | Pattern evidence |
|
||||||
|
| --- | --- | --- | --- | --- |
|
||||||
|
| `/files` space and folder panes | Browse managed and connected content without losing location | Directory/explorer | Low for navigation; medium for exposing filenames and provenance | Full-height two-pane workspace, bounded panes, stable selection and contextual Help Center link |
|
||||||
|
| `/files` toolbar and property filters | Find and act on the current selection | Explorer actions and local filtering | Medium for upload, move, copy, share and synchronization; high for delete | Actions remain beside the affected list, disabled controls explain permission/state/selection blockers, destructive work uses `ConfirmDialog` |
|
||||||
|
| Upload/archive, transfer, rename and connector-import dialogs | Supply and review one bounded change | Adaptive create/edit or guided import | Medium to high because files, paths and external bytes change | Shared `Dialog`, `FileDropZone`, validation, conflict review, unsaved inputs and explicit confirmation |
|
||||||
|
| File share dialog | Inspect and change access | Review/decision | High because another actor gains access | Shared dialog, access explanation, stable row actions and destructive confirmation |
|
||||||
|
| System/tenant/group/user connector surfaces | Compare connections, credentials and effective policy | Administration/configuration | High because endpoints, secrets and inherited policy control external access | Shared `ConnectionTree`, adaptive forms, `ActionBlockerHint`, policy provenance and contextual admin help |
|
||||||
|
| Connection and credential editors | Create or edit one governed endpoint or secret | Adaptive create/edit | High because a saved change may enable remote access | Relevant fields only, typed credential controls, discovery/test, advanced compatibility section, unsaved-change guard and disabled-save reasons |
|
||||||
|
| Connector policy card | Narrow inherited connector access | Effective-policy editor | High because deny/allow changes affect lower scopes | Typed reference selectors, deny precedence warning, effective source evidence and permission blocker |
|
||||||
|
| `files.widget.spaces` | See available managed/connected spaces and open Files | Dashboard widget | Low; names and provider state may still be sensitive | Shared loading/alert/status components, bounded item count, permission-filtered contribution |
|
||||||
|
| Files chooser capability used by another module | Select a managed snapshot without importing Files internals | Directory chooser | Medium because the exact selected version becomes another module's input | Shared dialog/confirmation, capability boundary and exact file/version evidence |
|
||||||
|
|
||||||
|
## State and consequence contract
|
||||||
|
|
||||||
|
- Loading, errors, success, empty results, access explanations and confirmation
|
||||||
|
use Core components. Files does not reproduce the application shell.
|
||||||
|
- A connector that comes from deployment settings remains visible but read-only;
|
||||||
|
its action explains that bootstrap configuration and a restart are required.
|
||||||
|
- Missing permission, target, selection, endpoint, or compatible provider is an
|
||||||
|
explained disabled state. It is not represented only by color or absence.
|
||||||
|
- Provider metadata JSON is an expert compatibility escape hatch inside the
|
||||||
|
collapsed shared advanced-options component. Ordinary connector setup uses
|
||||||
|
typed provider, endpoint, credential, capability and policy controls.
|
||||||
|
- Endpoint discovery and credential tests are explicit and report their result;
|
||||||
|
they do not save the draft. Save remains the only committing action.
|
||||||
|
- Connector/profile disable and managed-file delete remain confirmed actions and
|
||||||
|
state their immediate effect. Soft deletion must not be described as purge.
|
||||||
|
- External connector data, paths and credential references are rendered only in
|
||||||
|
already-authorized administration or explorer contexts. Secret values are
|
||||||
|
never returned for rendering.
|
||||||
|
|
||||||
|
## Accessibility and responsive evidence
|
||||||
|
|
||||||
|
Shared `Dialog` owns focus entry, Escape handling and focus return. Form and
|
||||||
|
toolbar DOM order is the keyboard order; disabled-action tooltips are themselves
|
||||||
|
focusable and expose the reason. The connector form uses semantic sections and
|
||||||
|
labels, status is textual as well as colored, and result alerts are announced by
|
||||||
|
the shared alert component. The explorer collapses to one column below 1050 px;
|
||||||
|
connector forms and action rows collapse below 760 px while preserving source
|
||||||
|
order. Long provider choices scroll inside the segmented control rather than
|
||||||
|
expanding the page.
|
||||||
|
|
||||||
|
The focused structural test guards these contracts, optional-module boundaries,
|
||||||
|
confirmation, contextual help, advanced-only JSON and responsive rules. Core's
|
||||||
|
TypeScript build, structural localization audit, module-permutation suite and
|
||||||
|
full-product bundle check provide the integration gates.
|
||||||
|
|
||||||
+8
-8
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@govoplan/files-webui",
|
"name": "@govoplan/files-webui",
|
||||||
"version": "0.1.9",
|
"version": "0.1.20",
|
||||||
"private": true,
|
"private": true,
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"main": "webui/src/index.ts",
|
"main": "webui/src/index.ts",
|
||||||
@@ -19,14 +19,14 @@
|
|||||||
"LICENSE"
|
"LICENSE"
|
||||||
],
|
],
|
||||||
"peerDependencies": {
|
"peerDependencies": {
|
||||||
"@govoplan/core-webui": "^0.1.9",
|
"@vitejs/plugin-react": "^5.2.0",
|
||||||
"lucide-react": "^1.23.0",
|
"vite": "^7.3.6",
|
||||||
"react": "^19.0.0",
|
|
||||||
"react-dom": "^19.0.0",
|
|
||||||
"react-router-dom": "^7.1.1",
|
|
||||||
"@vitejs/plugin-react": "^4.3.4",
|
|
||||||
"typescript": "^5.7.2",
|
"typescript": "^5.7.2",
|
||||||
"vite": "^6.0.6"
|
"react": ">=19.2.7 <20",
|
||||||
|
"react-dom": ">=19.2.7 <20",
|
||||||
|
"react-router": ">=8.3.0 <9",
|
||||||
|
"lucide-react": "^1.23.0",
|
||||||
|
"@govoplan/core-webui": "^0.1.18"
|
||||||
},
|
},
|
||||||
"peerDependenciesMeta": {
|
"peerDependenciesMeta": {
|
||||||
"@govoplan/core-webui": {
|
"@govoplan/core-webui": {
|
||||||
|
|||||||
+3
-2
@@ -4,15 +4,16 @@ build-backend = "setuptools.build_meta"
|
|||||||
|
|
||||||
[project]
|
[project]
|
||||||
name = "govoplan-files"
|
name = "govoplan-files"
|
||||||
version = "0.1.9"
|
version = "0.1.20"
|
||||||
description = "GovOPlaN files module with backend and WebUI integration."
|
description = "GovOPlaN files module with backend and WebUI integration."
|
||||||
readme = "README.md"
|
readme = "README.md"
|
||||||
requires-python = ">=3.12"
|
requires-python = ">=3.12"
|
||||||
license = { file = "LICENSE" }
|
license = { file = "LICENSE" }
|
||||||
authors = [{ name = "GovOPlaN" }]
|
authors = [{ name = "GovOPlaN" }]
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"govoplan-core>=0.1.9",
|
"govoplan-core>=0.1.20",
|
||||||
"defusedxml>=0.7,<1",
|
"defusedxml>=0.7,<1",
|
||||||
|
"pyzipper>=0.3.6,<1",
|
||||||
"python-multipart>=0.0.31,<1",
|
"python-multipart>=0.0.31,<1",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|||||||
@@ -5,13 +5,32 @@ import binascii
|
|||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from sqlalchemy import or_
|
from sqlalchemy import or_
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from govoplan_core.audit.logging import audit_from_principal
|
||||||
|
from govoplan_core.auth import ApiPrincipal, has_scope
|
||||||
from govoplan_core.core.access import AccessDecisionProvenance, PrincipalRef
|
from govoplan_core.core.access import AccessDecisionProvenance, PrincipalRef
|
||||||
from govoplan_core.core.files import FileAccessProvider
|
from govoplan_core.core.files import FileAccessProvider
|
||||||
|
from govoplan_core.core.files import (
|
||||||
|
ManagedTabularFile,
|
||||||
|
ManagedTabularFileAccessError,
|
||||||
|
ManagedTabularFileContent,
|
||||||
|
ManagedTabularFileNotFoundError,
|
||||||
|
ManagedTabularFileProvider,
|
||||||
|
ManagedTabularFileUnavailableError,
|
||||||
|
ManagedTabularFileValidationError,
|
||||||
|
PostboxFileReferenceRef,
|
||||||
|
PostboxFileReferenceRequest,
|
||||||
|
PostboxFileReferenceProvider,
|
||||||
|
ManagedArtifactRef,
|
||||||
|
ManagedArtifactStore,
|
||||||
|
ManagedArtifactWriteRequest,
|
||||||
|
)
|
||||||
from govoplan_core.core.modules import ModuleContext
|
from govoplan_core.core.modules import ModuleContext
|
||||||
from govoplan_core.security.module_permissions import scopes_grant_compatible
|
from govoplan_core.security.module_permissions import scopes_grant_compatible
|
||||||
from govoplan_files.backend.db.models import FileAsset, FileFolder, FileShare
|
from govoplan_files.backend.db.models import FileAsset, FileBlob, FileFolder, FileShare, FileVersion
|
||||||
from govoplan_files.backend.runtime import configure_runtime
|
from govoplan_files.backend.runtime import configure_runtime
|
||||||
|
from govoplan_files.backend.storage.common import FileStorageError
|
||||||
from govoplan_files.backend.storage.campaign_attachments import (
|
from govoplan_files.backend.storage.campaign_attachments import (
|
||||||
annotate_built_messages_with_managed_files,
|
annotate_built_messages_with_managed_files,
|
||||||
managed_match_payloads,
|
managed_match_payloads,
|
||||||
@@ -20,8 +39,16 @@ from govoplan_files.backend.storage.campaign_attachments import (
|
|||||||
share_assets_with_campaign,
|
share_assets_with_campaign,
|
||||||
)
|
)
|
||||||
from govoplan_files.backend.storage.campaign_usage import record_campaign_attachment_uses_for_jobs
|
from govoplan_files.backend.storage.campaign_usage import record_campaign_attachment_uses_for_jobs
|
||||||
from govoplan_files.backend.storage.files import current_version_and_blob
|
from govoplan_files.backend.storage.files import (
|
||||||
|
create_file_asset,
|
||||||
|
current_version_and_blob,
|
||||||
|
get_asset_for_user,
|
||||||
|
list_recent_assets_for_user,
|
||||||
|
read_asset_version_bytes,
|
||||||
|
sync_file_asset_from_source,
|
||||||
|
)
|
||||||
from govoplan_files.backend.storage.paths import normalize_folder
|
from govoplan_files.backend.storage.paths import normalize_folder
|
||||||
|
from govoplan_files.backend.storage.share_state import effective_file_share_clause
|
||||||
|
|
||||||
|
|
||||||
VIRTUAL_FOLDER_RESOURCE_PREFIX = "virtual-folder:v1"
|
VIRTUAL_FOLDER_RESOURCE_PREFIX = "virtual-folder:v1"
|
||||||
@@ -59,6 +86,214 @@ def campaign_capability(context: ModuleContext) -> FilesCampaignCapability:
|
|||||||
return FilesCampaignCapability()
|
return FilesCampaignCapability()
|
||||||
|
|
||||||
|
|
||||||
|
class FilesArtifactStore(ManagedArtifactStore):
|
||||||
|
def store_artifact(
|
||||||
|
self,
|
||||||
|
session: object,
|
||||||
|
principal: object,
|
||||||
|
*,
|
||||||
|
request: ManagedArtifactWriteRequest,
|
||||||
|
) -> ManagedArtifactRef:
|
||||||
|
if not hasattr(session, "query") or not hasattr(session, "flush"):
|
||||||
|
raise TypeError("Files artifact storage requires a SQLAlchemy session.")
|
||||||
|
if not hasattr(principal, "has") or not principal.has("files:file:upload"):
|
||||||
|
raise PermissionError("Managed artifact storage requires files:file:upload.")
|
||||||
|
user = getattr(principal, "user", None)
|
||||||
|
user_id = str(getattr(user, "id", "") or "")
|
||||||
|
tenant_id = str(getattr(principal, "tenant_id", "") or "")
|
||||||
|
if not user_id or not tenant_id:
|
||||||
|
raise PermissionError("Managed artifact storage requires a tenant user principal.")
|
||||||
|
metadata = dict(request.metadata)
|
||||||
|
if request.idempotency_key:
|
||||||
|
metadata["source_provenance"] = {
|
||||||
|
"source_type": "generated_artifact",
|
||||||
|
"connector_id": "files.artifact_store",
|
||||||
|
"provider": str(metadata.get("producer_module") or "platform"),
|
||||||
|
"external_id": request.idempotency_key,
|
||||||
|
"revision": str(metadata.get("output_sha256") or "") or None,
|
||||||
|
}
|
||||||
|
stored, _action, _previous_version_id = sync_file_asset_from_source(
|
||||||
|
session,
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
owner_type="user",
|
||||||
|
owner_id=user_id,
|
||||||
|
user_id=user_id,
|
||||||
|
filename=request.filename,
|
||||||
|
data=request.payload,
|
||||||
|
metadata=metadata,
|
||||||
|
folder=request.folder,
|
||||||
|
content_type=request.content_type,
|
||||||
|
conflict_strategy="rename",
|
||||||
|
is_admin=principal.has("files:file:admin"),
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
stored = create_file_asset(
|
||||||
|
session,
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
owner_type="user",
|
||||||
|
owner_id=user_id,
|
||||||
|
user_id=user_id,
|
||||||
|
filename=request.filename,
|
||||||
|
data=request.payload,
|
||||||
|
folder=request.folder,
|
||||||
|
content_type=request.content_type,
|
||||||
|
description=request.description,
|
||||||
|
metadata=metadata,
|
||||||
|
conflict_strategy="rename",
|
||||||
|
is_admin=principal.has("files:file:admin"),
|
||||||
|
)
|
||||||
|
return ManagedArtifactRef(
|
||||||
|
file_asset_id=stored.asset.id,
|
||||||
|
file_version_id=stored.version.id,
|
||||||
|
filename=stored.version.filename_at_upload,
|
||||||
|
display_path=stored.asset.display_path,
|
||||||
|
content_type=stored.version.content_type or request.content_type,
|
||||||
|
size_bytes=stored.version.size_bytes,
|
||||||
|
sha256=stored.version.checksum_sha256,
|
||||||
|
provenance={
|
||||||
|
"module": "files",
|
||||||
|
"owner_type": stored.asset.owner_type,
|
||||||
|
"managed": True,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def artifact_store_capability(context: ModuleContext) -> FilesArtifactStore:
|
||||||
|
configure_runtime(registry=context.registry, settings=context.settings)
|
||||||
|
return FilesArtifactStore()
|
||||||
|
|
||||||
|
|
||||||
|
class FilesPostboxReferenceService(PostboxFileReferenceProvider):
|
||||||
|
"""Resolve Postbox evidence without turning message access into file access."""
|
||||||
|
|
||||||
|
def resolve_postbox_references(
|
||||||
|
self,
|
||||||
|
session: object,
|
||||||
|
principal: object,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
requests: tuple[PostboxFileReferenceRequest, ...],
|
||||||
|
) -> tuple[PostboxFileReferenceRef, ...]:
|
||||||
|
if not hasattr(session, "get"):
|
||||||
|
raise TypeError("Postbox file resolution requires a SQLAlchemy session.")
|
||||||
|
principal_tenant = str(getattr(principal, "tenant_id", "") or "")
|
||||||
|
can_download = bool(
|
||||||
|
hasattr(principal, "has") and principal.has("files:file:download")
|
||||||
|
)
|
||||||
|
user = getattr(principal, "user", None)
|
||||||
|
user_id = str(getattr(user, "id", "") or "")
|
||||||
|
is_admin = bool(
|
||||||
|
hasattr(principal, "has") and principal.has("files:file:admin")
|
||||||
|
)
|
||||||
|
results: list[PostboxFileReferenceRef] = []
|
||||||
|
for request in requests:
|
||||||
|
if principal_tenant != tenant_id:
|
||||||
|
results.append(_unavailable_postbox_reference(request, "tenant_mismatch"))
|
||||||
|
continue
|
||||||
|
if not can_download or not user_id:
|
||||||
|
results.append(
|
||||||
|
_unavailable_postbox_reference(
|
||||||
|
request,
|
||||||
|
"download_permission_missing",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
reference_type = request.reference_type.strip().casefold()
|
||||||
|
version: FileVersion | None = None
|
||||||
|
if reference_type in {"file", "file_asset", "files:file"}:
|
||||||
|
results.append(
|
||||||
|
_unavailable_postbox_reference(request, "exact_version_required")
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
elif reference_type in {"file_version", "files:file_version"}:
|
||||||
|
version = session.get(FileVersion, request.reference_id) # type: ignore[attr-defined]
|
||||||
|
asset = (
|
||||||
|
session.get(FileAsset, version.file_asset_id) # type: ignore[attr-defined]
|
||||||
|
if version is not None
|
||||||
|
else None
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
results.append(
|
||||||
|
_unavailable_postbox_reference(
|
||||||
|
request,
|
||||||
|
"unsupported_reference_type",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
if (
|
||||||
|
asset is None
|
||||||
|
or version is None
|
||||||
|
or asset.tenant_id != tenant_id
|
||||||
|
or version.tenant_id != tenant_id
|
||||||
|
or asset.deleted_at is not None
|
||||||
|
):
|
||||||
|
results.append(_unavailable_postbox_reference(request, "file_not_found"))
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
get_asset_for_user(
|
||||||
|
session, # type: ignore[arg-type]
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
user_id=user_id,
|
||||||
|
asset_id=asset.id,
|
||||||
|
is_admin=is_admin,
|
||||||
|
)
|
||||||
|
except FileStorageError:
|
||||||
|
results.append(
|
||||||
|
_unavailable_postbox_reference(request, "file_access_denied")
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
blob = session.get(FileBlob, version.blob_id) # type: ignore[attr-defined]
|
||||||
|
if blob is None or blob.tenant_id != tenant_id:
|
||||||
|
results.append(
|
||||||
|
_unavailable_postbox_reference(request, "file_payload_missing")
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
results.append(
|
||||||
|
PostboxFileReferenceRef(
|
||||||
|
reference_type=request.reference_type,
|
||||||
|
reference_id=request.reference_id,
|
||||||
|
available=True,
|
||||||
|
reason_code="available",
|
||||||
|
file_asset_id=asset.id,
|
||||||
|
file_version_id=version.id,
|
||||||
|
filename=version.filename_at_upload or asset.filename,
|
||||||
|
content_type=version.content_type or blob.content_type,
|
||||||
|
size_bytes=version.size_bytes,
|
||||||
|
sha256=version.checksum_sha256,
|
||||||
|
download_path=(
|
||||||
|
f"/api/v1/files/{asset.id}/versions/{version.id}/download"
|
||||||
|
),
|
||||||
|
provenance={
|
||||||
|
"module": "files",
|
||||||
|
"postbox_id": request.postbox_id,
|
||||||
|
"message_id": request.message_id,
|
||||||
|
"display_path": asset.display_path,
|
||||||
|
"exact_version": True,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return tuple(results)
|
||||||
|
|
||||||
|
|
||||||
|
def _unavailable_postbox_reference(
|
||||||
|
request: PostboxFileReferenceRequest,
|
||||||
|
reason_code: str,
|
||||||
|
) -> PostboxFileReferenceRef:
|
||||||
|
return PostboxFileReferenceRef(
|
||||||
|
reference_type=request.reference_type,
|
||||||
|
reference_id=request.reference_id,
|
||||||
|
available=False,
|
||||||
|
reason_code=reason_code,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def postbox_reference_capability(
|
||||||
|
context: ModuleContext,
|
||||||
|
) -> FilesPostboxReferenceService:
|
||||||
|
configure_runtime(registry=context.registry, settings=context.settings)
|
||||||
|
return FilesPostboxReferenceService()
|
||||||
|
|
||||||
|
|
||||||
class FilesAccessService(FileAccessProvider):
|
class FilesAccessService(FileAccessProvider):
|
||||||
def explain_resource_provenance(
|
def explain_resource_provenance(
|
||||||
self,
|
self,
|
||||||
@@ -96,7 +331,7 @@ class FilesAccessService(FileAccessProvider):
|
|||||||
.filter(
|
.filter(
|
||||||
FileShare.tenant_id == asset.tenant_id,
|
FileShare.tenant_id == asset.tenant_id,
|
||||||
FileShare.file_asset_id == asset.id,
|
FileShare.file_asset_id == asset.id,
|
||||||
FileShare.revoked_at.is_(None),
|
effective_file_share_clause(),
|
||||||
FileShare.permission.in_(sorted(permission_values)),
|
FileShare.permission.in_(sorted(permission_values)),
|
||||||
or_(
|
or_(
|
||||||
(FileShare.target_type == "user") & (FileShare.target_id == principal.membership_id),
|
(FileShare.target_type == "user") & (FileShare.target_id == principal.membership_id),
|
||||||
@@ -197,6 +432,228 @@ def access_capability(context: ModuleContext) -> FilesAccessService:
|
|||||||
return FilesAccessService()
|
return FilesAccessService()
|
||||||
|
|
||||||
|
|
||||||
|
class FilesManagedTabularFileService(ManagedTabularFileProvider):
|
||||||
|
"""Expose authorized immutable CSV/XLSX versions to optional consumers."""
|
||||||
|
|
||||||
|
def list_tabular_files(
|
||||||
|
self,
|
||||||
|
session: object,
|
||||||
|
principal: object,
|
||||||
|
*,
|
||||||
|
query: str = "",
|
||||||
|
limit: int = 100,
|
||||||
|
) -> tuple[ManagedTabularFile, ...]:
|
||||||
|
db, api_principal, user_id = _tabular_context(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
required_scope="files:file:read",
|
||||||
|
)
|
||||||
|
normalized_query = str(query or "").strip().casefold()
|
||||||
|
requested_limit = max(1, min(int(limit), 100))
|
||||||
|
assets = list_recent_assets_for_user(
|
||||||
|
db,
|
||||||
|
tenant_id=api_principal.tenant_id,
|
||||||
|
user_id=user_id,
|
||||||
|
limit=min(500, requested_limit * 5),
|
||||||
|
is_admin=has_scope(api_principal, "files:file:admin"),
|
||||||
|
)
|
||||||
|
results: list[ManagedTabularFile] = []
|
||||||
|
for asset in assets:
|
||||||
|
if normalized_query and normalized_query not in (
|
||||||
|
f"{asset.filename} {asset.display_path} {asset.description or ''}"
|
||||||
|
).casefold():
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
version, blob = current_version_and_blob(db, asset)
|
||||||
|
except FileStorageError:
|
||||||
|
continue
|
||||||
|
if not _is_tabular_file(version.filename_at_upload, version.content_type):
|
||||||
|
continue
|
||||||
|
results.append(_managed_tabular_file(asset, version, blob))
|
||||||
|
if len(results) >= requested_limit:
|
||||||
|
break
|
||||||
|
return tuple(results)
|
||||||
|
|
||||||
|
def get_tabular_file(
|
||||||
|
self,
|
||||||
|
session: object,
|
||||||
|
principal: object,
|
||||||
|
*,
|
||||||
|
file_asset_id: str,
|
||||||
|
file_version_id: str | None = None,
|
||||||
|
) -> ManagedTabularFile | None:
|
||||||
|
db, api_principal, user_id = _tabular_context(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
required_scope="files:file:read",
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
asset = get_asset_for_user(
|
||||||
|
db,
|
||||||
|
tenant_id=api_principal.tenant_id,
|
||||||
|
user_id=user_id,
|
||||||
|
asset_id=file_asset_id,
|
||||||
|
is_admin=has_scope(api_principal, "files:file:admin"),
|
||||||
|
)
|
||||||
|
except FileStorageError:
|
||||||
|
return None
|
||||||
|
if file_version_id:
|
||||||
|
version = db.get(FileVersion, file_version_id)
|
||||||
|
if (
|
||||||
|
version is None
|
||||||
|
or version.file_asset_id != asset.id
|
||||||
|
or version.tenant_id != api_principal.tenant_id
|
||||||
|
):
|
||||||
|
return None
|
||||||
|
blob = db.get(FileBlob, version.blob_id)
|
||||||
|
if blob is None or blob.tenant_id != api_principal.tenant_id:
|
||||||
|
return None
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
version, blob = current_version_and_blob(db, asset)
|
||||||
|
except FileStorageError as exc:
|
||||||
|
raise ManagedTabularFileUnavailableError(
|
||||||
|
"Managed file version metadata is unavailable."
|
||||||
|
) from exc
|
||||||
|
if not _is_tabular_file(version.filename_at_upload, version.content_type):
|
||||||
|
return None
|
||||||
|
return _managed_tabular_file(asset, version, blob)
|
||||||
|
|
||||||
|
def read_tabular_file(
|
||||||
|
self,
|
||||||
|
session: object,
|
||||||
|
principal: object,
|
||||||
|
*,
|
||||||
|
file_asset_id: str,
|
||||||
|
file_version_id: str,
|
||||||
|
max_bytes: int,
|
||||||
|
) -> ManagedTabularFileContent:
|
||||||
|
db, api_principal, user_id = _tabular_context(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
required_scope="files:file:download",
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
asset = get_asset_for_user(
|
||||||
|
db,
|
||||||
|
tenant_id=api_principal.tenant_id,
|
||||||
|
user_id=user_id,
|
||||||
|
asset_id=file_asset_id,
|
||||||
|
is_admin=has_scope(api_principal, "files:file:admin"),
|
||||||
|
)
|
||||||
|
except FileStorageError as exc:
|
||||||
|
raise ManagedTabularFileNotFoundError(
|
||||||
|
"Managed tabular file not found."
|
||||||
|
) from exc
|
||||||
|
metadata = self.get_tabular_file(
|
||||||
|
db,
|
||||||
|
api_principal,
|
||||||
|
file_asset_id=asset.id,
|
||||||
|
file_version_id=file_version_id,
|
||||||
|
)
|
||||||
|
if metadata is None:
|
||||||
|
raise ManagedTabularFileNotFoundError(
|
||||||
|
"Managed tabular file version not found."
|
||||||
|
)
|
||||||
|
effective_max = max(1, int(max_bytes))
|
||||||
|
if metadata.size_bytes > effective_max:
|
||||||
|
raise ManagedTabularFileValidationError(
|
||||||
|
f"Managed tabular files are limited to {effective_max:,} bytes for this operation."
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
payload, version, blob = read_asset_version_bytes(
|
||||||
|
db,
|
||||||
|
asset,
|
||||||
|
file_version_id,
|
||||||
|
)
|
||||||
|
except FileStorageError as exc:
|
||||||
|
raise ManagedTabularFileUnavailableError(
|
||||||
|
"Managed tabular file content is unavailable or failed integrity verification."
|
||||||
|
) from exc
|
||||||
|
if len(payload) > effective_max:
|
||||||
|
raise ManagedTabularFileValidationError(
|
||||||
|
f"Managed tabular files are limited to {effective_max:,} bytes for this operation."
|
||||||
|
)
|
||||||
|
result = _managed_tabular_file(asset, version, blob)
|
||||||
|
audit_from_principal(
|
||||||
|
db,
|
||||||
|
api_principal,
|
||||||
|
action="files.tabular_content.read",
|
||||||
|
object_type="file_version",
|
||||||
|
object_id=version.id,
|
||||||
|
details={
|
||||||
|
"file_asset_id": asset.id,
|
||||||
|
"size_bytes": version.size_bytes,
|
||||||
|
"checksum_sha256": version.checksum_sha256,
|
||||||
|
"consumer": "tabular_content",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
return ManagedTabularFileContent(file=result, payload=payload)
|
||||||
|
|
||||||
|
|
||||||
|
def managed_tabular_file_capability(
|
||||||
|
context: ModuleContext,
|
||||||
|
) -> FilesManagedTabularFileService:
|
||||||
|
configure_runtime(registry=context.registry, settings=context.settings)
|
||||||
|
return FilesManagedTabularFileService()
|
||||||
|
|
||||||
|
|
||||||
|
def _tabular_context(
|
||||||
|
session: object,
|
||||||
|
principal: object,
|
||||||
|
*,
|
||||||
|
required_scope: str,
|
||||||
|
) -> tuple[Session, ApiPrincipal, str]:
|
||||||
|
if not isinstance(session, Session):
|
||||||
|
raise TypeError("Managed tabular file access requires a SQLAlchemy session.")
|
||||||
|
if not isinstance(principal, ApiPrincipal):
|
||||||
|
raise ManagedTabularFileAccessError(
|
||||||
|
"Managed tabular file access requires a tenant API principal."
|
||||||
|
)
|
||||||
|
if not (
|
||||||
|
has_scope(principal, required_scope)
|
||||||
|
or has_scope(principal, "files:file:admin")
|
||||||
|
):
|
||||||
|
raise ManagedTabularFileAccessError(
|
||||||
|
f"Managed tabular file access requires {required_scope}."
|
||||||
|
)
|
||||||
|
user_id = str(getattr(principal.user, "id", "") or "").strip()
|
||||||
|
if not principal.tenant_id or not user_id:
|
||||||
|
raise ManagedTabularFileAccessError(
|
||||||
|
"Managed tabular file access requires a tenant user principal."
|
||||||
|
)
|
||||||
|
return session, principal, user_id
|
||||||
|
|
||||||
|
|
||||||
|
def _managed_tabular_file(
|
||||||
|
asset: FileAsset,
|
||||||
|
version: FileVersion,
|
||||||
|
blob: FileBlob,
|
||||||
|
) -> ManagedTabularFile:
|
||||||
|
return ManagedTabularFile(
|
||||||
|
file_asset_id=asset.id,
|
||||||
|
file_version_id=version.id,
|
||||||
|
filename=version.filename_at_upload or asset.filename,
|
||||||
|
display_path=version.display_path_at_upload or asset.display_path,
|
||||||
|
content_type=version.content_type or blob.content_type,
|
||||||
|
size_bytes=version.size_bytes,
|
||||||
|
sha256=version.checksum_sha256,
|
||||||
|
updated_at=version.created_at,
|
||||||
|
current_version=asset.current_version_id == version.id,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _is_tabular_file(filename: str, content_type: str | None) -> bool:
|
||||||
|
normalized_name = str(filename or "").strip().casefold()
|
||||||
|
normalized_type = str(content_type or "").split(";", 1)[0].strip().casefold()
|
||||||
|
return normalized_name.endswith((".csv", ".xlsx")) or normalized_type in {
|
||||||
|
"text/csv",
|
||||||
|
"application/csv",
|
||||||
|
"application/vnd.ms-excel",
|
||||||
|
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def virtual_folder_resource_id(*, tenant_id: str, owner_type: str, owner_id: str, path: str) -> str:
|
def virtual_folder_resource_id(*, tenant_id: str, owner_type: str, owner_id: str, path: str) -> str:
|
||||||
normalized_path = normalize_folder(path)
|
normalized_path = normalize_folder(path)
|
||||||
encoded_path = base64.urlsafe_b64encode(normalized_path.encode("utf-8")).decode("ascii").rstrip("=")
|
encoded_path = base64.urlsafe_b64encode(normalized_path.encode("utf-8")).decode("ascii").rstrip("=")
|
||||||
|
|||||||
@@ -56,6 +56,10 @@ def _record_asset_change(session: OrmSession, asset: FileAsset) -> None:
|
|||||||
"filename",
|
"filename",
|
||||||
"description",
|
"description",
|
||||||
"deleted_at",
|
"deleted_at",
|
||||||
|
"retained_until",
|
||||||
|
"legal_hold",
|
||||||
|
"lifecycle_revision",
|
||||||
|
"lifecycle_reason",
|
||||||
"metadata_",
|
"metadata_",
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
@@ -79,6 +83,9 @@ def _record_asset_change(session: OrmSession, asset: FileAsset) -> None:
|
|||||||
"previous_path": previous_value(asset, "display_path"),
|
"previous_path": previous_value(asset, "display_path"),
|
||||||
"filename": asset.filename,
|
"filename": asset.filename,
|
||||||
"deleted_at": _isoformat(asset.deleted_at),
|
"deleted_at": _isoformat(asset.deleted_at),
|
||||||
|
"retained_until": _isoformat(asset.retained_until),
|
||||||
|
"legal_hold": asset.legal_hold,
|
||||||
|
"lifecycle_revision": asset.lifecycle_revision,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -117,7 +124,15 @@ def _record_share_visibility_change(session: OrmSession, share: FileShare) -> No
|
|||||||
return
|
return
|
||||||
if not state.pending and not has_attr_changes(
|
if not state.pending and not has_attr_changes(
|
||||||
state,
|
state,
|
||||||
("file_asset_id", "target_type", "target_id", "permission", "revoked_at"),
|
(
|
||||||
|
"file_asset_id",
|
||||||
|
"target_type",
|
||||||
|
"target_id",
|
||||||
|
"permission",
|
||||||
|
"expires_at",
|
||||||
|
"revoked_at",
|
||||||
|
"revoked_by_user_id",
|
||||||
|
),
|
||||||
):
|
):
|
||||||
return
|
return
|
||||||
_ensure_id(share)
|
_ensure_id(share)
|
||||||
@@ -136,7 +151,9 @@ def _record_share_visibility_change(session: OrmSession, share: FileShare) -> No
|
|||||||
"share_target_type": share.target_type,
|
"share_target_type": share.target_type,
|
||||||
"share_target_id": share.target_id,
|
"share_target_id": share.target_id,
|
||||||
"share_permission": share.permission,
|
"share_permission": share.permission,
|
||||||
|
"share_expires_at": _isoformat(share.expires_at),
|
||||||
"share_revoked_at": _isoformat(share.revoked_at),
|
"share_revoked_at": _isoformat(share.revoked_at),
|
||||||
|
"share_revoked_by_user_id": share.revoked_by_user_id,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,600 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from collections.abc import Mapping
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
from urllib.parse import urlsplit
|
||||||
|
|
||||||
|
from govoplan_core.core.configuration_packages import (
|
||||||
|
ConfigurationApplyResult,
|
||||||
|
ConfigurationDiagnostic,
|
||||||
|
ConfigurationExportResult,
|
||||||
|
ConfigurationExportSelection,
|
||||||
|
ConfigurationPackageFragment,
|
||||||
|
ConfigurationPlanItem,
|
||||||
|
ConfigurationPreflightContext,
|
||||||
|
ConfigurationPreflightResult,
|
||||||
|
ConfigurationProvider,
|
||||||
|
ConfigurationProviderDescription,
|
||||||
|
)
|
||||||
|
from govoplan_core.core.infrastructure_capabilities import (
|
||||||
|
InfrastructureCapability,
|
||||||
|
InfrastructureCapabilityReceipt,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.runtime import settings as runtime_settings
|
||||||
|
|
||||||
|
|
||||||
|
FILES_CONFIGURATION_CAPABILITY = "files.configuration"
|
||||||
|
MANAGED_STORAGE_FRAGMENT = "managed_storage"
|
||||||
|
_PAYLOAD_KEYS = frozenset(
|
||||||
|
{"capability_id", "expected_backend", "expected_source"}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class FilesConfigurationProvider(ConfigurationProvider):
|
||||||
|
module_id = "files"
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
settings: object | None = None,
|
||||||
|
environment: Mapping[str, str] | None = None,
|
||||||
|
) -> None:
|
||||||
|
self._settings = runtime_settings if settings is None else settings
|
||||||
|
self._environment = os.environ if environment is None else environment
|
||||||
|
|
||||||
|
def describe(self) -> ConfigurationProviderDescription:
|
||||||
|
return ConfigurationProviderDescription(
|
||||||
|
module_id=self.module_id,
|
||||||
|
fragment_types=(MANAGED_STORAGE_FRAGMENT,),
|
||||||
|
schema_refs={
|
||||||
|
MANAGED_STORAGE_FRAGMENT: "govoplan/files/configuration/managed-storage.v1"
|
||||||
|
},
|
||||||
|
exported_scopes=("system",),
|
||||||
|
)
|
||||||
|
|
||||||
|
def preflight(
|
||||||
|
self,
|
||||||
|
fragment: ConfigurationPackageFragment,
|
||||||
|
context: ConfigurationPreflightContext,
|
||||||
|
) -> ConfigurationPreflightResult:
|
||||||
|
if fragment.fragment_type != MANAGED_STORAGE_FRAGMENT:
|
||||||
|
return ConfigurationPreflightResult(
|
||||||
|
diagnostics=(_unsupported(fragment),),
|
||||||
|
plan=(_blocked_plan(fragment, "Fragment type is unsupported."),),
|
||||||
|
)
|
||||||
|
diagnostics, binding_ref = self._binding_diagnostics(fragment, context)
|
||||||
|
blocked = any(item.severity == "blocker" for item in diagnostics)
|
||||||
|
return ConfigurationPreflightResult(
|
||||||
|
diagnostics=tuple(diagnostics),
|
||||||
|
plan=(
|
||||||
|
ConfigurationPlanItem(
|
||||||
|
action="blocked" if blocked else "skip",
|
||||||
|
module_id="files",
|
||||||
|
fragment_type=MANAGED_STORAGE_FRAGMENT,
|
||||||
|
fragment_id=fragment.fragment_id or binding_ref,
|
||||||
|
summary=(
|
||||||
|
"Managed storage does not match the deployment receipt."
|
||||||
|
if blocked
|
||||||
|
else "Deployment-owned managed storage already matches the receipt; no module state is rewritten."
|
||||||
|
),
|
||||||
|
),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
def apply(
|
||||||
|
self,
|
||||||
|
fragment: ConfigurationPackageFragment,
|
||||||
|
supplied_data: Mapping[str, Any],
|
||||||
|
context: ConfigurationPreflightContext,
|
||||||
|
) -> ConfigurationApplyResult:
|
||||||
|
del supplied_data
|
||||||
|
if fragment.fragment_type != MANAGED_STORAGE_FRAGMENT:
|
||||||
|
return ConfigurationApplyResult(diagnostics=(_unsupported(fragment),))
|
||||||
|
diagnostics, _binding_ref = self._binding_diagnostics(fragment, context)
|
||||||
|
return ConfigurationApplyResult(
|
||||||
|
diagnostics=tuple(
|
||||||
|
item for item in diagnostics if item.severity == "blocker"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
def export(
|
||||||
|
self,
|
||||||
|
selection: ConfigurationExportSelection,
|
||||||
|
context: ConfigurationPreflightContext,
|
||||||
|
) -> ConfigurationExportResult:
|
||||||
|
del selection
|
||||||
|
receipt = context.infrastructure_receipt
|
||||||
|
if context.infrastructure_receipt_error:
|
||||||
|
return ConfigurationExportResult(
|
||||||
|
diagnostics=(
|
||||||
|
_receipt_error(
|
||||||
|
context.infrastructure_receipt_error,
|
||||||
|
object_ref="files.storage",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if receipt is None:
|
||||||
|
return ConfigurationExportResult(
|
||||||
|
diagnostics=(
|
||||||
|
ConfigurationDiagnostic(
|
||||||
|
severity="blocker",
|
||||||
|
code="infrastructure_receipt_missing",
|
||||||
|
message="Files managed-storage export requires the deployment capability receipt.",
|
||||||
|
module_id="files",
|
||||||
|
object_ref="files.storage",
|
||||||
|
resolution="Mount the installer-generated receipt before exporting deployment configuration.",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
capability = receipt.capability("files.storage")
|
||||||
|
if capability is None:
|
||||||
|
return ConfigurationExportResult(
|
||||||
|
diagnostics=(
|
||||||
|
ConfigurationDiagnostic(
|
||||||
|
severity="blocker",
|
||||||
|
code="infrastructure_capability_missing",
|
||||||
|
message="The deployment receipt does not declare managed file storage.",
|
||||||
|
module_id="files",
|
||||||
|
object_ref="files.storage",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return ConfigurationExportResult(
|
||||||
|
fragments=(
|
||||||
|
ConfigurationPackageFragment(
|
||||||
|
module_id="files",
|
||||||
|
fragment_type=MANAGED_STORAGE_FRAGMENT,
|
||||||
|
fragment_id=f"{receipt.installation_id}:files.storage",
|
||||||
|
payload={
|
||||||
|
"capability_id": "files.storage",
|
||||||
|
"expected_backend": _expected_backend(capability.source),
|
||||||
|
"expected_source": capability.source,
|
||||||
|
},
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
def health(
|
||||||
|
self,
|
||||||
|
import_result: ConfigurationApplyResult,
|
||||||
|
context: ConfigurationPreflightContext,
|
||||||
|
) -> tuple[ConfigurationDiagnostic, ...]:
|
||||||
|
del context
|
||||||
|
return tuple(
|
||||||
|
item for item in import_result.diagnostics if item.severity == "blocker"
|
||||||
|
)
|
||||||
|
|
||||||
|
def _binding_diagnostics(
|
||||||
|
self,
|
||||||
|
fragment: ConfigurationPackageFragment,
|
||||||
|
context: ConfigurationPreflightContext,
|
||||||
|
) -> tuple[list[ConfigurationDiagnostic], str]:
|
||||||
|
diagnostics: list[ConfigurationDiagnostic] = []
|
||||||
|
payload = _files_fragment_payload(fragment, diagnostics)
|
||||||
|
if payload is None:
|
||||||
|
return diagnostics, fragment.fragment_id or "files.storage"
|
||||||
|
receipt_state = _files_receipt_capability(
|
||||||
|
fragment,
|
||||||
|
context,
|
||||||
|
payload,
|
||||||
|
diagnostics,
|
||||||
|
)
|
||||||
|
if receipt_state is None:
|
||||||
|
return diagnostics, fragment.fragment_id or "files.storage"
|
||||||
|
_receipt, capability, binding_ref = receipt_state
|
||||||
|
expected_backend = _expected_backend(capability.source)
|
||||||
|
diagnostics.extend(
|
||||||
|
_package_expectation_diagnostics(
|
||||||
|
payload,
|
||||||
|
capability,
|
||||||
|
expected_backend,
|
||||||
|
binding_ref,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
active_backend = _normalized_backend(
|
||||||
|
getattr(self._settings, "file_storage_backend", "local")
|
||||||
|
)
|
||||||
|
if active_backend != expected_backend:
|
||||||
|
diagnostics.append(
|
||||||
|
ConfigurationDiagnostic(
|
||||||
|
severity="blocker",
|
||||||
|
code="files_storage_runtime_mismatch",
|
||||||
|
message=(
|
||||||
|
f"Files runtime uses {active_backend!r}, while the deployment receipt declares {expected_backend!r}."
|
||||||
|
),
|
||||||
|
module_id="files",
|
||||||
|
object_ref=binding_ref,
|
||||||
|
resolution="Reconcile deployment environment and receipt before starting or importing Files configuration.",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return diagnostics, binding_ref
|
||||||
|
diagnostics.extend(
|
||||||
|
self._runtime_storage_diagnostics(
|
||||||
|
expected_backend,
|
||||||
|
capability,
|
||||||
|
binding_ref,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if "files" not in capability.dependent_modules:
|
||||||
|
diagnostics.append(
|
||||||
|
ConfigurationDiagnostic(
|
||||||
|
severity="info",
|
||||||
|
code="infrastructure_consumer_not_declared",
|
||||||
|
message="Files is active but was not selected as a receipt consumer when the deployment plan was generated.",
|
||||||
|
module_id="files",
|
||||||
|
object_ref=binding_ref,
|
||||||
|
resolution="Regenerate the deployment plan so removal-impact inventory includes Files.",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return diagnostics, binding_ref
|
||||||
|
|
||||||
|
def _runtime_storage_diagnostics(
|
||||||
|
self,
|
||||||
|
expected_backend: str,
|
||||||
|
capability: InfrastructureCapability,
|
||||||
|
binding_ref: str,
|
||||||
|
) -> list[ConfigurationDiagnostic]:
|
||||||
|
if expected_backend == "s3":
|
||||||
|
return self._s3_diagnostics(
|
||||||
|
capability.endpoint,
|
||||||
|
capability.secret_refs,
|
||||||
|
capability.source,
|
||||||
|
binding_ref,
|
||||||
|
)
|
||||||
|
root = Path(
|
||||||
|
str(getattr(self._settings, "file_storage_local_root", "") or "")
|
||||||
|
)
|
||||||
|
if root.is_absolute():
|
||||||
|
return []
|
||||||
|
return [
|
||||||
|
ConfigurationDiagnostic(
|
||||||
|
severity="blocker",
|
||||||
|
code="files_storage_local_root_not_durable",
|
||||||
|
message="Receipt-bound local file storage requires an absolute deployment-managed path.",
|
||||||
|
module_id="files",
|
||||||
|
object_ref=binding_ref,
|
||||||
|
resolution="Set FILE_STORAGE_LOCAL_ROOT to the mounted persistent-volume path.",
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
|
def _s3_diagnostics(
|
||||||
|
self,
|
||||||
|
endpoint: Mapping[str, object],
|
||||||
|
secret_refs: tuple[str, ...],
|
||||||
|
source: str,
|
||||||
|
object_ref: str,
|
||||||
|
) -> list[ConfigurationDiagnostic]:
|
||||||
|
diagnostics: list[ConfigurationDiagnostic] = []
|
||||||
|
endpoint_url = str(
|
||||||
|
getattr(self._settings, "file_storage_s3_endpoint_url", "")
|
||||||
|
or getattr(self._settings, "s3_endpoint_url", "")
|
||||||
|
or ""
|
||||||
|
).strip()
|
||||||
|
runtime_endpoint = _redacted_endpoint(
|
||||||
|
endpoint_url,
|
||||||
|
default_port=int(endpoint.get("port") or 443),
|
||||||
|
)
|
||||||
|
receipt_endpoint = {
|
||||||
|
key: endpoint.get(key) for key in ("scheme", "host", "port") if endpoint.get(key) is not None
|
||||||
|
}
|
||||||
|
if receipt_endpoint and runtime_endpoint != receipt_endpoint:
|
||||||
|
diagnostics.append(
|
||||||
|
ConfigurationDiagnostic(
|
||||||
|
severity="blocker",
|
||||||
|
code="files_storage_endpoint_mismatch",
|
||||||
|
message="Files S3 endpoint does not match the sanitized deployment receipt endpoint.",
|
||||||
|
module_id="files",
|
||||||
|
object_ref=object_ref,
|
||||||
|
resolution="Reconcile FILE_STORAGE_S3_ENDPOINT_URL and the deployment plan before apply.",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
bucket = str(
|
||||||
|
getattr(self._settings, "file_storage_s3_bucket", "")
|
||||||
|
or getattr(self._settings, "s3_bucket", "")
|
||||||
|
or ""
|
||||||
|
).strip()
|
||||||
|
if not bucket:
|
||||||
|
diagnostics.append(
|
||||||
|
ConfigurationDiagnostic(
|
||||||
|
severity="blocker",
|
||||||
|
code="files_storage_bucket_missing",
|
||||||
|
message="Files S3 storage has no configured bucket.",
|
||||||
|
module_id="files",
|
||||||
|
object_ref=object_ref,
|
||||||
|
resolution="Set FILE_STORAGE_S3_BUCKET in the deployment environment.",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
for reference in secret_refs:
|
||||||
|
variable = reference.removeprefix("env:")
|
||||||
|
if not variable.startswith("FILE_STORAGE_"):
|
||||||
|
continue
|
||||||
|
if not str(self._environment.get(variable, "")).strip():
|
||||||
|
diagnostics.append(
|
||||||
|
ConfigurationDiagnostic(
|
||||||
|
severity="blocker",
|
||||||
|
code="files_storage_secret_reference_unresolved",
|
||||||
|
message=f"Required storage secret reference {reference} is not available to the runtime.",
|
||||||
|
module_id="files",
|
||||||
|
object_ref=object_ref,
|
||||||
|
resolution="Provide the referenced environment secret without copying its value into the package.",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
deployment_managed = bool(
|
||||||
|
getattr(
|
||||||
|
self._settings,
|
||||||
|
"file_storage_s3_deployment_managed",
|
||||||
|
False,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
endpoint_trusted = bool(
|
||||||
|
getattr(self._settings, "file_storage_s3_endpoint_trusted", False)
|
||||||
|
)
|
||||||
|
if source == "installer-managed-garage" and not deployment_managed:
|
||||||
|
diagnostics.append(
|
||||||
|
ConfigurationDiagnostic(
|
||||||
|
severity="blocker",
|
||||||
|
code="files_storage_management_boundary_mismatch",
|
||||||
|
message="Garage is installer-managed in the receipt but not marked deployment-managed in Files runtime.",
|
||||||
|
module_id="files",
|
||||||
|
object_ref=object_ref,
|
||||||
|
resolution="Set FILE_STORAGE_S3_DEPLOYMENT_MANAGED=true for the installer-managed Garage endpoint.",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if source == "operator-supplied-s3" and not endpoint_trusted:
|
||||||
|
diagnostics.append(
|
||||||
|
ConfigurationDiagnostic(
|
||||||
|
severity="blocker",
|
||||||
|
code="files_storage_trust_boundary_missing",
|
||||||
|
message="External S3 storage must be explicitly marked as a trusted deployment endpoint.",
|
||||||
|
module_id="files",
|
||||||
|
object_ref=object_ref,
|
||||||
|
resolution="Review the endpoint and set FILE_STORAGE_S3_ENDPOINT_TRUSTED=true in deployment configuration.",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return diagnostics
|
||||||
|
|
||||||
|
|
||||||
|
def _files_fragment_payload(
|
||||||
|
fragment: ConfigurationPackageFragment,
|
||||||
|
diagnostics: list[ConfigurationDiagnostic],
|
||||||
|
) -> Mapping[str, Any] | None:
|
||||||
|
payload = fragment.payload
|
||||||
|
if not isinstance(payload, Mapping):
|
||||||
|
diagnostics.append(
|
||||||
|
_invalid(fragment, "Managed-storage payload must be an object.")
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
unknown = sorted(set(payload) - _PAYLOAD_KEYS)
|
||||||
|
if not unknown:
|
||||||
|
return payload
|
||||||
|
secret_like = any(
|
||||||
|
marker in key.casefold()
|
||||||
|
for key in unknown
|
||||||
|
for marker in ("password", "secret", "token", "credential", "access_key")
|
||||||
|
)
|
||||||
|
diagnostics.append(
|
||||||
|
ConfigurationDiagnostic(
|
||||||
|
severity="blocker",
|
||||||
|
code=(
|
||||||
|
"files_configuration_secret_forbidden"
|
||||||
|
if secret_like
|
||||||
|
else "files_configuration_payload_invalid"
|
||||||
|
),
|
||||||
|
message=(
|
||||||
|
"Managed-storage fragments accept receipt references and non-secret expectations only."
|
||||||
|
if secret_like
|
||||||
|
else f"Managed-storage payload contains unsupported fields: {', '.join(unknown)}."
|
||||||
|
),
|
||||||
|
module_id="files",
|
||||||
|
object_ref=fragment.fragment_id or MANAGED_STORAGE_FRAGMENT,
|
||||||
|
resolution="Keep storage credentials in deployment environment references, never in a configuration package.",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return payload
|
||||||
|
|
||||||
|
|
||||||
|
def _files_receipt_capability(
|
||||||
|
fragment: ConfigurationPackageFragment,
|
||||||
|
context: ConfigurationPreflightContext,
|
||||||
|
payload: Mapping[str, Any],
|
||||||
|
diagnostics: list[ConfigurationDiagnostic],
|
||||||
|
) -> tuple[
|
||||||
|
InfrastructureCapabilityReceipt,
|
||||||
|
InfrastructureCapability,
|
||||||
|
str,
|
||||||
|
] | None:
|
||||||
|
if context.infrastructure_receipt_error:
|
||||||
|
diagnostics.append(
|
||||||
|
_receipt_error(
|
||||||
|
context.infrastructure_receipt_error,
|
||||||
|
object_ref=fragment.fragment_id or "files.storage",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
receipt = context.infrastructure_receipt
|
||||||
|
if receipt is None:
|
||||||
|
diagnostics.append(
|
||||||
|
ConfigurationDiagnostic(
|
||||||
|
severity="blocker",
|
||||||
|
code="infrastructure_receipt_missing",
|
||||||
|
message="Files managed-storage configuration requires the deployment capability receipt.",
|
||||||
|
module_id="files",
|
||||||
|
object_ref=fragment.fragment_id or "files.storage",
|
||||||
|
resolution="Mount the installer-generated receipt and rerun preflight.",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
binding_ref = fragment.fragment_id or f"{receipt.installation_id}:files.storage"
|
||||||
|
capability_id = _text(payload.get("capability_id")) or "files.storage"
|
||||||
|
if capability_id != "files.storage":
|
||||||
|
diagnostics.append(
|
||||||
|
_invalid(
|
||||||
|
fragment,
|
||||||
|
"Files managed-storage fragments must reference capability 'files.storage'.",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
capability = receipt.capability(capability_id)
|
||||||
|
if capability is None:
|
||||||
|
diagnostics.append(
|
||||||
|
ConfigurationDiagnostic(
|
||||||
|
severity="blocker",
|
||||||
|
code="infrastructure_capability_missing",
|
||||||
|
message="The deployment receipt does not declare managed file storage.",
|
||||||
|
module_id="files",
|
||||||
|
object_ref=binding_ref,
|
||||||
|
resolution="Regenerate the receipt from a deployment profile that declares managed storage.",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
state_diagnostic = _storage_capability_state_diagnostic(capability, binding_ref)
|
||||||
|
if state_diagnostic is not None:
|
||||||
|
diagnostics.append(state_diagnostic)
|
||||||
|
return None
|
||||||
|
return receipt, capability, binding_ref
|
||||||
|
|
||||||
|
|
||||||
|
def _storage_capability_state_diagnostic(
|
||||||
|
capability: InfrastructureCapability,
|
||||||
|
binding_ref: str,
|
||||||
|
) -> ConfigurationDiagnostic | None:
|
||||||
|
if capability.state == "unavailable":
|
||||||
|
return ConfigurationDiagnostic(
|
||||||
|
severity="blocker",
|
||||||
|
code="infrastructure_capability_unavailable",
|
||||||
|
message="The deployment receipt states that managed file storage is unavailable.",
|
||||||
|
module_id="files",
|
||||||
|
object_ref=binding_ref,
|
||||||
|
resolution="Select local, managed Garage, or external S3 storage in the deployment profile.",
|
||||||
|
)
|
||||||
|
if capability.state == "available_unconfigured":
|
||||||
|
return ConfigurationDiagnostic(
|
||||||
|
severity="blocker",
|
||||||
|
code="infrastructure_capability_unconfigured",
|
||||||
|
message="Managed file storage is available but has not been bound by the deployment runtime.",
|
||||||
|
module_id="files",
|
||||||
|
object_ref=binding_ref,
|
||||||
|
resolution="Complete the deployment-owned storage configuration before importing Files configuration.",
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _package_expectation_diagnostics(
|
||||||
|
payload: Mapping[str, Any],
|
||||||
|
capability: InfrastructureCapability,
|
||||||
|
expected_backend: str,
|
||||||
|
binding_ref: str,
|
||||||
|
) -> list[ConfigurationDiagnostic]:
|
||||||
|
diagnostics: list[ConfigurationDiagnostic] = []
|
||||||
|
package_backend = (
|
||||||
|
_text(payload.get("expected_backend")) or expected_backend
|
||||||
|
).casefold()
|
||||||
|
if package_backend != expected_backend:
|
||||||
|
diagnostics.append(
|
||||||
|
ConfigurationDiagnostic(
|
||||||
|
severity="blocker",
|
||||||
|
code="infrastructure_backend_mismatch",
|
||||||
|
message="The package storage backend expectation conflicts with the deployment receipt.",
|
||||||
|
module_id="files",
|
||||||
|
object_ref=binding_ref,
|
||||||
|
resolution="Use the receipt backend or regenerate the deployment package after review.",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
package_source = _text(payload.get("expected_source"))
|
||||||
|
if package_source and package_source != capability.source:
|
||||||
|
diagnostics.append(
|
||||||
|
ConfigurationDiagnostic(
|
||||||
|
severity="blocker",
|
||||||
|
code="infrastructure_source_mismatch",
|
||||||
|
message="The package storage source expectation conflicts with the deployment receipt.",
|
||||||
|
module_id="files",
|
||||||
|
object_ref=binding_ref,
|
||||||
|
resolution="Review the provider replacement and regenerate the package from the active receipt.",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return diagnostics
|
||||||
|
|
||||||
|
|
||||||
|
def _expected_backend(source: str) -> str:
|
||||||
|
return "local" if source == "host-local" else "s3"
|
||||||
|
|
||||||
|
|
||||||
|
def _normalized_backend(value: object) -> str:
|
||||||
|
clean = str(value or "local").strip().casefold()
|
||||||
|
if clean in {"local", "filesystem", "fs"}:
|
||||||
|
return "local"
|
||||||
|
if clean in {"s3", "garage"}:
|
||||||
|
return "s3"
|
||||||
|
return clean
|
||||||
|
|
||||||
|
|
||||||
|
def _redacted_endpoint(value: str, *, default_port: int) -> dict[str, object]:
|
||||||
|
try:
|
||||||
|
parsed = urlsplit(value)
|
||||||
|
host = parsed.hostname
|
||||||
|
if not parsed.scheme or not host:
|
||||||
|
return {"reference": "unresolved"}
|
||||||
|
port = parsed.port or default_port
|
||||||
|
except ValueError:
|
||||||
|
return {"reference": "unresolved"}
|
||||||
|
return {"scheme": parsed.scheme, "host": host, "port": port}
|
||||||
|
|
||||||
|
|
||||||
|
def _text(value: object) -> str | None:
|
||||||
|
if value is None:
|
||||||
|
return None
|
||||||
|
text = str(value).strip()
|
||||||
|
return text or None
|
||||||
|
|
||||||
|
|
||||||
|
def _receipt_error(error: str, *, object_ref: str) -> ConfigurationDiagnostic:
|
||||||
|
return ConfigurationDiagnostic(
|
||||||
|
severity="blocker",
|
||||||
|
code="infrastructure_receipt_invalid",
|
||||||
|
message=f"The deployment capability receipt is invalid: {error}",
|
||||||
|
module_id="files",
|
||||||
|
object_ref=object_ref,
|
||||||
|
resolution="Repair or regenerate the deployment receipt before importing Files configuration.",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _invalid(
|
||||||
|
fragment: ConfigurationPackageFragment,
|
||||||
|
message: str,
|
||||||
|
) -> ConfigurationDiagnostic:
|
||||||
|
return ConfigurationDiagnostic(
|
||||||
|
severity="blocker",
|
||||||
|
code="files_configuration_payload_invalid",
|
||||||
|
message=message,
|
||||||
|
module_id="files",
|
||||||
|
object_ref=fragment.fragment_id or fragment.fragment_type,
|
||||||
|
resolution="Review the Files configuration-package fragment and rerun preflight.",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _unsupported(fragment: ConfigurationPackageFragment) -> ConfigurationDiagnostic:
|
||||||
|
return ConfigurationDiagnostic(
|
||||||
|
severity="blocker",
|
||||||
|
code="fragment_type_unsupported",
|
||||||
|
message=f"Files configuration does not support fragment type {fragment.fragment_type!r}.",
|
||||||
|
module_id="files",
|
||||||
|
object_ref=fragment.fragment_id or fragment.fragment_type,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _blocked_plan(
|
||||||
|
fragment: ConfigurationPackageFragment,
|
||||||
|
summary: str,
|
||||||
|
) -> ConfigurationPlanItem:
|
||||||
|
return ConfigurationPlanItem(
|
||||||
|
action="blocked",
|
||||||
|
module_id="files",
|
||||||
|
fragment_type=fragment.fragment_type,
|
||||||
|
fragment_id=fragment.fragment_id,
|
||||||
|
summary=summary,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["FILES_CONFIGURATION_CAPABILITY", "FilesConfigurationProvider"]
|
||||||
@@ -4,7 +4,18 @@ import uuid
|
|||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from sqlalchemy import Boolean, DateTime, ForeignKey, Index, Integer, JSON, String, Text, UniqueConstraint, text
|
from sqlalchemy import (
|
||||||
|
Boolean,
|
||||||
|
DateTime,
|
||||||
|
ForeignKey,
|
||||||
|
Index,
|
||||||
|
Integer,
|
||||||
|
JSON,
|
||||||
|
String,
|
||||||
|
Text,
|
||||||
|
UniqueConstraint,
|
||||||
|
text,
|
||||||
|
)
|
||||||
from sqlalchemy.orm import Mapped, mapped_column
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
|
||||||
from govoplan_core.db.base import Base, TimestampMixin
|
from govoplan_core.db.base import Base, TimestampMixin
|
||||||
@@ -16,7 +27,15 @@ def new_uuid() -> str:
|
|||||||
|
|
||||||
class FileBlob(Base, TimestampMixin):
|
class FileBlob(Base, TimestampMixin):
|
||||||
__tablename__ = "file_blobs"
|
__tablename__ = "file_blobs"
|
||||||
__table_args__ = (UniqueConstraint("tenant_id", "checksum_sha256", "size_bytes", name="uq_file_blobs_tenant_checksum_size"),)
|
__table_args__ = (
|
||||||
|
UniqueConstraint(
|
||||||
|
"tenant_id",
|
||||||
|
"checksum_sha256",
|
||||||
|
"size_bytes",
|
||||||
|
"protection_discriminator",
|
||||||
|
name="uq_file_blobs_tenant_checksum_size_protection",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||||
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||||
@@ -25,9 +44,106 @@ class FileBlob(Base, TimestampMixin):
|
|||||||
storage_key: Mapped[str] = mapped_column(String(1000), nullable=False)
|
storage_key: Mapped[str] = mapped_column(String(1000), nullable=False)
|
||||||
checksum_sha256: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
checksum_sha256: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||||
size_bytes: Mapped[int] = mapped_column(Integer, nullable=False)
|
size_bytes: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||||
|
protection_discriminator: Mapped[str] = mapped_column(
|
||||||
|
String(320), default="plaintext", nullable=False, index=True
|
||||||
|
)
|
||||||
|
encryption_envelope_id: Mapped[str | None] = mapped_column(
|
||||||
|
String(255), nullable=True, index=True
|
||||||
|
)
|
||||||
|
storage_checksum_sha256: Mapped[str | None] = mapped_column(
|
||||||
|
String(64), nullable=True
|
||||||
|
)
|
||||||
|
storage_size_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||||
content_type: Mapped[str | None] = mapped_column(String(255))
|
content_type: Mapped[str | None] = mapped_column(String(255))
|
||||||
ref_count: Mapped[int] = mapped_column(Integer, default=1, nullable=False)
|
ref_count: Mapped[int] = mapped_column(Integer, default=1, nullable=False)
|
||||||
retained_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
retained_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
|
integrity_status: Mapped[str] = mapped_column(
|
||||||
|
String(30), default="unchecked", nullable=False, index=True
|
||||||
|
)
|
||||||
|
integrity_checked_at: Mapped[datetime | None] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=True, index=True
|
||||||
|
)
|
||||||
|
integrity_failure: Mapped[str | None] = mapped_column(String(100), nullable=True)
|
||||||
|
quarantined_at: Mapped[datetime | None] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=True, index=True
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class FileIntegrityScan(Base, TimestampMixin):
|
||||||
|
__tablename__ = "file_integrity_scans"
|
||||||
|
|
||||||
|
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||||
|
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||||
|
storage_backend: Mapped[str] = mapped_column(String(50), nullable=False)
|
||||||
|
storage_prefix: Mapped[str] = mapped_column(String(1000), nullable=False)
|
||||||
|
status: Mapped[str] = mapped_column(
|
||||||
|
String(30), default="pending", nullable=False, index=True
|
||||||
|
)
|
||||||
|
revision: Mapped[int] = mapped_column(Integer, default=1, nullable=False)
|
||||||
|
phase: Mapped[str] = mapped_column(String(30), default="blobs", nullable=False)
|
||||||
|
verify_checksums: Mapped[bool] = mapped_column(
|
||||||
|
Boolean, default=True, nullable=False
|
||||||
|
)
|
||||||
|
batch_size: Mapped[int] = mapped_column(Integer, default=100, nullable=False)
|
||||||
|
blob_cursor: Mapped[str | None] = mapped_column(String(36), nullable=True)
|
||||||
|
object_cursor: Mapped[str | None] = mapped_column(String(1000), nullable=True)
|
||||||
|
scanned_blob_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
|
||||||
|
verified_blob_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
|
||||||
|
quarantined_blob_count: Mapped[int] = mapped_column(
|
||||||
|
Integer, default=0, nullable=False
|
||||||
|
)
|
||||||
|
scanned_object_count: Mapped[int] = mapped_column(
|
||||||
|
Integer, default=0, nullable=False
|
||||||
|
)
|
||||||
|
orphan_object_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
|
||||||
|
created_by_user_id: Mapped[str | None] = mapped_column(
|
||||||
|
ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True
|
||||||
|
)
|
||||||
|
started_at: Mapped[datetime | None] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=True
|
||||||
|
)
|
||||||
|
completed_at: Mapped[datetime | None] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=True
|
||||||
|
)
|
||||||
|
last_error: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||||
|
|
||||||
|
|
||||||
|
class FileIntegrityFinding(Base, TimestampMixin):
|
||||||
|
__tablename__ = "file_integrity_findings"
|
||||||
|
__table_args__ = (
|
||||||
|
Index("ix_file_integrity_findings_scan_state", "scan_id", "state"),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||||
|
scan_id: Mapped[str] = mapped_column(
|
||||||
|
ForeignKey("file_integrity_scans.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||||
|
kind: Mapped[str] = mapped_column(String(40), nullable=False, index=True)
|
||||||
|
state: Mapped[str] = mapped_column(
|
||||||
|
String(30), default="open", nullable=False, index=True
|
||||||
|
)
|
||||||
|
revision: Mapped[int] = mapped_column(Integer, default=1, nullable=False)
|
||||||
|
blob_id: Mapped[str | None] = mapped_column(
|
||||||
|
ForeignKey("file_blobs.id", ondelete="SET NULL"), nullable=True, index=True
|
||||||
|
)
|
||||||
|
storage_key: Mapped[str] = mapped_column(String(1000), nullable=False)
|
||||||
|
expected_size_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||||
|
observed_size_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||||
|
expected_checksum_sha256: Mapped[str | None] = mapped_column(
|
||||||
|
String(64), nullable=True
|
||||||
|
)
|
||||||
|
observed_checksum_sha256: Mapped[str | None] = mapped_column(
|
||||||
|
String(64), nullable=True
|
||||||
|
)
|
||||||
|
resolved_at: Mapped[datetime | None] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=True
|
||||||
|
)
|
||||||
|
resolved_by_user_id: Mapped[str | None] = mapped_column(
|
||||||
|
ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class FileFolder(Base, TimestampMixin):
|
class FileFolder(Base, TimestampMixin):
|
||||||
@@ -35,14 +151,18 @@ class FileFolder(Base, TimestampMixin):
|
|||||||
__table_args__ = (
|
__table_args__ = (
|
||||||
Index(
|
Index(
|
||||||
"uq_file_folders_active_user_path",
|
"uq_file_folders_active_user_path",
|
||||||
"tenant_id", "owner_user_id", "path",
|
"tenant_id",
|
||||||
|
"owner_user_id",
|
||||||
|
"path",
|
||||||
unique=True,
|
unique=True,
|
||||||
sqlite_where=text("owner_type = 'user' AND deleted_at IS NULL"),
|
sqlite_where=text("owner_type = 'user' AND deleted_at IS NULL"),
|
||||||
postgresql_where=text("owner_type = 'user' AND deleted_at IS NULL"),
|
postgresql_where=text("owner_type = 'user' AND deleted_at IS NULL"),
|
||||||
),
|
),
|
||||||
Index(
|
Index(
|
||||||
"uq_file_folders_active_group_path",
|
"uq_file_folders_active_group_path",
|
||||||
"tenant_id", "owner_group_id", "path",
|
"tenant_id",
|
||||||
|
"owner_group_id",
|
||||||
|
"path",
|
||||||
unique=True,
|
unique=True,
|
||||||
sqlite_where=text("owner_type = 'group' AND deleted_at IS NULL"),
|
sqlite_where=text("owner_type = 'group' AND deleted_at IS NULL"),
|
||||||
postgresql_where=text("owner_type = 'group' AND deleted_at IS NULL"),
|
postgresql_where=text("owner_type = 'group' AND deleted_at IS NULL"),
|
||||||
@@ -52,12 +172,22 @@ class FileFolder(Base, TimestampMixin):
|
|||||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||||
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||||
owner_type: Mapped[str] = mapped_column(String(20), nullable=False, index=True)
|
owner_type: Mapped[str] = mapped_column(String(20), nullable=False, index=True)
|
||||||
owner_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
|
owner_user_id: Mapped[str | None] = mapped_column(
|
||||||
owner_group_id: Mapped[str | None] = mapped_column(ForeignKey("access_groups.id", ondelete="SET NULL"), nullable=True, index=True)
|
ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True
|
||||||
|
)
|
||||||
|
owner_group_id: Mapped[str | None] = mapped_column(
|
||||||
|
ForeignKey("access_groups.id", ondelete="SET NULL"), nullable=True, index=True
|
||||||
|
)
|
||||||
path: Mapped[str] = mapped_column(String(1000), nullable=False, index=True)
|
path: Mapped[str] = mapped_column(String(1000), nullable=False, index=True)
|
||||||
created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
|
created_by_user_id: Mapped[str | None] = mapped_column(
|
||||||
deleted_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
|
ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True
|
||||||
metadata_: Mapped[dict[str, Any] | None] = mapped_column("metadata", JSON, nullable=True)
|
)
|
||||||
|
deleted_at: Mapped[datetime | None] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=True, index=True
|
||||||
|
)
|
||||||
|
metadata_: Mapped[dict[str, Any] | None] = mapped_column(
|
||||||
|
"metadata", JSON, nullable=True
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class FileAsset(Base, TimestampMixin):
|
class FileAsset(Base, TimestampMixin):
|
||||||
@@ -66,46 +196,159 @@ class FileAsset(Base, TimestampMixin):
|
|||||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||||
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||||
owner_type: Mapped[str] = mapped_column(String(20), nullable=False, index=True)
|
owner_type: Mapped[str] = mapped_column(String(20), nullable=False, index=True)
|
||||||
owner_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
|
owner_user_id: Mapped[str | None] = mapped_column(
|
||||||
owner_group_id: Mapped[str | None] = mapped_column(ForeignKey("access_groups.id", ondelete="SET NULL"), nullable=True, index=True)
|
ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True
|
||||||
current_version_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
|
)
|
||||||
|
owner_group_id: Mapped[str | None] = mapped_column(
|
||||||
|
ForeignKey("access_groups.id", ondelete="SET NULL"), nullable=True, index=True
|
||||||
|
)
|
||||||
|
current_version_id: Mapped[str | None] = mapped_column(
|
||||||
|
String(36), nullable=True, index=True
|
||||||
|
)
|
||||||
display_path: Mapped[str] = mapped_column(String(1000), nullable=False, index=True)
|
display_path: Mapped[str] = mapped_column(String(1000), nullable=False, index=True)
|
||||||
filename: Mapped[str] = mapped_column(String(500), nullable=False, index=True)
|
filename: Mapped[str] = mapped_column(String(500), nullable=False, index=True)
|
||||||
description: Mapped[str | None] = mapped_column(Text)
|
description: Mapped[str | None] = mapped_column(Text)
|
||||||
created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
|
created_by_user_id: Mapped[str | None] = mapped_column(
|
||||||
deleted_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
|
ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True
|
||||||
metadata_: Mapped[dict[str, Any] | None] = mapped_column("metadata", JSON, nullable=True)
|
)
|
||||||
|
deleted_at: Mapped[datetime | None] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=True, index=True
|
||||||
|
)
|
||||||
|
retained_until: Mapped[datetime | None] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=True, index=True
|
||||||
|
)
|
||||||
|
legal_hold: Mapped[bool] = mapped_column(
|
||||||
|
Boolean, default=False, nullable=False, index=True
|
||||||
|
)
|
||||||
|
lifecycle_revision: Mapped[int] = mapped_column(
|
||||||
|
Integer, default=1, nullable=False
|
||||||
|
)
|
||||||
|
lifecycle_reason: Mapped[str | None] = mapped_column(String(500), nullable=True)
|
||||||
|
metadata_: Mapped[dict[str, Any] | None] = mapped_column(
|
||||||
|
"metadata", JSON, nullable=True
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class FileVersion(Base, TimestampMixin):
|
class FileVersion(Base, TimestampMixin):
|
||||||
__tablename__ = "file_versions"
|
__tablename__ = "file_versions"
|
||||||
__table_args__ = (UniqueConstraint("file_asset_id", "version_number", name="uq_file_versions_asset_number"),)
|
__table_args__ = (
|
||||||
|
UniqueConstraint(
|
||||||
|
"file_asset_id", "version_number", name="uq_file_versions_asset_number"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||||
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||||
file_asset_id: Mapped[str] = mapped_column(ForeignKey("file_assets.id", ondelete="CASCADE"), nullable=False, index=True)
|
file_asset_id: Mapped[str] = mapped_column(
|
||||||
blob_id: Mapped[str] = mapped_column(ForeignKey("file_blobs.id", ondelete="RESTRICT"), nullable=False, index=True)
|
ForeignKey("file_assets.id", ondelete="CASCADE"), nullable=False, index=True
|
||||||
|
)
|
||||||
|
blob_id: Mapped[str] = mapped_column(
|
||||||
|
ForeignKey("file_blobs.id", ondelete="RESTRICT"), nullable=False, index=True
|
||||||
|
)
|
||||||
version_number: Mapped[int] = mapped_column(Integer, nullable=False)
|
version_number: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||||
filename_at_upload: Mapped[str] = mapped_column(String(500), nullable=False)
|
filename_at_upload: Mapped[str] = mapped_column(String(500), nullable=False)
|
||||||
display_path_at_upload: Mapped[str] = mapped_column(String(1000), nullable=False)
|
display_path_at_upload: Mapped[str] = mapped_column(String(1000), nullable=False)
|
||||||
content_type: Mapped[str | None] = mapped_column(String(255))
|
content_type: Mapped[str | None] = mapped_column(String(255))
|
||||||
size_bytes: Mapped[int] = mapped_column(Integer, nullable=False)
|
size_bytes: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||||
checksum_sha256: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
checksum_sha256: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||||
created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
|
created_by_user_id: Mapped[str | None] = mapped_column(
|
||||||
|
ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class FileFormEvidenceGrant(Base, TimestampMixin):
|
||||||
|
__tablename__ = "file_form_evidence_grants"
|
||||||
|
__table_args__ = (
|
||||||
|
UniqueConstraint(
|
||||||
|
"tenant_id",
|
||||||
|
"idempotency_key",
|
||||||
|
name="uq_file_form_evidence_grants_idempotency",
|
||||||
|
),
|
||||||
|
Index(
|
||||||
|
"ix_file_form_evidence_grants_form",
|
||||||
|
"tenant_id",
|
||||||
|
"form_instance_id",
|
||||||
|
"form_definition_id",
|
||||||
|
"form_definition_revision",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||||
|
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||||
|
form_instance_id: Mapped[str] = mapped_column(
|
||||||
|
String(36), nullable=False, index=True
|
||||||
|
)
|
||||||
|
form_definition_id: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||||
|
form_definition_revision: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||||
|
token_sha256: Mapped[str] = mapped_column(String(64), nullable=False, unique=True)
|
||||||
|
idempotency_key: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||||
|
request_sha256: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||||
|
custodian_user_id: Mapped[str] = mapped_column(
|
||||||
|
ForeignKey("access_users.id", ondelete="RESTRICT"),
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
evidence_kind: Mapped[str] = mapped_column(String(30), nullable=False)
|
||||||
|
purpose: Mapped[str] = mapped_column(String(500), nullable=False)
|
||||||
|
status: Mapped[str] = mapped_column(
|
||||||
|
String(30), default="issued", nullable=False, index=True
|
||||||
|
)
|
||||||
|
expires_at: Mapped[datetime] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=False, index=True
|
||||||
|
)
|
||||||
|
max_size_bytes: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||||
|
allowed_content_types: Mapped[list[str]] = mapped_column(
|
||||||
|
JSON, default=list, nullable=False
|
||||||
|
)
|
||||||
|
file_asset_id: Mapped[str | None] = mapped_column(
|
||||||
|
ForeignKey("file_assets.id", ondelete="RESTRICT"), nullable=True, index=True
|
||||||
|
)
|
||||||
|
file_version_id: Mapped[str | None] = mapped_column(
|
||||||
|
ForeignKey("file_versions.id", ondelete="RESTRICT"), nullable=True, index=True
|
||||||
|
)
|
||||||
|
uploaded_at: Mapped[datetime | None] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=True
|
||||||
|
)
|
||||||
|
revoked_at: Mapped[datetime | None] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=True
|
||||||
|
)
|
||||||
|
metadata_: Mapped[dict[str, Any]] = mapped_column(
|
||||||
|
"metadata", JSON, default=dict, nullable=False
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class FileShare(Base, TimestampMixin):
|
class FileShare(Base, TimestampMixin):
|
||||||
__tablename__ = "file_shares"
|
__tablename__ = "file_shares"
|
||||||
__table_args__ = (UniqueConstraint("file_asset_id", "target_type", "target_id", "revoked_at", name="uq_file_shares_active_target"),)
|
__table_args__ = (
|
||||||
|
UniqueConstraint(
|
||||||
|
"file_asset_id",
|
||||||
|
"target_type",
|
||||||
|
"target_id",
|
||||||
|
"revoked_at",
|
||||||
|
name="uq_file_shares_active_target",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||||
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||||
file_asset_id: Mapped[str] = mapped_column(ForeignKey("file_assets.id", ondelete="CASCADE"), nullable=False, index=True)
|
file_asset_id: Mapped[str] = mapped_column(
|
||||||
|
ForeignKey("file_assets.id", ondelete="CASCADE"), nullable=False, index=True
|
||||||
|
)
|
||||||
target_type: Mapped[str] = mapped_column(String(20), nullable=False, index=True)
|
target_type: Mapped[str] = mapped_column(String(20), nullable=False, index=True)
|
||||||
target_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
target_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||||
permission: Mapped[str] = mapped_column(String(20), default="read", nullable=False)
|
permission: Mapped[str] = mapped_column(String(20), default="read", nullable=False)
|
||||||
created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
|
created_by_user_id: Mapped[str | None] = mapped_column(
|
||||||
revoked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
|
ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True
|
||||||
|
)
|
||||||
|
expires_at: Mapped[datetime | None] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=True, index=True
|
||||||
|
)
|
||||||
|
revoked_at: Mapped[datetime | None] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=True, index=True
|
||||||
|
)
|
||||||
|
revoked_by_user_id: Mapped[str | None] = mapped_column(
|
||||||
|
ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class FileConnectorProfile(Base, TimestampMixin):
|
class FileConnectorProfile(Base, TimestampMixin):
|
||||||
@@ -116,15 +359,23 @@ class FileConnectorProfile(Base, TimestampMixin):
|
|||||||
|
|
||||||
id: Mapped[str] = mapped_column(String(255), primary_key=True)
|
id: Mapped[str] = mapped_column(String(255), primary_key=True)
|
||||||
tenant_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
|
tenant_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
|
||||||
scope_type: Mapped[str] = mapped_column(String(20), default="tenant", nullable=False, index=True)
|
scope_type: Mapped[str] = mapped_column(
|
||||||
|
String(20), default="tenant", nullable=False, index=True
|
||||||
|
)
|
||||||
scope_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
|
scope_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
|
||||||
label: Mapped[str] = mapped_column(String(255), nullable=False)
|
label: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||||
provider: Mapped[str] = mapped_column(String(50), nullable=False, index=True)
|
provider: Mapped[str] = mapped_column(String(50), nullable=False, index=True)
|
||||||
endpoint_url: Mapped[str | None] = mapped_column(String(1000), nullable=True)
|
endpoint_url: Mapped[str | None] = mapped_column(String(1000), nullable=True)
|
||||||
base_path: Mapped[str | None] = mapped_column(String(1000), nullable=True)
|
base_path: Mapped[str | None] = mapped_column(String(1000), nullable=True)
|
||||||
enabled: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False, index=True)
|
enabled: Mapped[bool] = mapped_column(
|
||||||
credential_profile_id: Mapped[str | None] = mapped_column(String(255), nullable=True, index=True)
|
Boolean, default=True, nullable=False, index=True
|
||||||
credential_mode: Mapped[str] = mapped_column(String(30), default="none", nullable=False)
|
)
|
||||||
|
credential_profile_id: Mapped[str | None] = mapped_column(
|
||||||
|
String(255), nullable=True, index=True
|
||||||
|
)
|
||||||
|
credential_mode: Mapped[str] = mapped_column(
|
||||||
|
String(30), default="none", nullable=False
|
||||||
|
)
|
||||||
username: Mapped[str | None] = mapped_column(String(320), nullable=True)
|
username: Mapped[str | None] = mapped_column(String(320), nullable=True)
|
||||||
password_encrypted: Mapped[str | None] = mapped_column(Text, nullable=True)
|
password_encrypted: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
token_encrypted: Mapped[str | None] = mapped_column(Text, nullable=True)
|
token_encrypted: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
@@ -133,9 +384,15 @@ class FileConnectorProfile(Base, TimestampMixin):
|
|||||||
secret_ref: Mapped[str | None] = mapped_column(String(1000), nullable=True)
|
secret_ref: Mapped[str | None] = mapped_column(String(1000), nullable=True)
|
||||||
capabilities: Mapped[list[str] | None] = mapped_column(JSON, nullable=True)
|
capabilities: Mapped[list[str] | None] = mapped_column(JSON, nullable=True)
|
||||||
policy: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
|
policy: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
|
||||||
metadata_: Mapped[dict[str, Any] | None] = mapped_column("metadata", JSON, nullable=True)
|
metadata_: Mapped[dict[str, Any] | None] = mapped_column(
|
||||||
created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
|
"metadata", JSON, nullable=True
|
||||||
updated_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
|
)
|
||||||
|
created_by_user_id: Mapped[str | None] = mapped_column(
|
||||||
|
ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True
|
||||||
|
)
|
||||||
|
updated_by_user_id: Mapped[str | None] = mapped_column(
|
||||||
|
ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class FileConnectorCredential(Base, TimestampMixin):
|
class FileConnectorCredential(Base, TimestampMixin):
|
||||||
@@ -146,12 +403,18 @@ class FileConnectorCredential(Base, TimestampMixin):
|
|||||||
|
|
||||||
id: Mapped[str] = mapped_column(String(255), primary_key=True)
|
id: Mapped[str] = mapped_column(String(255), primary_key=True)
|
||||||
tenant_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
|
tenant_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
|
||||||
scope_type: Mapped[str] = mapped_column(String(20), default="tenant", nullable=False, index=True)
|
scope_type: Mapped[str] = mapped_column(
|
||||||
|
String(20), default="tenant", nullable=False, index=True
|
||||||
|
)
|
||||||
scope_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
|
scope_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
|
||||||
label: Mapped[str] = mapped_column(String(255), nullable=False)
|
label: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||||
provider: Mapped[str | None] = mapped_column(String(50), nullable=True, index=True)
|
provider: Mapped[str | None] = mapped_column(String(50), nullable=True, index=True)
|
||||||
enabled: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False, index=True)
|
enabled: Mapped[bool] = mapped_column(
|
||||||
credential_mode: Mapped[str] = mapped_column(String(30), default="none", nullable=False)
|
Boolean, default=True, nullable=False, index=True
|
||||||
|
)
|
||||||
|
credential_mode: Mapped[str] = mapped_column(
|
||||||
|
String(30), default="none", nullable=False
|
||||||
|
)
|
||||||
username: Mapped[str | None] = mapped_column(String(320), nullable=True)
|
username: Mapped[str | None] = mapped_column(String(320), nullable=True)
|
||||||
password_encrypted: Mapped[str | None] = mapped_column(Text, nullable=True)
|
password_encrypted: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
token_encrypted: Mapped[str | None] = mapped_column(Text, nullable=True)
|
token_encrypted: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
@@ -159,15 +422,26 @@ class FileConnectorCredential(Base, TimestampMixin):
|
|||||||
token_env: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
token_env: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||||
secret_ref: Mapped[str | None] = mapped_column(String(1000), nullable=True)
|
secret_ref: Mapped[str | None] = mapped_column(String(1000), nullable=True)
|
||||||
policy: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
|
policy: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
|
||||||
metadata_: Mapped[dict[str, Any] | None] = mapped_column("metadata", JSON, nullable=True)
|
metadata_: Mapped[dict[str, Any] | None] = mapped_column(
|
||||||
created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
|
"metadata", JSON, nullable=True
|
||||||
updated_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
|
)
|
||||||
|
created_by_user_id: Mapped[str | None] = mapped_column(
|
||||||
|
ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True
|
||||||
|
)
|
||||||
|
updated_by_user_id: Mapped[str | None] = mapped_column(
|
||||||
|
ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class FileConnectorPolicy(Base, TimestampMixin):
|
class FileConnectorPolicy(Base, TimestampMixin):
|
||||||
__tablename__ = "file_connector_policies"
|
__tablename__ = "file_connector_policies"
|
||||||
__table_args__ = (
|
__table_args__ = (
|
||||||
UniqueConstraint("tenant_id", "scope_type", "scope_id", name="uq_file_connector_policies_scope"),
|
UniqueConstraint(
|
||||||
|
"tenant_id",
|
||||||
|
"scope_type",
|
||||||
|
"scope_id",
|
||||||
|
name="uq_file_connector_policies_scope",
|
||||||
|
),
|
||||||
Index("ix_file_connector_policies_scope", "scope_type", "scope_id"),
|
Index("ix_file_connector_policies_scope", "scope_type", "scope_id"),
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -176,24 +450,38 @@ class FileConnectorPolicy(Base, TimestampMixin):
|
|||||||
scope_type: Mapped[str] = mapped_column(String(20), nullable=False, index=True)
|
scope_type: Mapped[str] = mapped_column(String(20), nullable=False, index=True)
|
||||||
scope_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
|
scope_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
|
||||||
policy: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict, nullable=False)
|
policy: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict, nullable=False)
|
||||||
created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
|
created_by_user_id: Mapped[str | None] = mapped_column(
|
||||||
updated_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
|
ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True
|
||||||
|
)
|
||||||
|
updated_by_user_id: Mapped[str | None] = mapped_column(
|
||||||
|
ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class FileConnectorSpace(Base, TimestampMixin):
|
class FileConnectorSpace(Base, TimestampMixin):
|
||||||
__tablename__ = "file_connector_spaces"
|
__tablename__ = "file_connector_spaces"
|
||||||
__table_args__ = (
|
__table_args__ = (
|
||||||
Index("ix_file_connector_spaces_owner", "tenant_id", "owner_type", "owner_user_id", "owner_group_id"),
|
Index(
|
||||||
|
"ix_file_connector_spaces_owner",
|
||||||
|
"tenant_id",
|
||||||
|
"owner_type",
|
||||||
|
"owner_user_id",
|
||||||
|
"owner_group_id",
|
||||||
|
),
|
||||||
Index(
|
Index(
|
||||||
"uq_file_connector_spaces_active_user_label",
|
"uq_file_connector_spaces_active_user_label",
|
||||||
"tenant_id", "owner_user_id", "label",
|
"tenant_id",
|
||||||
|
"owner_user_id",
|
||||||
|
"label",
|
||||||
unique=True,
|
unique=True,
|
||||||
sqlite_where=text("owner_type = 'user' AND deleted_at IS NULL"),
|
sqlite_where=text("owner_type = 'user' AND deleted_at IS NULL"),
|
||||||
postgresql_where=text("owner_type = 'user' AND deleted_at IS NULL"),
|
postgresql_where=text("owner_type = 'user' AND deleted_at IS NULL"),
|
||||||
),
|
),
|
||||||
Index(
|
Index(
|
||||||
"uq_file_connector_spaces_active_group_label",
|
"uq_file_connector_spaces_active_group_label",
|
||||||
"tenant_id", "owner_group_id", "label",
|
"tenant_id",
|
||||||
|
"owner_group_id",
|
||||||
|
"label",
|
||||||
unique=True,
|
unique=True,
|
||||||
sqlite_where=text("owner_type = 'group' AND deleted_at IS NULL"),
|
sqlite_where=text("owner_type = 'group' AND deleted_at IS NULL"),
|
||||||
postgresql_where=text("owner_type = 'group' AND deleted_at IS NULL"),
|
postgresql_where=text("owner_type = 'group' AND deleted_at IS NULL"),
|
||||||
@@ -203,41 +491,81 @@ class FileConnectorSpace(Base, TimestampMixin):
|
|||||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||||
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||||
owner_type: Mapped[str] = mapped_column(String(20), nullable=False, index=True)
|
owner_type: Mapped[str] = mapped_column(String(20), nullable=False, index=True)
|
||||||
owner_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
|
owner_user_id: Mapped[str | None] = mapped_column(
|
||||||
owner_group_id: Mapped[str | None] = mapped_column(ForeignKey("access_groups.id", ondelete="SET NULL"), nullable=True, index=True)
|
ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True
|
||||||
|
)
|
||||||
|
owner_group_id: Mapped[str | None] = mapped_column(
|
||||||
|
ForeignKey("access_groups.id", ondelete="SET NULL"), nullable=True, index=True
|
||||||
|
)
|
||||||
label: Mapped[str] = mapped_column(String(255), nullable=False)
|
label: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||||
connector_profile_id: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
|
connector_profile_id: Mapped[str] = mapped_column(
|
||||||
|
String(255), nullable=False, index=True
|
||||||
|
)
|
||||||
provider: Mapped[str] = mapped_column(String(50), nullable=False, index=True)
|
provider: Mapped[str] = mapped_column(String(50), nullable=False, index=True)
|
||||||
library_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
library_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||||
remote_path: Mapped[str] = mapped_column(String(1000), default="", nullable=False)
|
remote_path: Mapped[str] = mapped_column(String(1000), default="", nullable=False)
|
||||||
sync_mode: Mapped[str] = mapped_column(String(30), default="manual", nullable=False)
|
sync_mode: Mapped[str] = mapped_column(String(30), default="manual", nullable=False)
|
||||||
read_only: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False)
|
read_only: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False)
|
||||||
is_active: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False, index=True)
|
is_active: Mapped[bool] = mapped_column(
|
||||||
created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
|
Boolean, default=True, nullable=False, index=True
|
||||||
deleted_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
|
)
|
||||||
metadata_: Mapped[dict[str, Any] | None] = mapped_column("metadata", JSON, nullable=True)
|
created_by_user_id: Mapped[str | None] = mapped_column(
|
||||||
|
ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True
|
||||||
|
)
|
||||||
|
deleted_at: Mapped[datetime | None] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=True, index=True
|
||||||
|
)
|
||||||
|
metadata_: Mapped[dict[str, Any] | None] = mapped_column(
|
||||||
|
"metadata", JSON, nullable=True
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class CampaignAttachmentUse(Base, TimestampMixin):
|
class CampaignAttachmentUse(Base, TimestampMixin):
|
||||||
__tablename__ = "campaign_attachment_uses"
|
__tablename__ = "campaign_attachment_uses"
|
||||||
__table_args__ = (UniqueConstraint("campaign_job_id", "file_version_id", "filename_used", "use_stage", name="uq_campaign_attachment_uses_job_file_stage"),)
|
__table_args__ = (
|
||||||
|
UniqueConstraint(
|
||||||
|
"campaign_job_id",
|
||||||
|
"file_version_id",
|
||||||
|
"filename_used",
|
||||||
|
"use_stage",
|
||||||
|
name="uq_campaign_attachment_uses_job_file_stage",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||||
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||||
campaign_id: Mapped[str] = mapped_column(ForeignKey("campaigns.id", ondelete="CASCADE"), nullable=False, index=True)
|
campaign_id: Mapped[str] = mapped_column(
|
||||||
campaign_version_id: Mapped[str] = mapped_column(ForeignKey("campaign_versions.id", ondelete="CASCADE"), nullable=False, index=True)
|
ForeignKey("campaigns.id", ondelete="CASCADE"), nullable=False, index=True
|
||||||
campaign_job_id: Mapped[str | None] = mapped_column(ForeignKey("campaign_jobs.id", ondelete="SET NULL"), nullable=True, index=True)
|
)
|
||||||
|
campaign_version_id: Mapped[str] = mapped_column(
|
||||||
|
ForeignKey("campaign_versions.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
campaign_job_id: Mapped[str | None] = mapped_column(
|
||||||
|
ForeignKey("campaign_jobs.id", ondelete="SET NULL"), nullable=True, index=True
|
||||||
|
)
|
||||||
entry_index: Mapped[int | None] = mapped_column(Integer)
|
entry_index: Mapped[int | None] = mapped_column(Integer)
|
||||||
entry_id: Mapped[str | None] = mapped_column(String(255), index=True)
|
entry_id: Mapped[str | None] = mapped_column(String(255), index=True)
|
||||||
file_asset_id: Mapped[str] = mapped_column(ForeignKey("file_assets.id", ondelete="RESTRICT"), nullable=False, index=True)
|
file_asset_id: Mapped[str] = mapped_column(
|
||||||
file_version_id: Mapped[str] = mapped_column(ForeignKey("file_versions.id", ondelete="RESTRICT"), nullable=False, index=True)
|
ForeignKey("file_assets.id", ondelete="RESTRICT"), nullable=False, index=True
|
||||||
file_blob_id: Mapped[str] = mapped_column(ForeignKey("file_blobs.id", ondelete="RESTRICT"), nullable=False, index=True)
|
)
|
||||||
|
file_version_id: Mapped[str] = mapped_column(
|
||||||
|
ForeignKey("file_versions.id", ondelete="RESTRICT"), nullable=False, index=True
|
||||||
|
)
|
||||||
|
file_blob_id: Mapped[str] = mapped_column(
|
||||||
|
ForeignKey("file_blobs.id", ondelete="RESTRICT"), nullable=False, index=True
|
||||||
|
)
|
||||||
filename_used: Mapped[str] = mapped_column(String(500), nullable=False)
|
filename_used: Mapped[str] = mapped_column(String(500), nullable=False)
|
||||||
checksum_sha256: Mapped[str] = mapped_column(String(64), nullable=False)
|
checksum_sha256: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||||
size_bytes: Mapped[int] = mapped_column(Integer, nullable=False)
|
size_bytes: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||||
content_type: Mapped[str | None] = mapped_column(String(255))
|
content_type: Mapped[str | None] = mapped_column(String(255))
|
||||||
use_stage: Mapped[str] = mapped_column(String(20), default="built", nullable=False, index=True)
|
use_stage: Mapped[str] = mapped_column(
|
||||||
used_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
|
String(20), default="built", nullable=False, index=True
|
||||||
|
)
|
||||||
|
used_at: Mapped[datetime | None] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=True, index=True
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
@@ -249,6 +577,7 @@ __all__ = [
|
|||||||
"FileConnectorProfile",
|
"FileConnectorProfile",
|
||||||
"FileConnectorSpace",
|
"FileConnectorSpace",
|
||||||
"FileFolder",
|
"FileFolder",
|
||||||
|
"FileFormEvidenceGrant",
|
||||||
"FileShare",
|
"FileShare",
|
||||||
"FileVersion",
|
"FileVersion",
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ from govoplan_core.core.modules import (
|
|||||||
DocumentationLink,
|
DocumentationLink,
|
||||||
DocumentationTopic,
|
DocumentationTopic,
|
||||||
)
|
)
|
||||||
from govoplan_files.backend.storage.archives import ZIP_UPLOAD_MAX_FILES
|
from govoplan_files.backend.storage.archives import ARCHIVE_UPLOAD_MAX_ENTRIES
|
||||||
from govoplan_files.backend.storage.access import user_group_ids
|
from govoplan_files.backend.storage.access import user_group_ids
|
||||||
from govoplan_files.backend.storage.connector_visibility import (
|
from govoplan_files.backend.storage.connector_visibility import (
|
||||||
connector_profile_usable_for_import,
|
connector_profile_usable_for_import,
|
||||||
@@ -22,6 +22,9 @@ from govoplan_files.backend.storage.connector_visibility import (
|
|||||||
|
|
||||||
_DEFAULT_UPLOAD_MAX_BYTES = 50 * 1024 * 1024
|
_DEFAULT_UPLOAD_MAX_BYTES = 50 * 1024 * 1024
|
||||||
_DEFAULT_ZIP_MAX_BYTES = 250 * 1024 * 1024
|
_DEFAULT_ZIP_MAX_BYTES = 250 * 1024 * 1024
|
||||||
|
_DEFAULT_ARCHIVE_MAX_EXPANDED_BYTES = 2 * 1024 * 1024 * 1024
|
||||||
|
_DEFAULT_ARCHIVE_MAX_EXPANSION_RATIO = 100
|
||||||
|
_DEFAULT_ARCHIVE_PREVIEW_TTL_SECONDS = 30 * 60
|
||||||
_FILES_READ_SCOPE = "files:file:read"
|
_FILES_READ_SCOPE = "files:file:read"
|
||||||
_FILES_UPLOAD_SCOPE = "files:file:upload"
|
_FILES_UPLOAD_SCOPE = "files:file:upload"
|
||||||
|
|
||||||
@@ -42,11 +45,47 @@ def documentation_topics(
|
|||||||
"file_upload_zip_max_bytes",
|
"file_upload_zip_max_bytes",
|
||||||
default=_DEFAULT_ZIP_MAX_BYTES,
|
default=_DEFAULT_ZIP_MAX_BYTES,
|
||||||
)
|
)
|
||||||
|
archive_expanded_limit = _configured_positive_int(
|
||||||
|
context.settings,
|
||||||
|
"file_archive_max_expanded_bytes",
|
||||||
|
default=_DEFAULT_ARCHIVE_MAX_EXPANDED_BYTES,
|
||||||
|
)
|
||||||
|
archive_entry_limit = _configured_positive_int(
|
||||||
|
context.settings,
|
||||||
|
"file_archive_max_entries",
|
||||||
|
default=ARCHIVE_UPLOAD_MAX_ENTRIES,
|
||||||
|
)
|
||||||
|
archive_ratio_limit = _configured_positive_int(
|
||||||
|
context.settings,
|
||||||
|
"file_archive_max_expansion_ratio",
|
||||||
|
default=_DEFAULT_ARCHIVE_MAX_EXPANSION_RATIO,
|
||||||
|
)
|
||||||
|
archive_preview_ttl = _configured_positive_int(
|
||||||
|
context.settings,
|
||||||
|
"file_archive_preview_ttl_seconds",
|
||||||
|
default=_DEFAULT_ARCHIVE_PREVIEW_TTL_SECONDS,
|
||||||
|
)
|
||||||
topics: list[DocumentationTopic] = []
|
topics: list[DocumentationTopic] = []
|
||||||
if upload_limit is not None:
|
if upload_limit is not None:
|
||||||
topics.append(_upload_topic(upload_limit))
|
topics.append(_upload_topic(upload_limit))
|
||||||
if upload_limit is not None and zip_limit is not None:
|
if (
|
||||||
topics.append(_zip_topic(upload_limit, zip_limit))
|
upload_limit is not None
|
||||||
|
and zip_limit is not None
|
||||||
|
and archive_expanded_limit is not None
|
||||||
|
and archive_entry_limit is not None
|
||||||
|
and archive_ratio_limit is not None
|
||||||
|
and archive_preview_ttl is not None
|
||||||
|
):
|
||||||
|
topics.append(
|
||||||
|
_archive_topic(
|
||||||
|
upload_limit,
|
||||||
|
zip_limit,
|
||||||
|
archive_expanded_limit,
|
||||||
|
archive_entry_limit,
|
||||||
|
archive_ratio_limit,
|
||||||
|
archive_preview_ttl,
|
||||||
|
)
|
||||||
|
)
|
||||||
topics.append(_connector_import_topic(context))
|
topics.append(_connector_import_topic(context))
|
||||||
return tuple(topics)
|
return tuple(topics)
|
||||||
|
|
||||||
@@ -118,19 +157,29 @@ def _upload_topic(max_bytes: int) -> DocumentationTopic:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def _zip_topic(max_file_bytes: int, max_zip_bytes: int) -> DocumentationTopic:
|
def _archive_topic(
|
||||||
|
max_file_bytes: int,
|
||||||
|
max_archive_request_bytes: int,
|
||||||
|
max_expanded_bytes: int,
|
||||||
|
max_entries: int,
|
||||||
|
max_expansion_ratio: int,
|
||||||
|
preview_ttl_seconds: int,
|
||||||
|
) -> DocumentationTopic:
|
||||||
member_limit = _format_byte_limit(max_file_bytes)
|
member_limit = _format_byte_limit(max_file_bytes)
|
||||||
archive_limit = _format_byte_limit(max_zip_bytes)
|
request_limit = _format_byte_limit(max_archive_request_bytes)
|
||||||
|
expanded_limit = _format_byte_limit(max_expanded_bytes)
|
||||||
|
preview_minutes = max(1, preview_ttl_seconds // 60)
|
||||||
return DocumentationTopic(
|
return DocumentationTopic(
|
||||||
id="files.workflow.upload-and-unpack-zip",
|
id="files.workflow.upload-and-unpack-zip",
|
||||||
title="Upload and unpack a ZIP archive",
|
title="Preview and unpack an archive",
|
||||||
summary=(
|
summary=(
|
||||||
f"Safely unpack up to {ZIP_UPLOAD_MAX_FILES:,} files from a ZIP whose request and extracted total are each limited to {archive_limit}."
|
f"Review and selectively unpack up to {max_entries:,} entries from ZIP or TAR archives before any managed file is created."
|
||||||
),
|
),
|
||||||
body=(
|
body=(
|
||||||
f"The current deployment limits the ZIP request and actual extracted total to {archive_limit}, each member to {member_limit}, "
|
f"The deployment accepts ZIP, TAR, TAR.GZ, TAR.BZ2, and TAR.XZ requests up to {request_limit}, limits actual expanded data to {expanded_limit}, "
|
||||||
f"and the archive to {ZIP_UPLOAD_MAX_FILES:,} non-directory members. Encrypted archives and unsafe member paths are rejected. "
|
f"each member to {member_limit}, the archive to {max_entries:,} entries, and expansion to {max_expansion_ratio}:1. "
|
||||||
"Actual extracted bytes are counted instead of trusting ZIP headers."
|
f"The server-issued preview expires after {preview_minutes} minutes. Password-protected ZIP archives are supported; passwords remain request-only. "
|
||||||
|
"Unsafe paths and special filesystem entries are rejected. Actual extracted bytes are counted instead of trusting archive headers."
|
||||||
),
|
),
|
||||||
layer="configured",
|
layer="configured",
|
||||||
documentation_types=("user",),
|
documentation_types=("user",),
|
||||||
@@ -161,35 +210,42 @@ def _zip_topic(max_file_bytes: int, max_zip_bytes: int) -> DocumentationTopic:
|
|||||||
"help_contexts": ["files.list"],
|
"help_contexts": ["files.list"],
|
||||||
"prerequisites": [
|
"prerequisites": [
|
||||||
"You may view and upload managed files.",
|
"You may view and upload managed files.",
|
||||||
"The archive is not encrypted and fits the current configured limits.",
|
"The archive fits the configured request, expansion, size, and entry limits.",
|
||||||
"The destination personal or group space grants this account write access.",
|
"The destination personal or group space grants this account write access.",
|
||||||
],
|
],
|
||||||
"steps": [
|
"steps": [
|
||||||
"Open Files and choose the managed destination space and folder.",
|
"Open Files and choose the managed destination space and folder.",
|
||||||
"Enable Unpack ZIP uploads, then choose or drag the ZIP archive.",
|
"Enable Preview and unpack archive, then choose or drag one supported archive.",
|
||||||
|
"Review the discovered entries, supply a ZIP password when required, and select the files or folders to import.",
|
||||||
"Resolve every destination conflict explicitly.",
|
"Resolve every destination conflict explicitly.",
|
||||||
"Wait for extraction and finalization to finish before leaving the page.",
|
"Confirm the selection, then wait for extraction and finalization to finish before leaving the page.",
|
||||||
],
|
],
|
||||||
"outcome": "Accepted archive members are stored as separate governed managed assets below the selected folder.",
|
"outcome": "Accepted archive members are stored as separate governed managed assets below the selected folder.",
|
||||||
"verification": "Confirm the expected member paths and inspect representative file sizes, checksums, and versions.",
|
"verification": "Confirm the expected member paths and inspect representative file sizes, checksums, and versions.",
|
||||||
"constraints": [
|
"constraints": [
|
||||||
{
|
{
|
||||||
"id": "zip-request-and-total",
|
"id": "archive-request-and-total",
|
||||||
"label": "Maximum ZIP request and extracted total",
|
"label": "Maximum archive request and expanded total",
|
||||||
"description": f"Both the compressed request and the actual extracted total are limited to {archive_limit}.",
|
"description": f"The compressed request is limited to {request_limit}; actual expanded data is limited to {expanded_limit}.",
|
||||||
"values": [archive_limit],
|
"values": [request_limit, expanded_limit],
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"id": "zip-member-size",
|
"id": "archive-member-size",
|
||||||
"label": "Maximum extracted member size",
|
"label": "Maximum extracted member size",
|
||||||
"description": f"Each extracted file is limited to {member_limit}.",
|
"description": f"Each extracted file is limited to {member_limit}.",
|
||||||
"values": [member_limit],
|
"values": [member_limit],
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"id": "zip-member-count",
|
"id": "archive-entry-count",
|
||||||
"label": "Maximum file count",
|
"label": "Maximum entry count",
|
||||||
"description": f"A ZIP may contain at most {ZIP_UPLOAD_MAX_FILES:,} non-directory members.",
|
"description": f"An archive may contain at most {max_entries:,} declared entries.",
|
||||||
"values": [f"{ZIP_UPLOAD_MAX_FILES:,} files"],
|
"values": [f"{max_entries:,} entries"],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "archive-expansion-ratio",
|
||||||
|
"label": "Maximum expansion ratio",
|
||||||
|
"description": f"Declared and actual output may not exceed {max_expansion_ratio} times the compressed request size.",
|
||||||
|
"values": [f"{max_expansion_ratio}:1"],
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
"related_topic_ids": [
|
"related_topic_ids": [
|
||||||
|
|||||||
@@ -0,0 +1,879 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from collections.abc import Sequence
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
|
from sqlalchemy import inspect, or_
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from govoplan_core.core.dsar import (
|
||||||
|
DsarErasureActionRef,
|
||||||
|
DsarExecutionResultRef,
|
||||||
|
DsarRecordRef,
|
||||||
|
DsarSubjectRef,
|
||||||
|
dsar_capability_name,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.db.models import (
|
||||||
|
CampaignAttachmentUse,
|
||||||
|
FileAsset,
|
||||||
|
FileConnectorCredential,
|
||||||
|
FileConnectorPolicy,
|
||||||
|
FileConnectorProfile,
|
||||||
|
FileConnectorSpace,
|
||||||
|
FileFolder,
|
||||||
|
FileFormEvidenceGrant,
|
||||||
|
FileIntegrityFinding,
|
||||||
|
FileIntegrityScan,
|
||||||
|
FileShare,
|
||||||
|
FileVersion,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
FILES_DSAR_CAPABILITY = dsar_capability_name("files")
|
||||||
|
_MAX_RECORDS = 5_000
|
||||||
|
|
||||||
|
|
||||||
|
class FilesDsarProvider:
|
||||||
|
provider_id = "files"
|
||||||
|
module_id = "files"
|
||||||
|
|
||||||
|
def search_subject(
|
||||||
|
self,
|
||||||
|
session: object,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
subject: DsarSubjectRef,
|
||||||
|
) -> Sequence[DsarRecordRef]:
|
||||||
|
db = _session(session)
|
||||||
|
subject_user_id = _subject_user_id(subject)
|
||||||
|
if subject_user_id is None:
|
||||||
|
return ()
|
||||||
|
|
||||||
|
records: list[DsarRecordRef] = []
|
||||||
|
form_evidence_available = _has_table(db, FileFormEvidenceGrant)
|
||||||
|
campaign_evidence_available = _has_table(db, CampaignAttachmentUse)
|
||||||
|
retention_reasons: dict[str, str | None] = {}
|
||||||
|
|
||||||
|
def retention_reason(asset: FileAsset) -> str | None:
|
||||||
|
if asset.id not in retention_reasons:
|
||||||
|
retention_reasons[asset.id] = _asset_retention_reason(
|
||||||
|
db,
|
||||||
|
asset,
|
||||||
|
form_evidence_available=form_evidence_available,
|
||||||
|
campaign_evidence_available=campaign_evidence_available,
|
||||||
|
)
|
||||||
|
return retention_reasons[asset.id]
|
||||||
|
|
||||||
|
def append(record: DsarRecordRef) -> None:
|
||||||
|
if len(records) >= _MAX_RECORDS:
|
||||||
|
raise ValueError(
|
||||||
|
"Files DSAR match limit exceeded; narrow the subject selectors."
|
||||||
|
)
|
||||||
|
records.append(record)
|
||||||
|
|
||||||
|
assets = _bounded_rows(
|
||||||
|
db.query(FileAsset)
|
||||||
|
.filter(
|
||||||
|
FileAsset.tenant_id == tenant_id,
|
||||||
|
or_(
|
||||||
|
FileAsset.owner_user_id == subject_user_id,
|
||||||
|
FileAsset.created_by_user_id == subject_user_id,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.order_by(FileAsset.id)
|
||||||
|
)
|
||||||
|
for asset in assets:
|
||||||
|
match_fields = _matching_fields(
|
||||||
|
asset,
|
||||||
|
subject_user_id,
|
||||||
|
("owner_user_id", "created_by_user_id"),
|
||||||
|
)
|
||||||
|
asset_retention_reason = retention_reason(asset)
|
||||||
|
append(
|
||||||
|
_record(
|
||||||
|
"file_asset",
|
||||||
|
asset.id,
|
||||||
|
"managed_file",
|
||||||
|
asset.filename,
|
||||||
|
{
|
||||||
|
"match_fields": match_fields,
|
||||||
|
"owner_type": asset.owner_type,
|
||||||
|
"display_path": asset.display_path,
|
||||||
|
"filename": asset.filename,
|
||||||
|
"description": asset.description,
|
||||||
|
"deleted_at": _iso(asset.deleted_at),
|
||||||
|
"retained_until": _iso(asset.retained_until),
|
||||||
|
"legal_hold": asset.legal_hold,
|
||||||
|
"lifecycle_reason": asset.lifecycle_reason,
|
||||||
|
"lifecycle_revision": asset.lifecycle_revision,
|
||||||
|
},
|
||||||
|
observed_at=asset.updated_at,
|
||||||
|
immutable=asset_retention_reason is not None,
|
||||||
|
retention_reason=asset_retention_reason,
|
||||||
|
source_path=f"/files?file={asset.id}",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
version_rows = _bounded_rows(
|
||||||
|
db.query(FileVersion, FileAsset)
|
||||||
|
.join(FileAsset, FileAsset.id == FileVersion.file_asset_id)
|
||||||
|
.filter(
|
||||||
|
FileVersion.tenant_id == tenant_id,
|
||||||
|
FileAsset.tenant_id == tenant_id,
|
||||||
|
or_(
|
||||||
|
FileAsset.owner_user_id == subject_user_id,
|
||||||
|
FileVersion.created_by_user_id == subject_user_id,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.order_by(FileVersion.id)
|
||||||
|
)
|
||||||
|
for version, asset in version_rows:
|
||||||
|
asset_retention_reason = retention_reason(asset)
|
||||||
|
append(
|
||||||
|
_record(
|
||||||
|
"file_version",
|
||||||
|
version.id,
|
||||||
|
"managed_file_version",
|
||||||
|
version.filename_at_upload,
|
||||||
|
{
|
||||||
|
"match_fields": (
|
||||||
|
["asset.owner_user_id"]
|
||||||
|
if asset.owner_user_id == subject_user_id
|
||||||
|
else []
|
||||||
|
)
|
||||||
|
+ (
|
||||||
|
["created_by_user_id"]
|
||||||
|
if version.created_by_user_id == subject_user_id
|
||||||
|
else []
|
||||||
|
),
|
||||||
|
"file_asset_id": version.file_asset_id,
|
||||||
|
"version_number": version.version_number,
|
||||||
|
"filename_at_upload": version.filename_at_upload,
|
||||||
|
"display_path_at_upload": version.display_path_at_upload,
|
||||||
|
"content_type": version.content_type,
|
||||||
|
"size_bytes": version.size_bytes,
|
||||||
|
"checksum_sha256": version.checksum_sha256,
|
||||||
|
"created_at": _iso(version.created_at),
|
||||||
|
},
|
||||||
|
observed_at=version.updated_at,
|
||||||
|
immutable=asset_retention_reason is not None,
|
||||||
|
retention_reason=asset_retention_reason,
|
||||||
|
source_path=(
|
||||||
|
f"/api/v1/files/{version.file_asset_id}/versions/"
|
||||||
|
f"{version.id}/download"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
for folder in _bounded_rows(
|
||||||
|
db.query(FileFolder)
|
||||||
|
.filter(
|
||||||
|
FileFolder.tenant_id == tenant_id,
|
||||||
|
or_(
|
||||||
|
FileFolder.owner_user_id == subject_user_id,
|
||||||
|
FileFolder.created_by_user_id == subject_user_id,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.order_by(FileFolder.id)
|
||||||
|
):
|
||||||
|
append(
|
||||||
|
_record(
|
||||||
|
"file_folder",
|
||||||
|
folder.id,
|
||||||
|
"managed_folder",
|
||||||
|
folder.path,
|
||||||
|
{
|
||||||
|
"match_fields": _matching_fields(
|
||||||
|
folder,
|
||||||
|
subject_user_id,
|
||||||
|
("owner_user_id", "created_by_user_id"),
|
||||||
|
),
|
||||||
|
"owner_type": folder.owner_type,
|
||||||
|
"path": folder.path,
|
||||||
|
"deleted_at": _iso(folder.deleted_at),
|
||||||
|
},
|
||||||
|
observed_at=folder.updated_at,
|
||||||
|
source_path="/files",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
for share in _bounded_rows(
|
||||||
|
db.query(FileShare)
|
||||||
|
.filter(
|
||||||
|
FileShare.tenant_id == tenant_id,
|
||||||
|
or_(
|
||||||
|
(FileShare.target_type == "user")
|
||||||
|
& (FileShare.target_id == subject_user_id),
|
||||||
|
FileShare.created_by_user_id == subject_user_id,
|
||||||
|
FileShare.revoked_by_user_id == subject_user_id,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.order_by(FileShare.id)
|
||||||
|
):
|
||||||
|
target_matches = (
|
||||||
|
share.target_type == "user" and share.target_id == subject_user_id
|
||||||
|
)
|
||||||
|
match_fields = _matching_fields(
|
||||||
|
share,
|
||||||
|
subject_user_id,
|
||||||
|
("created_by_user_id", "revoked_by_user_id"),
|
||||||
|
)
|
||||||
|
if target_matches:
|
||||||
|
match_fields.insert(0, "target_id")
|
||||||
|
append(
|
||||||
|
_record(
|
||||||
|
"file_share",
|
||||||
|
share.id,
|
||||||
|
"file_access_evidence",
|
||||||
|
f"Share for file {share.file_asset_id}",
|
||||||
|
{
|
||||||
|
"match_fields": match_fields,
|
||||||
|
"file_asset_id": share.file_asset_id,
|
||||||
|
"target_type": share.target_type,
|
||||||
|
"target_is_subject": target_matches,
|
||||||
|
"permission": share.permission,
|
||||||
|
"expires_at": _iso(share.expires_at),
|
||||||
|
"revoked_at": _iso(share.revoked_at),
|
||||||
|
},
|
||||||
|
observed_at=share.updated_at,
|
||||||
|
immutable=True,
|
||||||
|
retention_reason=(
|
||||||
|
"File-sharing history is institutional access evidence."
|
||||||
|
),
|
||||||
|
source_path=f"/files?file={share.file_asset_id}",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
if form_evidence_available:
|
||||||
|
evidence_rows = _bounded_rows(
|
||||||
|
db.query(FileFormEvidenceGrant)
|
||||||
|
.outerjoin(
|
||||||
|
FileAsset,
|
||||||
|
FileAsset.id == FileFormEvidenceGrant.file_asset_id,
|
||||||
|
)
|
||||||
|
.filter(
|
||||||
|
FileFormEvidenceGrant.tenant_id == tenant_id,
|
||||||
|
or_(
|
||||||
|
FileFormEvidenceGrant.custodian_user_id == subject_user_id,
|
||||||
|
FileAsset.owner_user_id == subject_user_id,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.order_by(FileFormEvidenceGrant.id)
|
||||||
|
)
|
||||||
|
for grant in evidence_rows:
|
||||||
|
append(
|
||||||
|
_record(
|
||||||
|
"file_form_evidence",
|
||||||
|
grant.id,
|
||||||
|
"form_evidence",
|
||||||
|
f"Form evidence {grant.form_definition_id}",
|
||||||
|
{
|
||||||
|
"match_fields": (
|
||||||
|
["custodian_user_id"]
|
||||||
|
if grant.custodian_user_id == subject_user_id
|
||||||
|
else ["asset.owner_user_id"]
|
||||||
|
),
|
||||||
|
"form_instance_id": grant.form_instance_id,
|
||||||
|
"form_definition_id": grant.form_definition_id,
|
||||||
|
"form_definition_revision": grant.form_definition_revision,
|
||||||
|
"evidence_kind": grant.evidence_kind,
|
||||||
|
"purpose": grant.purpose,
|
||||||
|
"status": grant.status,
|
||||||
|
"expires_at": _iso(grant.expires_at),
|
||||||
|
"file_asset_id": grant.file_asset_id,
|
||||||
|
"file_version_id": grant.file_version_id,
|
||||||
|
},
|
||||||
|
observed_at=grant.updated_at,
|
||||||
|
immutable=True,
|
||||||
|
retention_reason=(
|
||||||
|
"Submitted Form attachment evidence follows the owning "
|
||||||
|
"process retention and cannot be erased through Files alone."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
if campaign_evidence_available:
|
||||||
|
campaign_rows = _bounded_rows(
|
||||||
|
db.query(CampaignAttachmentUse)
|
||||||
|
.join(FileAsset, FileAsset.id == CampaignAttachmentUse.file_asset_id)
|
||||||
|
.filter(
|
||||||
|
CampaignAttachmentUse.tenant_id == tenant_id,
|
||||||
|
FileAsset.owner_user_id == subject_user_id,
|
||||||
|
)
|
||||||
|
.order_by(CampaignAttachmentUse.id)
|
||||||
|
)
|
||||||
|
for use in campaign_rows:
|
||||||
|
append(
|
||||||
|
_record(
|
||||||
|
"campaign_attachment_use",
|
||||||
|
use.id,
|
||||||
|
"delivery_evidence",
|
||||||
|
use.filename_used,
|
||||||
|
{
|
||||||
|
"match_fields": ["asset.owner_user_id"],
|
||||||
|
"campaign_id": use.campaign_id,
|
||||||
|
"campaign_version_id": use.campaign_version_id,
|
||||||
|
"campaign_job_id": use.campaign_job_id,
|
||||||
|
"file_asset_id": use.file_asset_id,
|
||||||
|
"file_version_id": use.file_version_id,
|
||||||
|
"filename_used": use.filename_used,
|
||||||
|
"checksum_sha256": use.checksum_sha256,
|
||||||
|
"size_bytes": use.size_bytes,
|
||||||
|
"use_stage": use.use_stage,
|
||||||
|
"used_at": _iso(use.used_at),
|
||||||
|
},
|
||||||
|
observed_at=use.updated_at,
|
||||||
|
immutable=True,
|
||||||
|
retention_reason=(
|
||||||
|
"Campaign attachment use is immutable delivery evidence."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
self._append_configuration_references(
|
||||||
|
db,
|
||||||
|
append=append,
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
subject_user_id=subject_user_id,
|
||||||
|
)
|
||||||
|
self._append_integrity_references(
|
||||||
|
db,
|
||||||
|
append=append,
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
subject_user_id=subject_user_id,
|
||||||
|
)
|
||||||
|
return tuple(records)
|
||||||
|
|
||||||
|
def plan_erasure(
|
||||||
|
self,
|
||||||
|
session: object,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
subject: DsarSubjectRef,
|
||||||
|
records: Sequence[DsarRecordRef],
|
||||||
|
) -> Sequence[DsarErasureActionRef]:
|
||||||
|
del session
|
||||||
|
subject_user_id = _subject_user_id(subject)
|
||||||
|
if subject_user_id is None:
|
||||||
|
return ()
|
||||||
|
actions: list[DsarErasureActionRef] = []
|
||||||
|
for record in records:
|
||||||
|
if record.provider_id != self.provider_id or record.module_id != self.module_id:
|
||||||
|
raise ValueError("Files DSAR received a foreign provider record.")
|
||||||
|
match_fields = {
|
||||||
|
str(value) for value in record.data.get("match_fields", ())
|
||||||
|
}
|
||||||
|
if record.immutable_evidence:
|
||||||
|
actions.append(
|
||||||
|
_action(
|
||||||
|
f"files:retain:{record.resource_type}:{record.resource_id}",
|
||||||
|
"retain",
|
||||||
|
record,
|
||||||
|
f"Retain {record.title}",
|
||||||
|
record.retention_reason
|
||||||
|
or "Institutional evidence must be retained.",
|
||||||
|
executable=False,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
elif record.resource_type in {
|
||||||
|
"file_asset",
|
||||||
|
"file_version",
|
||||||
|
"file_folder",
|
||||||
|
}:
|
||||||
|
actions.append(
|
||||||
|
_action(
|
||||||
|
f"files:review:{record.resource_type}:{record.resource_id}",
|
||||||
|
"manual_review",
|
||||||
|
record,
|
||||||
|
f"Review {record.title}",
|
||||||
|
(
|
||||||
|
"Managed file content, names, paths, ownership, and shared "
|
||||||
|
"references require a case decision. Approved byte erasure "
|
||||||
|
"must use the separate Files purge workflow."
|
||||||
|
),
|
||||||
|
executable=False,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
record.resource_type == "file_share"
|
||||||
|
and "target_id" in match_fields
|
||||||
|
and record.data.get("revoked_at") is None
|
||||||
|
):
|
||||||
|
actions.append(
|
||||||
|
_action(
|
||||||
|
f"files:revoke:file_share:{record.resource_id}",
|
||||||
|
"revoke",
|
||||||
|
record,
|
||||||
|
"Revoke active file share",
|
||||||
|
"The active user-targeted share can be revoked without deleting file evidence.",
|
||||||
|
executable=True,
|
||||||
|
metadata={"subject_user_id": subject_user_id},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
for field_name in sorted(
|
||||||
|
match_fields.intersection(_detachable_fields(record.resource_type))
|
||||||
|
):
|
||||||
|
actions.append(
|
||||||
|
_action(
|
||||||
|
(
|
||||||
|
f"files:detach:{record.resource_type}:"
|
||||||
|
f"{field_name}:{record.resource_id}"
|
||||||
|
),
|
||||||
|
"detach",
|
||||||
|
record,
|
||||||
|
f"Detach {field_name.replace('_', ' ')}",
|
||||||
|
(
|
||||||
|
"Remove the mutable subject reference while preserving the "
|
||||||
|
"governed resource and DSAR evidence."
|
||||||
|
),
|
||||||
|
executable=True,
|
||||||
|
metadata={
|
||||||
|
"field": field_name,
|
||||||
|
"subject_user_id": subject_user_id,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
action_ids = [action.action_id for action in actions]
|
||||||
|
if len(action_ids) != len(set(action_ids)):
|
||||||
|
raise ValueError("Files DSAR produced duplicate action ids.")
|
||||||
|
return tuple(actions)
|
||||||
|
|
||||||
|
def execute_erasure(
|
||||||
|
self,
|
||||||
|
session: object,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
subject: DsarSubjectRef,
|
||||||
|
actions: Sequence[DsarErasureActionRef],
|
||||||
|
request_id: str,
|
||||||
|
) -> Sequence[DsarExecutionResultRef]:
|
||||||
|
db = _session(session)
|
||||||
|
subject_user_id = _subject_user_id(subject)
|
||||||
|
if subject_user_id is None:
|
||||||
|
return tuple(
|
||||||
|
_blocked(action, "Files requires a direct membership subject reference.")
|
||||||
|
for action in actions
|
||||||
|
)
|
||||||
|
results: list[DsarExecutionResultRef] = []
|
||||||
|
for action in actions:
|
||||||
|
if (
|
||||||
|
action.provider_id != self.provider_id
|
||||||
|
or action.module_id != self.module_id
|
||||||
|
or action.metadata.get("subject_user_id") != subject_user_id
|
||||||
|
):
|
||||||
|
results.append(_blocked(action, "The Files DSAR action is stale or invalid."))
|
||||||
|
continue
|
||||||
|
if action.action_id.startswith("files:revoke:file_share:"):
|
||||||
|
results.append(
|
||||||
|
_revoke_share(
|
||||||
|
db,
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
subject_user_id=subject_user_id,
|
||||||
|
action=action,
|
||||||
|
request_id=request_id,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
elif action.action_id.startswith("files:detach:"):
|
||||||
|
results.append(
|
||||||
|
_detach_reference(
|
||||||
|
db,
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
subject_user_id=subject_user_id,
|
||||||
|
action=action,
|
||||||
|
request_id=request_id,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
results.append(_blocked(action, "Files does not execute this action kind."))
|
||||||
|
db.flush()
|
||||||
|
return tuple(results)
|
||||||
|
|
||||||
|
def _append_configuration_references(
|
||||||
|
self,
|
||||||
|
db: Session,
|
||||||
|
*,
|
||||||
|
append: object,
|
||||||
|
tenant_id: str,
|
||||||
|
subject_user_id: str,
|
||||||
|
) -> None:
|
||||||
|
configurations = (
|
||||||
|
(FileConnectorProfile, "connector_profile", ("created_by_user_id", "updated_by_user_id")),
|
||||||
|
(FileConnectorCredential, "connector_credential", ("created_by_user_id", "updated_by_user_id")),
|
||||||
|
(FileConnectorPolicy, "connector_policy", ("created_by_user_id", "updated_by_user_id")),
|
||||||
|
(FileConnectorSpace, "connector_space", ("owner_user_id", "created_by_user_id")),
|
||||||
|
)
|
||||||
|
for model, resource_type, fields in configurations:
|
||||||
|
if not _has_table(db, model):
|
||||||
|
continue
|
||||||
|
conditions = [getattr(model, field) == subject_user_id for field in fields]
|
||||||
|
query = db.query(model).filter(or_(*conditions))
|
||||||
|
if hasattr(model, "tenant_id"):
|
||||||
|
query = query.filter(model.tenant_id == tenant_id)
|
||||||
|
for row in _bounded_rows(query.order_by(model.id)):
|
||||||
|
match_fields = _matching_fields(row, subject_user_id, fields)
|
||||||
|
data: dict[str, object] = {
|
||||||
|
"match_fields": match_fields,
|
||||||
|
"label": getattr(row, "label", None),
|
||||||
|
"provider": getattr(row, "provider", None),
|
||||||
|
}
|
||||||
|
if isinstance(row, FileConnectorCredential):
|
||||||
|
data["credential_mode"] = row.credential_mode
|
||||||
|
elif isinstance(row, FileConnectorSpace):
|
||||||
|
data.update(
|
||||||
|
{
|
||||||
|
"remote_path": row.remote_path,
|
||||||
|
"sync_mode": row.sync_mode,
|
||||||
|
"read_only": row.read_only,
|
||||||
|
"deleted_at": _iso(row.deleted_at),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
append( # type: ignore[operator]
|
||||||
|
_record(
|
||||||
|
resource_type,
|
||||||
|
row.id,
|
||||||
|
"connector_configuration_evidence",
|
||||||
|
getattr(row, "label", None) or resource_type.replace("_", " "),
|
||||||
|
data,
|
||||||
|
observed_at=row.updated_at,
|
||||||
|
immutable=True,
|
||||||
|
retention_reason=(
|
||||||
|
"Connector configuration history is institutional evidence; "
|
||||||
|
"credential secrets are excluded from the DSAR export."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
def _append_integrity_references(
|
||||||
|
self,
|
||||||
|
db: Session,
|
||||||
|
*,
|
||||||
|
append: object,
|
||||||
|
tenant_id: str,
|
||||||
|
subject_user_id: str,
|
||||||
|
) -> None:
|
||||||
|
if _has_table(db, FileIntegrityScan):
|
||||||
|
for scan in _bounded_rows(
|
||||||
|
db.query(FileIntegrityScan)
|
||||||
|
.filter(
|
||||||
|
FileIntegrityScan.tenant_id == tenant_id,
|
||||||
|
FileIntegrityScan.created_by_user_id == subject_user_id,
|
||||||
|
)
|
||||||
|
.order_by(FileIntegrityScan.id)
|
||||||
|
):
|
||||||
|
append( # type: ignore[operator]
|
||||||
|
_record(
|
||||||
|
"file_integrity_scan",
|
||||||
|
scan.id,
|
||||||
|
"storage_integrity_evidence",
|
||||||
|
f"Integrity scan {scan.id}",
|
||||||
|
{
|
||||||
|
"match_fields": ["created_by_user_id"],
|
||||||
|
"storage_backend": scan.storage_backend,
|
||||||
|
"status": scan.status,
|
||||||
|
"started_at": _iso(scan.started_at),
|
||||||
|
"completed_at": _iso(scan.completed_at),
|
||||||
|
},
|
||||||
|
observed_at=scan.updated_at,
|
||||||
|
immutable=True,
|
||||||
|
retention_reason="Storage integrity scans are operator evidence.",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if _has_table(db, FileIntegrityFinding):
|
||||||
|
for finding in _bounded_rows(
|
||||||
|
db.query(FileIntegrityFinding)
|
||||||
|
.filter(
|
||||||
|
FileIntegrityFinding.tenant_id == tenant_id,
|
||||||
|
FileIntegrityFinding.resolved_by_user_id == subject_user_id,
|
||||||
|
)
|
||||||
|
.order_by(FileIntegrityFinding.id)
|
||||||
|
):
|
||||||
|
append( # type: ignore[operator]
|
||||||
|
_record(
|
||||||
|
"file_integrity_finding",
|
||||||
|
finding.id,
|
||||||
|
"storage_integrity_evidence",
|
||||||
|
f"Integrity finding {finding.kind}",
|
||||||
|
{
|
||||||
|
"match_fields": ["resolved_by_user_id"],
|
||||||
|
"kind": finding.kind,
|
||||||
|
"state": finding.state,
|
||||||
|
"resolved_at": _iso(finding.resolved_at),
|
||||||
|
},
|
||||||
|
observed_at=finding.updated_at,
|
||||||
|
immutable=True,
|
||||||
|
retention_reason="Integrity resolution is operator evidence.",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _asset_retention_reason(
|
||||||
|
session: Session,
|
||||||
|
asset: FileAsset,
|
||||||
|
*,
|
||||||
|
form_evidence_available: bool,
|
||||||
|
campaign_evidence_available: bool,
|
||||||
|
) -> str | None:
|
||||||
|
reasons: list[str] = []
|
||||||
|
if asset.legal_hold:
|
||||||
|
reasons.append("The file is under legal hold.")
|
||||||
|
retained_until = _aware(asset.retained_until)
|
||||||
|
if retained_until is not None and retained_until > datetime.now(timezone.utc):
|
||||||
|
reasons.append(f"The file is retained until {retained_until.isoformat()}.")
|
||||||
|
if form_evidence_available:
|
||||||
|
has_form_evidence = (
|
||||||
|
session.query(FileFormEvidenceGrant.id)
|
||||||
|
.filter(
|
||||||
|
FileFormEvidenceGrant.tenant_id == asset.tenant_id,
|
||||||
|
FileFormEvidenceGrant.file_asset_id == asset.id,
|
||||||
|
)
|
||||||
|
.first()
|
||||||
|
is not None
|
||||||
|
)
|
||||||
|
if has_form_evidence:
|
||||||
|
reasons.append("The file is referenced by submitted Form evidence.")
|
||||||
|
if campaign_evidence_available:
|
||||||
|
has_campaign_evidence = (
|
||||||
|
session.query(CampaignAttachmentUse.id)
|
||||||
|
.filter(
|
||||||
|
CampaignAttachmentUse.tenant_id == asset.tenant_id,
|
||||||
|
CampaignAttachmentUse.file_asset_id == asset.id,
|
||||||
|
)
|
||||||
|
.first()
|
||||||
|
is not None
|
||||||
|
)
|
||||||
|
if has_campaign_evidence:
|
||||||
|
reasons.append("The file is referenced by Campaign delivery evidence.")
|
||||||
|
return " ".join(reasons) or None
|
||||||
|
|
||||||
|
|
||||||
|
_DETACHABLE_MODELS: dict[str, tuple[type[object], frozenset[str]]] = {
|
||||||
|
"file_asset": (FileAsset, frozenset({"created_by_user_id"})),
|
||||||
|
"file_version": (FileVersion, frozenset({"created_by_user_id"})),
|
||||||
|
"file_folder": (FileFolder, frozenset({"created_by_user_id"})),
|
||||||
|
"file_share": (
|
||||||
|
FileShare,
|
||||||
|
frozenset({"created_by_user_id", "revoked_by_user_id"}),
|
||||||
|
),
|
||||||
|
"connector_profile": (
|
||||||
|
FileConnectorProfile,
|
||||||
|
frozenset({"created_by_user_id", "updated_by_user_id"}),
|
||||||
|
),
|
||||||
|
"connector_credential": (
|
||||||
|
FileConnectorCredential,
|
||||||
|
frozenset({"created_by_user_id", "updated_by_user_id"}),
|
||||||
|
),
|
||||||
|
"connector_policy": (
|
||||||
|
FileConnectorPolicy,
|
||||||
|
frozenset({"created_by_user_id", "updated_by_user_id"}),
|
||||||
|
),
|
||||||
|
"connector_space": (FileConnectorSpace, frozenset({"created_by_user_id"})),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _detachable_fields(resource_type: str) -> frozenset[str]:
|
||||||
|
entry = _DETACHABLE_MODELS.get(resource_type)
|
||||||
|
return entry[1] if entry is not None else frozenset()
|
||||||
|
|
||||||
|
|
||||||
|
def _revoke_share(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
subject_user_id: str,
|
||||||
|
action: DsarErasureActionRef,
|
||||||
|
request_id: str,
|
||||||
|
) -> DsarExecutionResultRef:
|
||||||
|
row = (
|
||||||
|
session.query(FileShare)
|
||||||
|
.filter(FileShare.id == action.resource_id)
|
||||||
|
.with_for_update()
|
||||||
|
.one_or_none()
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
row is None
|
||||||
|
or row.tenant_id != tenant_id
|
||||||
|
or row.target_type != "user"
|
||||||
|
or row.target_id != subject_user_id
|
||||||
|
):
|
||||||
|
return _blocked(action, "The subject-targeted file share is no longer available.")
|
||||||
|
if row.revoked_at is not None:
|
||||||
|
return _result(
|
||||||
|
action,
|
||||||
|
"unchanged",
|
||||||
|
"The file share was already revoked.",
|
||||||
|
{"request_id": request_id, "revoked_at": _iso(row.revoked_at)},
|
||||||
|
)
|
||||||
|
row.revoked_at = datetime.now(timezone.utc)
|
||||||
|
row.revoked_by_user_id = None
|
||||||
|
return _result(
|
||||||
|
action,
|
||||||
|
"executed",
|
||||||
|
"The subject-targeted file share was revoked.",
|
||||||
|
{"request_id": request_id, "revoked_at": _iso(row.revoked_at)},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _detach_reference(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
subject_user_id: str,
|
||||||
|
action: DsarErasureActionRef,
|
||||||
|
request_id: str,
|
||||||
|
) -> DsarExecutionResultRef:
|
||||||
|
entry = _DETACHABLE_MODELS.get(action.resource_type)
|
||||||
|
field_name = str(action.metadata.get("field") or "")
|
||||||
|
if entry is None or field_name not in entry[1]:
|
||||||
|
return _blocked(action, "The requested Files subject reference is not detachable.")
|
||||||
|
model = entry[0]
|
||||||
|
row = (
|
||||||
|
session.query(model)
|
||||||
|
.filter(getattr(model, "id") == action.resource_id)
|
||||||
|
.with_for_update()
|
||||||
|
.one_or_none()
|
||||||
|
)
|
||||||
|
if row is None or getattr(row, "tenant_id", None) != tenant_id:
|
||||||
|
return _blocked(action, "The Files resource is no longer available.")
|
||||||
|
current = getattr(row, field_name)
|
||||||
|
if current is None:
|
||||||
|
return _result(
|
||||||
|
action,
|
||||||
|
"unchanged",
|
||||||
|
"The subject reference was already detached.",
|
||||||
|
{"request_id": request_id, "field": field_name},
|
||||||
|
)
|
||||||
|
if current != subject_user_id:
|
||||||
|
return _blocked(action, "The Files subject reference changed after planning.")
|
||||||
|
setattr(row, field_name, None)
|
||||||
|
return _result(
|
||||||
|
action,
|
||||||
|
"executed",
|
||||||
|
"The mutable subject reference was detached.",
|
||||||
|
{"request_id": request_id, "field": field_name},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _subject_user_id(subject: DsarSubjectRef) -> str | None:
|
||||||
|
candidates: list[str] = []
|
||||||
|
if subject.membership_id:
|
||||||
|
candidates.append(subject.membership_id)
|
||||||
|
for key, value in subject.external_references.items():
|
||||||
|
if key in {
|
||||||
|
"files.user",
|
||||||
|
"files.membership",
|
||||||
|
"access.membership",
|
||||||
|
"membership_id",
|
||||||
|
}:
|
||||||
|
candidates.append(value)
|
||||||
|
normalized = {value.strip() for value in candidates if value.strip()}
|
||||||
|
if len(normalized) != 1:
|
||||||
|
return None
|
||||||
|
return normalized.pop()
|
||||||
|
|
||||||
|
|
||||||
|
def _matching_fields(
|
||||||
|
row: object,
|
||||||
|
subject_user_id: str,
|
||||||
|
fields: Sequence[str],
|
||||||
|
) -> list[str]:
|
||||||
|
return [field for field in fields if getattr(row, field) == subject_user_id]
|
||||||
|
|
||||||
|
|
||||||
|
def _record(
|
||||||
|
resource_type: str,
|
||||||
|
resource_id: str,
|
||||||
|
category: str,
|
||||||
|
title: str,
|
||||||
|
data: dict[str, object],
|
||||||
|
*,
|
||||||
|
observed_at: datetime | None = None,
|
||||||
|
immutable: bool = False,
|
||||||
|
retention_reason: str | None = None,
|
||||||
|
source_path: str | None = None,
|
||||||
|
) -> DsarRecordRef:
|
||||||
|
return DsarRecordRef(
|
||||||
|
provider_id="files",
|
||||||
|
module_id="files",
|
||||||
|
resource_type=resource_type,
|
||||||
|
resource_id=resource_id,
|
||||||
|
category=category,
|
||||||
|
title=title,
|
||||||
|
data=data,
|
||||||
|
observed_at=observed_at,
|
||||||
|
immutable_evidence=immutable,
|
||||||
|
retention_reason=retention_reason,
|
||||||
|
source_path=source_path,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _action(
|
||||||
|
action_id: str,
|
||||||
|
kind: str,
|
||||||
|
record: DsarRecordRef,
|
||||||
|
title: str,
|
||||||
|
rationale: str,
|
||||||
|
*,
|
||||||
|
executable: bool,
|
||||||
|
metadata: dict[str, object] | None = None,
|
||||||
|
) -> DsarErasureActionRef:
|
||||||
|
return DsarErasureActionRef(
|
||||||
|
action_id=action_id,
|
||||||
|
provider_id="files",
|
||||||
|
module_id="files",
|
||||||
|
kind=kind, # type: ignore[arg-type]
|
||||||
|
resource_type=record.resource_type,
|
||||||
|
resource_id=record.resource_id,
|
||||||
|
title=title,
|
||||||
|
rationale=rationale,
|
||||||
|
executable=executable,
|
||||||
|
metadata=metadata or {},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _result(
|
||||||
|
action: DsarErasureActionRef,
|
||||||
|
status: str,
|
||||||
|
summary: str,
|
||||||
|
evidence: dict[str, object] | None = None,
|
||||||
|
) -> DsarExecutionResultRef:
|
||||||
|
return DsarExecutionResultRef(
|
||||||
|
action_id=action.action_id,
|
||||||
|
status=status, # type: ignore[arg-type]
|
||||||
|
summary=summary,
|
||||||
|
evidence=evidence or {},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _blocked(action: DsarErasureActionRef, summary: str) -> DsarExecutionResultRef:
|
||||||
|
return _result(action, "blocked", summary)
|
||||||
|
|
||||||
|
|
||||||
|
def _session(value: object) -> Session:
|
||||||
|
if not isinstance(value, Session):
|
||||||
|
raise TypeError("Files DSAR provider requires a SQLAlchemy session.")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def _bounded_rows(query: object) -> list[object]:
|
||||||
|
rows = query.limit(_MAX_RECORDS + 1).all() # type: ignore[attr-defined]
|
||||||
|
if len(rows) > _MAX_RECORDS:
|
||||||
|
raise ValueError("Files DSAR match limit exceeded; narrow the subject selectors.")
|
||||||
|
return rows
|
||||||
|
|
||||||
|
|
||||||
|
def _has_table(session: Session, model: type[object]) -> bool:
|
||||||
|
return inspect(session.connection()).has_table(model.__tablename__)
|
||||||
|
|
||||||
|
|
||||||
|
def _aware(value: datetime | None) -> datetime | None:
|
||||||
|
if value is None or value.tzinfo is not None:
|
||||||
|
return value
|
||||||
|
return value.replace(tzinfo=timezone.utc)
|
||||||
|
|
||||||
|
|
||||||
|
def _iso(value: datetime | None) -> str | None:
|
||||||
|
aware = _aware(value)
|
||||||
|
return aware.isoformat() if aware else None
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["FILES_DSAR_CAPABILITY", "FilesDsarProvider"]
|
||||||
@@ -0,0 +1,449 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from collections.abc import Mapping, Sequence
|
||||||
|
from datetime import UTC, datetime, timedelta
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import secrets
|
||||||
|
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from govoplan_core.core.access import CAPABILITY_ACCESS_DIRECTORY, AccessDirectory
|
||||||
|
from govoplan_core.core.form_evidence import (
|
||||||
|
FormEvidenceContractError,
|
||||||
|
FormEvidenceGrant,
|
||||||
|
FormEvidenceGrantRequest,
|
||||||
|
FormEvidenceInspection,
|
||||||
|
FormEvidenceInspectionRequest,
|
||||||
|
FormEvidenceState,
|
||||||
|
)
|
||||||
|
from govoplan_core.core.institutional import EvidenceReference
|
||||||
|
from govoplan_core.core.modules import ModuleContext
|
||||||
|
from govoplan_files.backend.db.models import (
|
||||||
|
FileAsset,
|
||||||
|
FileBlob,
|
||||||
|
FileFormEvidenceGrant,
|
||||||
|
FileVersion,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.runtime import configure_runtime
|
||||||
|
|
||||||
|
|
||||||
|
PROVIDER_ID = "files"
|
||||||
|
CAPABILITY_FORM_EVIDENCE_FILES = "forms_runtime.evidence.files"
|
||||||
|
MAX_GRANT_TTL = timedelta(minutes=15)
|
||||||
|
|
||||||
|
|
||||||
|
class FilesFormEvidenceProvider:
|
||||||
|
provider_id = PROVIDER_ID
|
||||||
|
|
||||||
|
def __init__(self, registry: object | None, settings: object) -> None:
|
||||||
|
self._registry = registry
|
||||||
|
self._settings = settings
|
||||||
|
|
||||||
|
def supported_kinds(self) -> Sequence[str]:
|
||||||
|
return ("document",)
|
||||||
|
|
||||||
|
def create_upload_grant(
|
||||||
|
self,
|
||||||
|
session: object,
|
||||||
|
principal: object,
|
||||||
|
*,
|
||||||
|
request: FormEvidenceGrantRequest,
|
||||||
|
) -> FormEvidenceGrant:
|
||||||
|
db = _session(session)
|
||||||
|
_assert_tenant(principal, request.tenant_id)
|
||||||
|
if request.evidence_kind != "document":
|
||||||
|
raise FormEvidenceContractError(
|
||||||
|
"Files can accept only document evidence for Forms Runtime."
|
||||||
|
)
|
||||||
|
custodian_user_id = _custodian_user_id(request.custodian_ref)
|
||||||
|
self._assert_active_custodian(
|
||||||
|
tenant_id=request.tenant_id,
|
||||||
|
user_id=custodian_user_id,
|
||||||
|
)
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
if request.expires_at <= now:
|
||||||
|
raise FormEvidenceContractError(
|
||||||
|
"Form evidence upload grant expiry must be in the future."
|
||||||
|
)
|
||||||
|
allowed_content_types = _content_types(request.allowed_content_types)
|
||||||
|
configured_max = int(getattr(self._settings, "file_upload_max_bytes"))
|
||||||
|
max_size_bytes = min(request.max_size_bytes or configured_max, configured_max)
|
||||||
|
request_sha256 = _request_sha256(
|
||||||
|
request,
|
||||||
|
custodian_user_id=custodian_user_id,
|
||||||
|
max_size_bytes=max_size_bytes,
|
||||||
|
allowed_content_types=allowed_content_types,
|
||||||
|
)
|
||||||
|
existing = (
|
||||||
|
db.query(FileFormEvidenceGrant)
|
||||||
|
.filter(
|
||||||
|
FileFormEvidenceGrant.tenant_id == request.tenant_id,
|
||||||
|
FileFormEvidenceGrant.idempotency_key == request.idempotency_key,
|
||||||
|
)
|
||||||
|
.one_or_none()
|
||||||
|
)
|
||||||
|
if existing is not None:
|
||||||
|
if not secrets.compare_digest(existing.request_sha256, request_sha256):
|
||||||
|
raise FormEvidenceContractError(
|
||||||
|
"Form evidence grant idempotency conflict."
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
existing.status in {"expired", "revoked"}
|
||||||
|
or _aware(existing.expires_at) <= now
|
||||||
|
):
|
||||||
|
raise FormEvidenceContractError(
|
||||||
|
"The existing Form evidence upload grant is no longer usable; "
|
||||||
|
"request a new grant with a new idempotency key."
|
||||||
|
)
|
||||||
|
return _grant_response(existing, upload_token=None, replayed=True)
|
||||||
|
|
||||||
|
metadata = _bounded_metadata(request.metadata)
|
||||||
|
remaining_attachments = metadata.get("remaining_attachments")
|
||||||
|
if isinstance(remaining_attachments, int):
|
||||||
|
existing_attachment_ids = set(metadata.get("existing_attachment_ids", ()))
|
||||||
|
active_grants = (
|
||||||
|
db.query(FileFormEvidenceGrant)
|
||||||
|
.filter(
|
||||||
|
FileFormEvidenceGrant.tenant_id == request.tenant_id,
|
||||||
|
FileFormEvidenceGrant.form_instance_id == request.instance_id,
|
||||||
|
FileFormEvidenceGrant.form_definition_id
|
||||||
|
== request.definition_ref.object_id,
|
||||||
|
FileFormEvidenceGrant.form_definition_revision
|
||||||
|
== str(request.definition_ref.version),
|
||||||
|
FileFormEvidenceGrant.status.in_(("issued", "uploaded")),
|
||||||
|
FileFormEvidenceGrant.expires_at > now,
|
||||||
|
)
|
||||||
|
.with_for_update()
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
outstanding = sum(
|
||||||
|
1
|
||||||
|
for item in active_grants
|
||||||
|
if not item.file_asset_id
|
||||||
|
or item.file_asset_id not in existing_attachment_ids
|
||||||
|
)
|
||||||
|
if outstanding >= remaining_attachments:
|
||||||
|
raise FormEvidenceContractError(
|
||||||
|
"This Form already has the maximum number of active attachment uploads."
|
||||||
|
)
|
||||||
|
|
||||||
|
upload_token = secrets.token_urlsafe(32)
|
||||||
|
grant = FileFormEvidenceGrant(
|
||||||
|
tenant_id=request.tenant_id,
|
||||||
|
form_instance_id=request.instance_id,
|
||||||
|
form_definition_id=request.definition_ref.object_id,
|
||||||
|
form_definition_revision=str(request.definition_ref.version),
|
||||||
|
token_sha256=_token_sha256(upload_token),
|
||||||
|
idempotency_key=request.idempotency_key,
|
||||||
|
request_sha256=request_sha256,
|
||||||
|
custodian_user_id=custodian_user_id,
|
||||||
|
evidence_kind=request.evidence_kind,
|
||||||
|
purpose=request.purpose,
|
||||||
|
status="issued",
|
||||||
|
expires_at=min(request.expires_at, now + MAX_GRANT_TTL),
|
||||||
|
max_size_bytes=max_size_bytes,
|
||||||
|
allowed_content_types=list(allowed_content_types),
|
||||||
|
metadata_=metadata,
|
||||||
|
)
|
||||||
|
db.add(grant)
|
||||||
|
db.flush()
|
||||||
|
return _grant_response(grant, upload_token=upload_token, replayed=False)
|
||||||
|
|
||||||
|
def inspect_evidence(
|
||||||
|
self,
|
||||||
|
session: object,
|
||||||
|
principal: object,
|
||||||
|
*,
|
||||||
|
request: FormEvidenceInspectionRequest,
|
||||||
|
) -> FormEvidenceInspection:
|
||||||
|
db = _session(session)
|
||||||
|
_assert_tenant(principal, request.tenant_id)
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
if request.evidence.owner_module != PROVIDER_ID:
|
||||||
|
return _inspection(
|
||||||
|
request.evidence,
|
||||||
|
state="rejected",
|
||||||
|
observed_at=now,
|
||||||
|
reason="The evidence owner does not match the Files provider.",
|
||||||
|
)
|
||||||
|
if request.evidence.kind != "document" or not request.evidence.version:
|
||||||
|
return _inspection(
|
||||||
|
request.evidence,
|
||||||
|
state="rejected",
|
||||||
|
observed_at=now,
|
||||||
|
reason="Files evidence requires an exact document version.",
|
||||||
|
)
|
||||||
|
grant = (
|
||||||
|
db.query(FileFormEvidenceGrant)
|
||||||
|
.filter(
|
||||||
|
FileFormEvidenceGrant.tenant_id == request.tenant_id,
|
||||||
|
FileFormEvidenceGrant.form_instance_id == request.instance_id,
|
||||||
|
FileFormEvidenceGrant.form_definition_id
|
||||||
|
== request.definition_ref.object_id,
|
||||||
|
FileFormEvidenceGrant.form_definition_revision
|
||||||
|
== str(request.definition_ref.version),
|
||||||
|
FileFormEvidenceGrant.file_asset_id == request.evidence.evidence_id,
|
||||||
|
FileFormEvidenceGrant.file_version_id == request.evidence.version,
|
||||||
|
)
|
||||||
|
.one_or_none()
|
||||||
|
)
|
||||||
|
if grant is None:
|
||||||
|
return _inspection(
|
||||||
|
request.evidence,
|
||||||
|
state="rejected",
|
||||||
|
observed_at=now,
|
||||||
|
reason="The document was not captured for this exact Form submission.",
|
||||||
|
)
|
||||||
|
if grant.status == "revoked":
|
||||||
|
return _inspection(
|
||||||
|
request.evidence,
|
||||||
|
state="revoked",
|
||||||
|
observed_at=now,
|
||||||
|
reason="The Form evidence grant was revoked.",
|
||||||
|
)
|
||||||
|
if grant.status != "uploaded":
|
||||||
|
return _inspection(
|
||||||
|
request.evidence,
|
||||||
|
state="pending",
|
||||||
|
observed_at=now,
|
||||||
|
retryable=True,
|
||||||
|
reason="The Form evidence upload has not completed.",
|
||||||
|
)
|
||||||
|
asset = db.get(FileAsset, grant.file_asset_id)
|
||||||
|
version = db.get(FileVersion, grant.file_version_id)
|
||||||
|
blob = db.get(FileBlob, version.blob_id) if version is not None else None
|
||||||
|
if asset is None or version is None or blob is None:
|
||||||
|
return _inspection(
|
||||||
|
request.evidence,
|
||||||
|
state="unavailable",
|
||||||
|
observed_at=now,
|
||||||
|
retryable=True,
|
||||||
|
reason="The managed document cannot currently be reconstructed.",
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
asset.tenant_id != request.tenant_id
|
||||||
|
or version.tenant_id != request.tenant_id
|
||||||
|
or blob.tenant_id != request.tenant_id
|
||||||
|
or version.file_asset_id != asset.id
|
||||||
|
):
|
||||||
|
return _inspection(
|
||||||
|
request.evidence,
|
||||||
|
state="rejected",
|
||||||
|
observed_at=now,
|
||||||
|
reason="The managed document crosses an evidence ownership boundary.",
|
||||||
|
)
|
||||||
|
if asset.deleted_at is not None:
|
||||||
|
return _inspection(
|
||||||
|
request.evidence,
|
||||||
|
state="revoked",
|
||||||
|
observed_at=now,
|
||||||
|
reason="The managed document is no longer active.",
|
||||||
|
)
|
||||||
|
if blob.quarantined_at is not None or blob.integrity_status == "quarantined":
|
||||||
|
return _inspection(
|
||||||
|
request.evidence,
|
||||||
|
state="rejected",
|
||||||
|
observed_at=now,
|
||||||
|
reason="The managed document failed its integrity gate.",
|
||||||
|
)
|
||||||
|
if blob.integrity_status != "verified":
|
||||||
|
return _inspection(
|
||||||
|
request.evidence,
|
||||||
|
state="pending",
|
||||||
|
observed_at=now,
|
||||||
|
retryable=True,
|
||||||
|
reason="The managed document is awaiting integrity verification.",
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
not request.evidence.checksum
|
||||||
|
or not secrets.compare_digest(
|
||||||
|
request.evidence.checksum,
|
||||||
|
version.checksum_sha256,
|
||||||
|
)
|
||||||
|
or not secrets.compare_digest(
|
||||||
|
version.checksum_sha256,
|
||||||
|
blob.checksum_sha256,
|
||||||
|
)
|
||||||
|
):
|
||||||
|
return _inspection(
|
||||||
|
request.evidence,
|
||||||
|
state="rejected",
|
||||||
|
observed_at=now,
|
||||||
|
reason="The managed document checksum does not match the evidence.",
|
||||||
|
)
|
||||||
|
return _inspection(
|
||||||
|
request.evidence,
|
||||||
|
state="accepted",
|
||||||
|
observed_at=now,
|
||||||
|
metadata={
|
||||||
|
"content_type": version.content_type,
|
||||||
|
"size_bytes": version.size_bytes,
|
||||||
|
"integrity_status": blob.integrity_status,
|
||||||
|
"grant_id": grant.id,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
def _assert_active_custodian(self, *, tenant_id: str, user_id: str) -> None:
|
||||||
|
registry = self._registry
|
||||||
|
if registry is None or not hasattr(registry, "has_capability"):
|
||||||
|
raise FormEvidenceContractError(
|
||||||
|
"The Access directory is unavailable for Form evidence custody."
|
||||||
|
)
|
||||||
|
if not registry.has_capability(CAPABILITY_ACCESS_DIRECTORY):
|
||||||
|
raise FormEvidenceContractError(
|
||||||
|
"The Access directory is unavailable for Form evidence custody."
|
||||||
|
)
|
||||||
|
directory = registry.require_capability(CAPABILITY_ACCESS_DIRECTORY)
|
||||||
|
if not isinstance(directory, AccessDirectory):
|
||||||
|
raise FormEvidenceContractError(
|
||||||
|
"The Access directory capability is invalid."
|
||||||
|
)
|
||||||
|
user = directory.get_user(user_id)
|
||||||
|
if user is None or user.tenant_id != tenant_id or user.status != "active":
|
||||||
|
raise FormEvidenceContractError(
|
||||||
|
"Form evidence requires an active same-tenant custodian."
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def create_files_form_evidence_provider(
|
||||||
|
context: ModuleContext,
|
||||||
|
) -> FilesFormEvidenceProvider:
|
||||||
|
configure_runtime(registry=context.registry, settings=context.settings)
|
||||||
|
return FilesFormEvidenceProvider(context.registry, context.settings)
|
||||||
|
|
||||||
|
|
||||||
|
def _grant_response(
|
||||||
|
grant: FileFormEvidenceGrant,
|
||||||
|
*,
|
||||||
|
upload_token: str | None,
|
||||||
|
replayed: bool,
|
||||||
|
) -> FormEvidenceGrant:
|
||||||
|
return FormEvidenceGrant(
|
||||||
|
provider_id=PROVIDER_ID,
|
||||||
|
grant_id=grant.id,
|
||||||
|
upload_token=upload_token,
|
||||||
|
upload_url="/api/v1/files/form-evidence/upload",
|
||||||
|
expires_at=_aware(grant.expires_at),
|
||||||
|
max_size_bytes=grant.max_size_bytes,
|
||||||
|
allowed_content_types=tuple(grant.allowed_content_types),
|
||||||
|
replayed=replayed,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _inspection(
|
||||||
|
reference: EvidenceReference,
|
||||||
|
*,
|
||||||
|
state: FormEvidenceState,
|
||||||
|
observed_at: datetime,
|
||||||
|
retryable: bool = False,
|
||||||
|
reason: str | None = None,
|
||||||
|
metadata: Mapping[str, object] | None = None,
|
||||||
|
) -> FormEvidenceInspection:
|
||||||
|
return FormEvidenceInspection(
|
||||||
|
provider_id=PROVIDER_ID,
|
||||||
|
reference=reference,
|
||||||
|
state=state,
|
||||||
|
observed_at=observed_at,
|
||||||
|
retryable=retryable,
|
||||||
|
reason=reason,
|
||||||
|
metadata=dict(metadata or {}),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _session(value: object) -> Session:
|
||||||
|
if not isinstance(value, Session):
|
||||||
|
raise TypeError("Files Form evidence requires a SQLAlchemy session.")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def _assert_tenant(principal: object, tenant_id: str) -> None:
|
||||||
|
principal_tenant = str(getattr(principal, "tenant_id", "") or "").strip()
|
||||||
|
if not principal_tenant or principal_tenant != tenant_id:
|
||||||
|
raise PermissionError("Form evidence cannot cross tenants.")
|
||||||
|
|
||||||
|
|
||||||
|
def _custodian_user_id(value: str | None) -> str:
|
||||||
|
clean = str(value or "").strip()
|
||||||
|
if not clean.startswith("user:") or len(clean) <= len("user:"):
|
||||||
|
raise FormEvidenceContractError(
|
||||||
|
"Files Form evidence requires a user custodian."
|
||||||
|
)
|
||||||
|
return clean.removeprefix("user:")
|
||||||
|
|
||||||
|
|
||||||
|
def _content_types(values: Sequence[str]) -> tuple[str, ...]:
|
||||||
|
cleaned = tuple(
|
||||||
|
dict.fromkeys(
|
||||||
|
value.split(";", 1)[0].strip().casefold()
|
||||||
|
for value in values
|
||||||
|
if value.strip()
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if len(cleaned) > 50 or any(len(value) > 255 for value in cleaned):
|
||||||
|
raise FormEvidenceContractError(
|
||||||
|
"Form evidence content-type restrictions are too large."
|
||||||
|
)
|
||||||
|
return cleaned
|
||||||
|
|
||||||
|
|
||||||
|
def _bounded_metadata(value: Mapping[str, object]) -> dict[str, object]:
|
||||||
|
remaining = value.get("remaining_attachments")
|
||||||
|
result: dict[str, object] = {}
|
||||||
|
if isinstance(remaining, int) and remaining >= 0:
|
||||||
|
result["remaining_attachments"] = remaining
|
||||||
|
raw_ids = value.get("existing_attachment_ids")
|
||||||
|
if isinstance(raw_ids, (list, tuple)):
|
||||||
|
clean_ids = tuple(
|
||||||
|
dict.fromkeys(
|
||||||
|
item.strip()
|
||||||
|
for item in raw_ids
|
||||||
|
if isinstance(item, str) and item.strip() and len(item.strip()) <= 255
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if len(clean_ids) <= 1000:
|
||||||
|
result["existing_attachment_ids"] = clean_ids
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def _request_sha256(
|
||||||
|
request: FormEvidenceGrantRequest,
|
||||||
|
*,
|
||||||
|
custodian_user_id: str,
|
||||||
|
max_size_bytes: int,
|
||||||
|
allowed_content_types: Sequence[str],
|
||||||
|
) -> str:
|
||||||
|
return hashlib.sha256(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"tenant_id": request.tenant_id,
|
||||||
|
"instance_id": request.instance_id,
|
||||||
|
"definition_ref": request.definition_ref.to_dict(),
|
||||||
|
"evidence_kind": request.evidence_kind,
|
||||||
|
"purpose": request.purpose,
|
||||||
|
"expires_at": request.expires_at.isoformat(),
|
||||||
|
"custodian_user_id": custodian_user_id,
|
||||||
|
"max_size_bytes": max_size_bytes,
|
||||||
|
"allowed_content_types": list(allowed_content_types),
|
||||||
|
"metadata": _bounded_metadata(request.metadata),
|
||||||
|
},
|
||||||
|
sort_keys=True,
|
||||||
|
separators=(",", ":"),
|
||||||
|
).encode("utf-8")
|
||||||
|
).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def _token_sha256(token: str) -> str:
|
||||||
|
return hashlib.sha256(token.encode("utf-8")).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def _aware(value: datetime) -> datetime:
|
||||||
|
return value if value.tzinfo is not None else value.replace(tzinfo=UTC)
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"CAPABILITY_FORM_EVIDENCE_FILES",
|
||||||
|
"FilesFormEvidenceProvider",
|
||||||
|
"PROVIDER_ID",
|
||||||
|
"create_files_form_evidence_provider",
|
||||||
|
]
|
||||||
+1330
-100
File diff suppressed because it is too large
Load Diff
+147
@@ -0,0 +1,147 @@
|
|||||||
|
"""add purpose-bound Form evidence upload grants
|
||||||
|
|
||||||
|
Revision ID: a2b3c4d5e6f8
|
||||||
|
Revises: f1a2b3c4d5e7
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
|
||||||
|
revision = "a2b3c4d5e6f8"
|
||||||
|
down_revision = "f1a2b3c4d5e7"
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.create_table(
|
||||||
|
"file_form_evidence_grants",
|
||||||
|
sa.Column("id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("form_instance_id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("form_definition_id", sa.String(length=255), nullable=False),
|
||||||
|
sa.Column("form_definition_revision", sa.String(length=255), nullable=False),
|
||||||
|
sa.Column("token_sha256", sa.String(length=64), nullable=False),
|
||||||
|
sa.Column("idempotency_key", sa.String(length=255), nullable=False),
|
||||||
|
sa.Column("request_sha256", sa.String(length=64), nullable=False),
|
||||||
|
sa.Column("custodian_user_id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("evidence_kind", sa.String(length=30), nullable=False),
|
||||||
|
sa.Column("purpose", sa.String(length=500), nullable=False),
|
||||||
|
sa.Column("status", sa.String(length=30), nullable=False),
|
||||||
|
sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column("max_size_bytes", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("allowed_content_types", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("file_asset_id", sa.String(length=36), nullable=True),
|
||||||
|
sa.Column("file_version_id", sa.String(length=36), nullable=True),
|
||||||
|
sa.Column("uploaded_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column("revoked_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column("metadata", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["custodian_user_id"],
|
||||||
|
["access_users.id"],
|
||||||
|
ondelete="RESTRICT",
|
||||||
|
),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["file_asset_id"],
|
||||||
|
["file_assets.id"],
|
||||||
|
ondelete="RESTRICT",
|
||||||
|
),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["file_version_id"],
|
||||||
|
["file_versions.id"],
|
||||||
|
ondelete="RESTRICT",
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint("id"),
|
||||||
|
sa.UniqueConstraint(
|
||||||
|
"tenant_id",
|
||||||
|
"idempotency_key",
|
||||||
|
name="uq_file_form_evidence_grants_idempotency",
|
||||||
|
),
|
||||||
|
sa.UniqueConstraint("token_sha256"),
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_file_form_evidence_grants_tenant_id",
|
||||||
|
"file_form_evidence_grants",
|
||||||
|
["tenant_id"],
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_file_form_evidence_grants_form_instance_id",
|
||||||
|
"file_form_evidence_grants",
|
||||||
|
["form_instance_id"],
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_file_form_evidence_grants_custodian_user_id",
|
||||||
|
"file_form_evidence_grants",
|
||||||
|
["custodian_user_id"],
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_file_form_evidence_grants_status",
|
||||||
|
"file_form_evidence_grants",
|
||||||
|
["status"],
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_file_form_evidence_grants_expires_at",
|
||||||
|
"file_form_evidence_grants",
|
||||||
|
["expires_at"],
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_file_form_evidence_grants_file_asset_id",
|
||||||
|
"file_form_evidence_grants",
|
||||||
|
["file_asset_id"],
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_file_form_evidence_grants_file_version_id",
|
||||||
|
"file_form_evidence_grants",
|
||||||
|
["file_version_id"],
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_file_form_evidence_grants_form",
|
||||||
|
"file_form_evidence_grants",
|
||||||
|
[
|
||||||
|
"tenant_id",
|
||||||
|
"form_instance_id",
|
||||||
|
"form_definition_id",
|
||||||
|
"form_definition_revision",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_index(
|
||||||
|
"ix_file_form_evidence_grants_form",
|
||||||
|
table_name="file_form_evidence_grants",
|
||||||
|
)
|
||||||
|
op.drop_index(
|
||||||
|
"ix_file_form_evidence_grants_file_version_id",
|
||||||
|
table_name="file_form_evidence_grants",
|
||||||
|
)
|
||||||
|
op.drop_index(
|
||||||
|
"ix_file_form_evidence_grants_file_asset_id",
|
||||||
|
table_name="file_form_evidence_grants",
|
||||||
|
)
|
||||||
|
op.drop_index(
|
||||||
|
"ix_file_form_evidence_grants_expires_at",
|
||||||
|
table_name="file_form_evidence_grants",
|
||||||
|
)
|
||||||
|
op.drop_index(
|
||||||
|
"ix_file_form_evidence_grants_status",
|
||||||
|
table_name="file_form_evidence_grants",
|
||||||
|
)
|
||||||
|
op.drop_index(
|
||||||
|
"ix_file_form_evidence_grants_custodian_user_id",
|
||||||
|
table_name="file_form_evidence_grants",
|
||||||
|
)
|
||||||
|
op.drop_index(
|
||||||
|
"ix_file_form_evidence_grants_form_instance_id",
|
||||||
|
table_name="file_form_evidence_grants",
|
||||||
|
)
|
||||||
|
op.drop_index(
|
||||||
|
"ix_file_form_evidence_grants_tenant_id",
|
||||||
|
table_name="file_form_evidence_grants",
|
||||||
|
)
|
||||||
|
op.drop_table("file_form_evidence_grants")
|
||||||
@@ -0,0 +1,52 @@
|
|||||||
|
"""govern managed file retention and legal holds
|
||||||
|
|
||||||
|
Revision ID: a2b3c4d5e6f9
|
||||||
|
Revises: f1a2b3c4d5e7
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
|
||||||
|
revision = "a2b3c4d5e6f9"
|
||||||
|
down_revision = "f1a2b3c4d5e7"
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
with op.batch_alter_table("file_assets") as batch_op:
|
||||||
|
batch_op.add_column(
|
||||||
|
sa.Column("retained_until", sa.DateTime(timezone=True), nullable=True)
|
||||||
|
)
|
||||||
|
batch_op.add_column(
|
||||||
|
sa.Column(
|
||||||
|
"legal_hold", sa.Boolean(), nullable=False, server_default=sa.false()
|
||||||
|
)
|
||||||
|
)
|
||||||
|
batch_op.add_column(
|
||||||
|
sa.Column(
|
||||||
|
"lifecycle_revision", sa.Integer(), nullable=False, server_default="1"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
batch_op.add_column(
|
||||||
|
sa.Column("lifecycle_reason", sa.String(length=500), nullable=True)
|
||||||
|
)
|
||||||
|
batch_op.create_index(
|
||||||
|
op.f("ix_file_assets_retained_until"), ["retained_until"], unique=False
|
||||||
|
)
|
||||||
|
batch_op.create_index(
|
||||||
|
op.f("ix_file_assets_legal_hold"), ["legal_hold"], unique=False
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
with op.batch_alter_table("file_assets") as batch_op:
|
||||||
|
batch_op.drop_index(op.f("ix_file_assets_legal_hold"))
|
||||||
|
batch_op.drop_index(op.f("ix_file_assets_retained_until"))
|
||||||
|
batch_op.drop_column("lifecycle_reason")
|
||||||
|
batch_op.drop_column("lifecycle_revision")
|
||||||
|
batch_op.drop_column("legal_hold")
|
||||||
|
batch_op.drop_column("retained_until")
|
||||||
@@ -0,0 +1,53 @@
|
|||||||
|
"""file share lifecycle
|
||||||
|
|
||||||
|
Revision ID: b8c9d0e1f2a4
|
||||||
|
Revises: a7b8c9d0e1f3
|
||||||
|
Create Date: 2026-07-30 00:00:00.000000
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
|
||||||
|
revision = "b8c9d0e1f2a4"
|
||||||
|
down_revision = "a7b8c9d0e1f3"
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
with op.batch_alter_table("file_shares") as batch_op:
|
||||||
|
batch_op.add_column(
|
||||||
|
sa.Column("expires_at", sa.DateTime(timezone=True), nullable=True)
|
||||||
|
)
|
||||||
|
batch_op.add_column(
|
||||||
|
sa.Column("revoked_by_user_id", sa.String(length=36), nullable=True)
|
||||||
|
)
|
||||||
|
batch_op.create_foreign_key(
|
||||||
|
op.f("fk_file_shares_revoked_by_user_id_access_users"),
|
||||||
|
"access_users",
|
||||||
|
["revoked_by_user_id"],
|
||||||
|
["id"],
|
||||||
|
ondelete="SET NULL",
|
||||||
|
)
|
||||||
|
batch_op.create_index(
|
||||||
|
op.f("ix_file_shares_expires_at"), ["expires_at"], unique=False
|
||||||
|
)
|
||||||
|
batch_op.create_index(
|
||||||
|
op.f("ix_file_shares_revoked_by_user_id"),
|
||||||
|
["revoked_by_user_id"],
|
||||||
|
unique=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
with op.batch_alter_table("file_shares") as batch_op:
|
||||||
|
batch_op.drop_index(op.f("ix_file_shares_revoked_by_user_id"))
|
||||||
|
batch_op.drop_index(op.f("ix_file_shares_expires_at"))
|
||||||
|
batch_op.drop_constraint(
|
||||||
|
op.f("fk_file_shares_revoked_by_user_id_access_users"),
|
||||||
|
type_="foreignkey",
|
||||||
|
)
|
||||||
|
batch_op.drop_column("revoked_by_user_id")
|
||||||
|
batch_op.drop_column("expires_at")
|
||||||
+168
@@ -0,0 +1,168 @@
|
|||||||
|
"""file integrity reconciliation
|
||||||
|
|
||||||
|
Revision ID: c9d0e1f2a3b5
|
||||||
|
Revises: b8c9d0e1f2a4
|
||||||
|
Create Date: 2026-07-30 00:00:00.000000
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
|
||||||
|
revision = "c9d0e1f2a3b5"
|
||||||
|
down_revision = "b8c9d0e1f2a4"
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
with op.batch_alter_table("file_blobs") as batch_op:
|
||||||
|
batch_op.add_column(
|
||||||
|
sa.Column(
|
||||||
|
"integrity_status",
|
||||||
|
sa.String(length=30),
|
||||||
|
nullable=False,
|
||||||
|
server_default="unchecked",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
batch_op.add_column(
|
||||||
|
sa.Column("integrity_checked_at", sa.DateTime(timezone=True), nullable=True)
|
||||||
|
)
|
||||||
|
batch_op.add_column(
|
||||||
|
sa.Column("integrity_failure", sa.String(length=100), nullable=True)
|
||||||
|
)
|
||||||
|
batch_op.add_column(
|
||||||
|
sa.Column("quarantined_at", sa.DateTime(timezone=True), nullable=True)
|
||||||
|
)
|
||||||
|
batch_op.create_index(
|
||||||
|
op.f("ix_file_blobs_integrity_status"),
|
||||||
|
["integrity_status"],
|
||||||
|
unique=False,
|
||||||
|
)
|
||||||
|
batch_op.create_index(
|
||||||
|
op.f("ix_file_blobs_integrity_checked_at"),
|
||||||
|
["integrity_checked_at"],
|
||||||
|
unique=False,
|
||||||
|
)
|
||||||
|
batch_op.create_index(
|
||||||
|
op.f("ix_file_blobs_quarantined_at"),
|
||||||
|
["quarantined_at"],
|
||||||
|
unique=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
op.create_table(
|
||||||
|
"file_integrity_scans",
|
||||||
|
sa.Column("id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("storage_backend", sa.String(length=50), nullable=False),
|
||||||
|
sa.Column("storage_prefix", sa.String(length=1000), nullable=False),
|
||||||
|
sa.Column("status", sa.String(length=30), nullable=False),
|
||||||
|
sa.Column("phase", sa.String(length=30), nullable=False),
|
||||||
|
sa.Column("verify_checksums", sa.Boolean(), nullable=False),
|
||||||
|
sa.Column("batch_size", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("blob_cursor", sa.String(length=36), nullable=True),
|
||||||
|
sa.Column("object_cursor", sa.String(length=1000), nullable=True),
|
||||||
|
sa.Column("scanned_blob_count", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("verified_blob_count", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("quarantined_blob_count", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("scanned_object_count", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("orphan_object_count", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("created_by_user_id", sa.String(length=36), nullable=True),
|
||||||
|
sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column("last_error", sa.String(length=255), nullable=True),
|
||||||
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["created_by_user_id"],
|
||||||
|
["access_users.id"],
|
||||||
|
name=op.f(
|
||||||
|
"fk_file_integrity_scans_created_by_user_id_access_users"
|
||||||
|
),
|
||||||
|
ondelete="SET NULL",
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint("id", name=op.f("pk_file_integrity_scans")),
|
||||||
|
)
|
||||||
|
for column in ("tenant_id", "status", "created_by_user_id"):
|
||||||
|
op.create_index(
|
||||||
|
op.f(f"ix_file_integrity_scans_{column}"),
|
||||||
|
"file_integrity_scans",
|
||||||
|
[column],
|
||||||
|
unique=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
op.create_table(
|
||||||
|
"file_integrity_findings",
|
||||||
|
sa.Column("id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("scan_id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("kind", sa.String(length=40), nullable=False),
|
||||||
|
sa.Column("state", sa.String(length=30), nullable=False),
|
||||||
|
sa.Column("blob_id", sa.String(length=36), nullable=True),
|
||||||
|
sa.Column("storage_key", sa.String(length=1000), nullable=False),
|
||||||
|
sa.Column("expected_size_bytes", sa.Integer(), nullable=True),
|
||||||
|
sa.Column("observed_size_bytes", sa.Integer(), nullable=True),
|
||||||
|
sa.Column("expected_checksum_sha256", sa.String(length=64), nullable=True),
|
||||||
|
sa.Column("observed_checksum_sha256", sa.String(length=64), nullable=True),
|
||||||
|
sa.Column("resolved_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column("resolved_by_user_id", sa.String(length=36), nullable=True),
|
||||||
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["blob_id"],
|
||||||
|
["file_blobs.id"],
|
||||||
|
name=op.f("fk_file_integrity_findings_blob_id_file_blobs"),
|
||||||
|
ondelete="SET NULL",
|
||||||
|
),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["resolved_by_user_id"],
|
||||||
|
["access_users.id"],
|
||||||
|
name=op.f(
|
||||||
|
"fk_file_integrity_findings_resolved_by_user_id_access_users"
|
||||||
|
),
|
||||||
|
ondelete="SET NULL",
|
||||||
|
),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["scan_id"],
|
||||||
|
["file_integrity_scans.id"],
|
||||||
|
name=op.f(
|
||||||
|
"fk_file_integrity_findings_scan_id_file_integrity_scans"
|
||||||
|
),
|
||||||
|
ondelete="CASCADE",
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint("id", name=op.f("pk_file_integrity_findings")),
|
||||||
|
)
|
||||||
|
for column in (
|
||||||
|
"scan_id",
|
||||||
|
"tenant_id",
|
||||||
|
"kind",
|
||||||
|
"state",
|
||||||
|
"blob_id",
|
||||||
|
"resolved_by_user_id",
|
||||||
|
):
|
||||||
|
op.create_index(
|
||||||
|
op.f(f"ix_file_integrity_findings_{column}"),
|
||||||
|
"file_integrity_findings",
|
||||||
|
[column],
|
||||||
|
unique=False,
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_file_integrity_findings_scan_state",
|
||||||
|
"file_integrity_findings",
|
||||||
|
["scan_id", "state"],
|
||||||
|
unique=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_table("file_integrity_findings")
|
||||||
|
op.drop_table("file_integrity_scans")
|
||||||
|
with op.batch_alter_table("file_blobs") as batch_op:
|
||||||
|
batch_op.drop_index(op.f("ix_file_blobs_quarantined_at"))
|
||||||
|
batch_op.drop_index(op.f("ix_file_blobs_integrity_checked_at"))
|
||||||
|
batch_op.drop_index(op.f("ix_file_blobs_integrity_status"))
|
||||||
|
batch_op.drop_column("quarantined_at")
|
||||||
|
batch_op.drop_column("integrity_failure")
|
||||||
|
batch_op.drop_column("integrity_checked_at")
|
||||||
|
batch_op.drop_column("integrity_status")
|
||||||
+79
@@ -0,0 +1,79 @@
|
|||||||
|
"""file content-protection metadata
|
||||||
|
|
||||||
|
Revision ID: d0e1f2a3b4c6
|
||||||
|
Revises: c9d0e1f2a3b5
|
||||||
|
Create Date: 2026-08-02 00:00:00.000000
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
|
||||||
|
revision = "d0e1f2a3b4c6"
|
||||||
|
down_revision = "c9d0e1f2a3b5"
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
with op.batch_alter_table("file_blobs") as batch_op:
|
||||||
|
batch_op.add_column(
|
||||||
|
sa.Column(
|
||||||
|
"protection_discriminator",
|
||||||
|
sa.String(length=320),
|
||||||
|
nullable=False,
|
||||||
|
server_default="plaintext",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
batch_op.add_column(
|
||||||
|
sa.Column("encryption_envelope_id", sa.String(length=255), nullable=True)
|
||||||
|
)
|
||||||
|
batch_op.add_column(
|
||||||
|
sa.Column("storage_checksum_sha256", sa.String(length=64), nullable=True)
|
||||||
|
)
|
||||||
|
batch_op.add_column(
|
||||||
|
sa.Column("storage_size_bytes", sa.Integer(), nullable=True)
|
||||||
|
)
|
||||||
|
batch_op.drop_constraint(
|
||||||
|
"uq_file_blobs_tenant_checksum_size",
|
||||||
|
type_="unique",
|
||||||
|
)
|
||||||
|
batch_op.create_unique_constraint(
|
||||||
|
"uq_file_blobs_tenant_checksum_size_protection",
|
||||||
|
[
|
||||||
|
"tenant_id",
|
||||||
|
"checksum_sha256",
|
||||||
|
"size_bytes",
|
||||||
|
"protection_discriminator",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
batch_op.create_index(
|
||||||
|
op.f("ix_file_blobs_protection_discriminator"),
|
||||||
|
["protection_discriminator"],
|
||||||
|
unique=False,
|
||||||
|
)
|
||||||
|
batch_op.create_index(
|
||||||
|
op.f("ix_file_blobs_encryption_envelope_id"),
|
||||||
|
["encryption_envelope_id"],
|
||||||
|
unique=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
with op.batch_alter_table("file_blobs") as batch_op:
|
||||||
|
batch_op.drop_index(op.f("ix_file_blobs_encryption_envelope_id"))
|
||||||
|
batch_op.drop_index(op.f("ix_file_blobs_protection_discriminator"))
|
||||||
|
batch_op.drop_constraint(
|
||||||
|
"uq_file_blobs_tenant_checksum_size_protection",
|
||||||
|
type_="unique",
|
||||||
|
)
|
||||||
|
batch_op.create_unique_constraint(
|
||||||
|
"uq_file_blobs_tenant_checksum_size",
|
||||||
|
["tenant_id", "checksum_sha256", "size_bytes"],
|
||||||
|
)
|
||||||
|
batch_op.drop_column("storage_size_bytes")
|
||||||
|
batch_op.drop_column("storage_checksum_sha256")
|
||||||
|
batch_op.drop_column("encryption_envelope_id")
|
||||||
|
batch_op.drop_column("protection_discriminator")
|
||||||
+34
@@ -0,0 +1,34 @@
|
|||||||
|
"""add stale-action revisions to Files integrity operations
|
||||||
|
|
||||||
|
Revision ID: f1a2b3c4d5e7
|
||||||
|
Revises: d0e1f2a3b4c6
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
|
||||||
|
revision = "f1a2b3c4d5e7"
|
||||||
|
down_revision = "d0e1f2a3b4c6"
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
with op.batch_alter_table("file_integrity_scans") as batch_op:
|
||||||
|
batch_op.add_column(
|
||||||
|
sa.Column("revision", sa.Integer(), nullable=False, server_default="1")
|
||||||
|
)
|
||||||
|
with op.batch_alter_table("file_integrity_findings") as batch_op:
|
||||||
|
batch_op.add_column(
|
||||||
|
sa.Column("revision", sa.Integer(), nullable=False, server_default="1")
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
with op.batch_alter_table("file_integrity_findings") as batch_op:
|
||||||
|
batch_op.drop_column("revision")
|
||||||
|
with op.batch_alter_table("file_integrity_scans") as batch_op:
|
||||||
|
batch_op.drop_column("revision")
|
||||||
@@ -0,0 +1,67 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import secrets
|
||||||
|
from uuid import uuid4
|
||||||
|
|
||||||
|
from govoplan_core.core.operations import OperationalCheck
|
||||||
|
from govoplan_files.backend.storage.backends import (
|
||||||
|
StorageBackendError,
|
||||||
|
get_storage_backend,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def managed_storage_roundtrip_check() -> OperationalCheck:
|
||||||
|
"""Exercise the configured managed store without retaining probe data."""
|
||||||
|
|
||||||
|
backend = get_storage_backend()
|
||||||
|
key = f".govoplan-health/probes/{uuid4().hex}.bin"
|
||||||
|
payload = secrets.token_bytes(64)
|
||||||
|
expected_digest = hashlib.sha256(payload).hexdigest()
|
||||||
|
delete_error: Exception | None = None
|
||||||
|
try:
|
||||||
|
backend.put_bytes(key, payload, content_type="application/octet-stream")
|
||||||
|
stored = backend.get_bytes(key)
|
||||||
|
info = backend.stat(key)
|
||||||
|
if info.size_bytes != len(payload):
|
||||||
|
raise StorageBackendError("Managed storage returned an unexpected object size")
|
||||||
|
if hashlib.sha256(stored).hexdigest() != expected_digest:
|
||||||
|
raise StorageBackendError("Managed storage returned different bytes than were written")
|
||||||
|
except Exception as exc: # noqa: BLE001 - operational boundary reports provider failures.
|
||||||
|
return OperationalCheck(
|
||||||
|
id="files.managed_storage_roundtrip",
|
||||||
|
label="Managed file storage",
|
||||||
|
state="error",
|
||||||
|
detail=(
|
||||||
|
"The configured managed file store failed a bounded write/read/stat/delete "
|
||||||
|
f"probe ({type(exc).__name__})."
|
||||||
|
),
|
||||||
|
readiness_critical=True,
|
||||||
|
metrics={"backend": backend.name, "probe_bytes": len(payload)},
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
try:
|
||||||
|
backend.delete(key)
|
||||||
|
except Exception as exc: # noqa: BLE001 - reported below when the data probe passed.
|
||||||
|
delete_error = exc
|
||||||
|
|
||||||
|
if delete_error is not None:
|
||||||
|
return OperationalCheck(
|
||||||
|
id="files.managed_storage_roundtrip",
|
||||||
|
label="Managed file storage",
|
||||||
|
state="error",
|
||||||
|
detail=(
|
||||||
|
"Managed file bytes round-tripped, but probe cleanup failed "
|
||||||
|
f"({type(delete_error).__name__})."
|
||||||
|
),
|
||||||
|
readiness_critical=True,
|
||||||
|
metrics={"backend": backend.name, "probe_bytes": len(payload)},
|
||||||
|
)
|
||||||
|
return OperationalCheck(
|
||||||
|
id="files.managed_storage_roundtrip",
|
||||||
|
label="Managed file storage",
|
||||||
|
state="ok",
|
||||||
|
detail="The configured managed file store passed write, read, stat, integrity, and delete checks.",
|
||||||
|
metrics={"backend": backend.name, "probe_bytes": len(payload)},
|
||||||
|
)
|
||||||
|
|
||||||
@@ -0,0 +1,133 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from collections import defaultdict
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
|
||||||
|
from sqlalchemy import or_, select
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from govoplan_core.core.provider_governance import (
|
||||||
|
ExternalProviderRuntimeState,
|
||||||
|
ExternalProviderStateContext,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.db.models import FileConnectorProfile, FileConnectorSpace
|
||||||
|
from govoplan_files.backend.storage.connector_providers import (
|
||||||
|
ConnectorProviderDescriptor,
|
||||||
|
connector_provider_descriptors,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
REMOTE_STORAGE_PROVIDER_ID = "files.remote_storage"
|
||||||
|
|
||||||
|
|
||||||
|
def remote_storage_provider_states(
|
||||||
|
context: ExternalProviderStateContext,
|
||||||
|
) -> tuple[ExternalProviderRuntimeState, ...]:
|
||||||
|
if not isinstance(context.session, Session):
|
||||||
|
raise RuntimeError("Files provider state requires a database session.")
|
||||||
|
statement = select(FileConnectorProfile)
|
||||||
|
if context.tenant_id is not None:
|
||||||
|
statement = statement.where(
|
||||||
|
or_(
|
||||||
|
FileConnectorProfile.tenant_id.is_(None),
|
||||||
|
FileConnectorProfile.tenant_id == context.tenant_id,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
profiles = tuple(
|
||||||
|
context.session.scalars(
|
||||||
|
statement.order_by(
|
||||||
|
FileConnectorProfile.tenant_id,
|
||||||
|
FileConnectorProfile.id,
|
||||||
|
).limit(context.max_items + 1)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if not profiles:
|
||||||
|
return ()
|
||||||
|
|
||||||
|
profile_ids = tuple(item.id for item in profiles)
|
||||||
|
space_statement = select(FileConnectorSpace).where(
|
||||||
|
FileConnectorSpace.connector_profile_id.in_(profile_ids),
|
||||||
|
FileConnectorSpace.deleted_at.is_(None),
|
||||||
|
)
|
||||||
|
if context.tenant_id is not None:
|
||||||
|
space_statement = space_statement.where(
|
||||||
|
FileConnectorSpace.tenant_id == context.tenant_id
|
||||||
|
)
|
||||||
|
spaces_by_profile: dict[str, list[FileConnectorSpace]] = defaultdict(list)
|
||||||
|
for space in context.session.scalars(space_statement):
|
||||||
|
spaces_by_profile[space.connector_profile_id].append(space)
|
||||||
|
|
||||||
|
descriptors = {
|
||||||
|
item.provider: item for item in connector_provider_descriptors()
|
||||||
|
}
|
||||||
|
observed_at = datetime.now(UTC)
|
||||||
|
return tuple(
|
||||||
|
_profile_state(
|
||||||
|
profile,
|
||||||
|
spaces=spaces_by_profile.get(profile.id, []),
|
||||||
|
descriptor=descriptors.get(profile.provider),
|
||||||
|
observed_at=observed_at,
|
||||||
|
)
|
||||||
|
for profile in profiles
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _profile_state(
|
||||||
|
profile: FileConnectorProfile,
|
||||||
|
*,
|
||||||
|
spaces: list[FileConnectorSpace],
|
||||||
|
descriptor: ConnectorProviderDescriptor | None,
|
||||||
|
observed_at: datetime,
|
||||||
|
) -> ExternalProviderRuntimeState:
|
||||||
|
active_spaces = tuple(item for item in spaces if item.is_active)
|
||||||
|
active = bool(profile.enabled)
|
||||||
|
implementation_ready = bool(
|
||||||
|
descriptor is not None and descriptor.implemented and descriptor.installed
|
||||||
|
)
|
||||||
|
health = (
|
||||||
|
"inactive"
|
||||||
|
if not active
|
||||||
|
else "error"
|
||||||
|
if not implementation_ready
|
||||||
|
else "unknown"
|
||||||
|
)
|
||||||
|
recovery = (
|
||||||
|
"not_applicable"
|
||||||
|
if not active
|
||||||
|
else "unsupported"
|
||||||
|
if not implementation_ready
|
||||||
|
else "attention"
|
||||||
|
)
|
||||||
|
detail = (
|
||||||
|
"Remote-storage connector profile is disabled."
|
||||||
|
if not active
|
||||||
|
else "The configured provider is not available in this runtime."
|
||||||
|
if not implementation_ready
|
||||||
|
else "Software support is available; no live remote health observation is retained."
|
||||||
|
)
|
||||||
|
return ExternalProviderRuntimeState(
|
||||||
|
provider_id=REMOTE_STORAGE_PROVIDER_ID,
|
||||||
|
binding_ref=f"files:connector-profile:{profile.id}",
|
||||||
|
authority_mode="external_mirror",
|
||||||
|
observed_at=observed_at,
|
||||||
|
configured=True,
|
||||||
|
active=active,
|
||||||
|
health=health,
|
||||||
|
freshness="unknown" if active else "not_applicable",
|
||||||
|
conflict="not_applicable",
|
||||||
|
recovery=recovery,
|
||||||
|
detail=detail,
|
||||||
|
metrics={
|
||||||
|
"provider": profile.provider,
|
||||||
|
"configured_spaces": len(spaces),
|
||||||
|
"active_spaces": len(active_spaces),
|
||||||
|
"write_requested_spaces": sum(
|
||||||
|
1 for item in active_spaces if not item.read_only
|
||||||
|
),
|
||||||
|
"software_implemented": bool(descriptor and descriptor.implemented),
|
||||||
|
"optional_dependency_available": bool(descriptor and descriptor.installed),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["REMOTE_STORAGE_PROVIDER_ID", "remote_storage_provider_states"]
|
||||||
@@ -0,0 +1,113 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from collections.abc import Sequence
|
||||||
|
from urllib.parse import quote
|
||||||
|
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from govoplan_core.core.records import (
|
||||||
|
RecordContractError,
|
||||||
|
RecordSourceLocator,
|
||||||
|
RecordSourceReference,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.db.models import FileBlob, FileVersion
|
||||||
|
from govoplan_files.backend.storage.common import FileStorageError
|
||||||
|
from govoplan_files.backend.storage.files import get_asset_for_user
|
||||||
|
|
||||||
|
|
||||||
|
CAPABILITY_RECORD_SOURCE_FILES = "records.source.files"
|
||||||
|
|
||||||
|
|
||||||
|
class FilesRecordSource:
|
||||||
|
provider_id = "files"
|
||||||
|
|
||||||
|
def resource_types(self) -> Sequence[str]:
|
||||||
|
return ("file_version",)
|
||||||
|
|
||||||
|
def resolve(
|
||||||
|
self,
|
||||||
|
session: object,
|
||||||
|
principal: object,
|
||||||
|
*,
|
||||||
|
locator: RecordSourceLocator,
|
||||||
|
purpose: str,
|
||||||
|
) -> RecordSourceReference:
|
||||||
|
if not isinstance(session, Session):
|
||||||
|
raise RecordContractError(
|
||||||
|
"Files record references require a database session."
|
||||||
|
)
|
||||||
|
tenant_id = str(getattr(principal, "tenant_id", "") or "").strip()
|
||||||
|
if not tenant_id or locator.tenant_id != tenant_id:
|
||||||
|
raise RecordContractError("Files record references cannot cross tenants.")
|
||||||
|
if locator.source_module != "files" or locator.resource_type != "file_version":
|
||||||
|
raise RecordContractError("Unsupported Files record source type.")
|
||||||
|
if not str(purpose or "").strip():
|
||||||
|
raise RecordContractError("Files record references require a purpose.")
|
||||||
|
if not hasattr(principal, "has") or not (
|
||||||
|
principal.has("files:file:read") or principal.has("files:file:admin")
|
||||||
|
):
|
||||||
|
raise RecordContractError("Current Files read permission is required.")
|
||||||
|
user = getattr(principal, "user", None)
|
||||||
|
user_id = str(
|
||||||
|
getattr(user, "id", "") or getattr(principal, "membership_id", "") or ""
|
||||||
|
).strip()
|
||||||
|
if not user_id:
|
||||||
|
raise RecordContractError(
|
||||||
|
"Files record references require a tenant user principal."
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
asset = get_asset_for_user(
|
||||||
|
session,
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
user_id=user_id,
|
||||||
|
asset_id=locator.resource_id,
|
||||||
|
is_admin=principal.has("files:file:admin"),
|
||||||
|
)
|
||||||
|
except FileStorageError as exc:
|
||||||
|
raise RecordContractError(str(exc)) from exc
|
||||||
|
version_query = session.query(FileVersion).filter(
|
||||||
|
FileVersion.tenant_id == tenant_id,
|
||||||
|
FileVersion.file_asset_id == asset.id,
|
||||||
|
)
|
||||||
|
revision = locator.source_revision.strip()
|
||||||
|
version = version_query.filter(FileVersion.id == revision).one_or_none()
|
||||||
|
if version is None:
|
||||||
|
raise RecordContractError("The exact file version does not exist.")
|
||||||
|
blob = session.get(FileBlob, version.blob_id)
|
||||||
|
if blob is None or blob.tenant_id != tenant_id:
|
||||||
|
raise RecordContractError(
|
||||||
|
"The exact file version has no managed content object."
|
||||||
|
)
|
||||||
|
if blob.quarantined_at is not None or blob.integrity_status == "failed":
|
||||||
|
raise RecordContractError(
|
||||||
|
"The exact file version failed the current integrity gate."
|
||||||
|
)
|
||||||
|
return RecordSourceReference(
|
||||||
|
locator=locator,
|
||||||
|
label=version.filename_at_upload,
|
||||||
|
authority_mode="external_authoritative",
|
||||||
|
content_sha256=version.checksum_sha256,
|
||||||
|
content_type=version.content_type,
|
||||||
|
size_bytes=version.size_bytes,
|
||||||
|
recorded_at=version.created_at,
|
||||||
|
launch_url=(
|
||||||
|
f"/files?fileId={quote(asset.id, safe='')}&versionId={quote(version.id, safe='')}"
|
||||||
|
),
|
||||||
|
metadata={
|
||||||
|
"display_path": version.display_path_at_upload,
|
||||||
|
"version_number": version.version_number,
|
||||||
|
"integrity_status": blob.integrity_status,
|
||||||
|
"protection": blob.protection_discriminator,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def create_files_record_source(_context: object) -> FilesRecordSource:
|
||||||
|
return FilesRecordSource()
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"CAPABILITY_RECORD_SOURCE_FILES",
|
||||||
|
"FilesRecordSource",
|
||||||
|
"create_files_record_source",
|
||||||
|
]
|
||||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1 @@
|
|||||||
|
"""Focused HTTP route modules for the Files API."""
|
||||||
@@ -0,0 +1,175 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from io import BytesIO
|
||||||
|
from fastapi import APIRouter, Depends
|
||||||
|
from fastapi.responses import StreamingResponse
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from govoplan_core.auth import ApiPrincipal, require_scope
|
||||||
|
from govoplan_files.backend.schemas import (
|
||||||
|
BulkDeleteRequest,
|
||||||
|
BulkDeleteResponse,
|
||||||
|
FileAssetResponse,
|
||||||
|
)
|
||||||
|
from govoplan_core.db.session import get_session
|
||||||
|
from govoplan_files.backend.storage.common import FileStorageError
|
||||||
|
from govoplan_files.backend.storage.files import (
|
||||||
|
get_asset_for_user,
|
||||||
|
read_asset_bytes,
|
||||||
|
read_asset_version_bytes,
|
||||||
|
soft_delete_assets,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
from govoplan_files.backend.route_support import (
|
||||||
|
_asset_response,
|
||||||
|
_attachment_disposition,
|
||||||
|
_audit_connector_event,
|
||||||
|
_http_error,
|
||||||
|
_is_admin,
|
||||||
|
)
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/files", tags=["files"])
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/{file_id}", response_model=FileAssetResponse)
|
||||||
|
def get_file(
|
||||||
|
file_id: str,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(require_scope("files:file:read")),
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
asset = get_asset_for_user(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
asset_id=file_id,
|
||||||
|
is_admin=_is_admin(principal),
|
||||||
|
)
|
||||||
|
return _asset_response(session, asset, include_shares=True)
|
||||||
|
except FileStorageError as exc:
|
||||||
|
raise _http_error(exc, not_found=True) from exc
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/{file_id}/download")
|
||||||
|
def download_file(
|
||||||
|
file_id: str,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(require_scope("files:file:download")),
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
asset = get_asset_for_user(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
asset_id=file_id,
|
||||||
|
is_admin=_is_admin(principal),
|
||||||
|
)
|
||||||
|
data, version, blob = read_asset_bytes(session, asset)
|
||||||
|
_audit_connector_event(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
action="files.connector.accessed",
|
||||||
|
asset=asset,
|
||||||
|
version=version,
|
||||||
|
blob=blob,
|
||||||
|
operation="download",
|
||||||
|
commit=True,
|
||||||
|
)
|
||||||
|
except FileStorageError as exc:
|
||||||
|
raise _http_error(exc, not_found=True) from exc
|
||||||
|
headers = {"Content-Disposition": _attachment_disposition(asset.filename)}
|
||||||
|
return StreamingResponse(
|
||||||
|
BytesIO(data),
|
||||||
|
media_type=blob.content_type or "application/octet-stream",
|
||||||
|
headers=headers,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/{file_id}/versions/{version_id}/download")
|
||||||
|
def download_file_version(
|
||||||
|
file_id: str,
|
||||||
|
version_id: str,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(require_scope("files:file:download")),
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
asset = get_asset_for_user(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
asset_id=file_id,
|
||||||
|
is_admin=_is_admin(principal),
|
||||||
|
)
|
||||||
|
data, version, blob = read_asset_version_bytes(session, asset, version_id)
|
||||||
|
_audit_connector_event(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
action="files.connector.accessed",
|
||||||
|
asset=asset,
|
||||||
|
version=version,
|
||||||
|
blob=blob,
|
||||||
|
operation="download-version",
|
||||||
|
commit=True,
|
||||||
|
)
|
||||||
|
except FileStorageError as exc:
|
||||||
|
raise _http_error(exc, not_found=True) from exc
|
||||||
|
headers = {
|
||||||
|
"Content-Disposition": _attachment_disposition(
|
||||||
|
version.filename_at_upload or asset.filename
|
||||||
|
)
|
||||||
|
}
|
||||||
|
return StreamingResponse(
|
||||||
|
BytesIO(data),
|
||||||
|
media_type=blob.content_type or "application/octet-stream",
|
||||||
|
headers=headers,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.delete("/{file_id}", response_model=BulkDeleteResponse)
|
||||||
|
def delete_file(
|
||||||
|
file_id: str,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(require_scope("files:file:delete")),
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
asset = get_asset_for_user(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
asset_id=file_id,
|
||||||
|
require_write=True,
|
||||||
|
is_admin=_is_admin(principal),
|
||||||
|
)
|
||||||
|
count = soft_delete_assets(session, [asset])
|
||||||
|
session.commit()
|
||||||
|
return BulkDeleteResponse(deleted_count=count)
|
||||||
|
except FileStorageError as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _http_error(exc, not_found=True) from exc
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/bulk-delete", response_model=BulkDeleteResponse)
|
||||||
|
def bulk_delete_files(
|
||||||
|
payload: BulkDeleteRequest,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(require_scope("files:file:delete")),
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
assets = [
|
||||||
|
get_asset_for_user(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
asset_id=file_id,
|
||||||
|
require_write=True,
|
||||||
|
is_admin=_is_admin(principal),
|
||||||
|
)
|
||||||
|
for file_id in payload.file_ids
|
||||||
|
]
|
||||||
|
count = soft_delete_assets(session, assets)
|
||||||
|
session.commit()
|
||||||
|
return BulkDeleteResponse(deleted_count=count)
|
||||||
|
except FileStorageError as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _http_error(exc) from exc
|
||||||
@@ -0,0 +1,631 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from fastapi import APIRouter, Depends, HTTPException, status
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from govoplan_core.auth import ApiPrincipal, require_any_scope, require_scope
|
||||||
|
from govoplan_core.audit.logging import audit_from_principal
|
||||||
|
from govoplan_files.backend.schemas import (
|
||||||
|
FileConnectorBrowseItem,
|
||||||
|
FileConnectorBrowseResponse,
|
||||||
|
FileConnectorFolderSyncItemResponse,
|
||||||
|
FileConnectorFolderSyncRequest,
|
||||||
|
FileConnectorFolderSyncResponse,
|
||||||
|
FileConnectorFolderSyncSummary,
|
||||||
|
FileConnectorImportRequest,
|
||||||
|
FileConnectorSyncResponse,
|
||||||
|
FileConnectorWriteRequest,
|
||||||
|
FileConnectorWriteResponse,
|
||||||
|
FileUploadResponse,
|
||||||
|
)
|
||||||
|
from govoplan_core.db.session import get_session
|
||||||
|
from govoplan_files.backend.storage.paths import UnsafeFilePathError
|
||||||
|
from govoplan_files.backend.storage.common import FileStorageError
|
||||||
|
from govoplan_files.backend.storage.connector_browse import (
|
||||||
|
ConnectorBrowseError,
|
||||||
|
ConnectorBrowseUnsupported,
|
||||||
|
browse_connector_profile,
|
||||||
|
normalize_connector_browse_path,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.storage.connector_imports import (
|
||||||
|
ConnectorImportError,
|
||||||
|
ConnectorImportUnsupported,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.storage.connector_folder_sync import (
|
||||||
|
connector_relative_path,
|
||||||
|
discover_connector_folder,
|
||||||
|
join_connector_path,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.storage.connector_deployment import (
|
||||||
|
connector_effective_endpoint_url,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.storage.connector_policy import (
|
||||||
|
ConnectorAccessRequest,
|
||||||
|
ConnectorPolicyDenied,
|
||||||
|
connector_policy_decision,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.storage.files import (
|
||||||
|
create_file_asset,
|
||||||
|
get_asset_for_user,
|
||||||
|
read_asset_bytes,
|
||||||
|
sync_file_asset_from_source,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.storage.connector_spaces import (
|
||||||
|
connector_space_owner_id,
|
||||||
|
get_connector_space_for_user,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.storage.connector_writes import write_connector_file
|
||||||
|
|
||||||
|
|
||||||
|
from govoplan_files.backend.route_support import (
|
||||||
|
_asset_response,
|
||||||
|
_audit_connector_imports,
|
||||||
|
_audit_connector_sync,
|
||||||
|
_connector_browse_next_token,
|
||||||
|
_connector_policy_error,
|
||||||
|
_connector_space_policy_decision,
|
||||||
|
_download_connector_payload,
|
||||||
|
_ensure_campaign_file_access,
|
||||||
|
_http_error,
|
||||||
|
_is_admin,
|
||||||
|
_visible_connector_profile,
|
||||||
|
)
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/files", tags=["files"])
|
||||||
|
|
||||||
|
|
||||||
|
@router.post(
|
||||||
|
"/connector-spaces/{space_id}/sync",
|
||||||
|
response_model=FileConnectorFolderSyncResponse,
|
||||||
|
)
|
||||||
|
def sync_connector_space_folder(
|
||||||
|
space_id: str,
|
||||||
|
payload: FileConnectorFolderSyncRequest,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(require_scope("files:file:upload")),
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
space = get_connector_space_for_user(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
space_id=space_id,
|
||||||
|
is_admin=_is_admin(principal),
|
||||||
|
)
|
||||||
|
except FileStorageError as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _http_error(exc, not_found=True) from exc
|
||||||
|
|
||||||
|
try:
|
||||||
|
if space.sync_mode != "manual":
|
||||||
|
raise FileStorageError("This connector space is not configured for manual sync")
|
||||||
|
profile = _visible_connector_profile(
|
||||||
|
session, principal, space.connector_profile_id
|
||||||
|
)
|
||||||
|
remote_path = join_connector_path(space.remote_path, payload.path)
|
||||||
|
target_folder = join_connector_path(payload.target_folder)
|
||||||
|
decision = _connector_space_policy_decision(
|
||||||
|
profile,
|
||||||
|
library_id=space.library_id,
|
||||||
|
remote_path=remote_path,
|
||||||
|
operation="sync",
|
||||||
|
)
|
||||||
|
if not decision.allowed:
|
||||||
|
raise ConnectorPolicyDenied(decision)
|
||||||
|
discovery = discover_connector_folder(
|
||||||
|
profile,
|
||||||
|
path=remote_path,
|
||||||
|
library_id=space.library_id,
|
||||||
|
recursive=payload.recursive,
|
||||||
|
max_files=payload.max_files,
|
||||||
|
max_depth=payload.max_depth,
|
||||||
|
)
|
||||||
|
except ConnectorPolicyDenied as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _connector_policy_error(exc) from exc
|
||||||
|
except ConnectorBrowseUnsupported as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_501_NOT_IMPLEMENTED, detail=str(exc)
|
||||||
|
) from exc
|
||||||
|
except (ConnectorBrowseError, FileStorageError, ValueError) as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _http_error(exc) from exc
|
||||||
|
|
||||||
|
owner_id = connector_space_owner_id(space)
|
||||||
|
items: list[FileConnectorFolderSyncItemResponse] = [
|
||||||
|
FileConnectorFolderSyncItemResponse(
|
||||||
|
source_path=skipped.path,
|
||||||
|
action="skipped",
|
||||||
|
detail=skipped.reason,
|
||||||
|
)
|
||||||
|
for skipped in discovery.skipped
|
||||||
|
]
|
||||||
|
counts = {
|
||||||
|
"created": 0,
|
||||||
|
"updated": 0,
|
||||||
|
"unchanged": 0,
|
||||||
|
"skipped": len(discovery.skipped),
|
||||||
|
"conflicts": 0,
|
||||||
|
"policy_denied": 0,
|
||||||
|
"failed": 0,
|
||||||
|
}
|
||||||
|
|
||||||
|
for source in discovery.files:
|
||||||
|
source_path = normalize_connector_browse_path(source.path)
|
||||||
|
try:
|
||||||
|
relative_path = connector_relative_path(
|
||||||
|
space_root=space.remote_path,
|
||||||
|
item_path=source_path,
|
||||||
|
)
|
||||||
|
target_path = join_connector_path(target_folder, relative_path)
|
||||||
|
except (ConnectorBrowseError, ValueError) as exc:
|
||||||
|
counts["failed"] += 1
|
||||||
|
items.append(
|
||||||
|
FileConnectorFolderSyncItemResponse(
|
||||||
|
source_path=source_path,
|
||||||
|
action="failed",
|
||||||
|
detail=str(exc),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
|
||||||
|
try:
|
||||||
|
with session.begin_nested():
|
||||||
|
file_payload = FileConnectorImportRequest(
|
||||||
|
library_id=space.library_id or "",
|
||||||
|
path=source_path,
|
||||||
|
owner_type=space.owner_type,
|
||||||
|
owner_id=owner_id,
|
||||||
|
target_path=target_path,
|
||||||
|
conflict_strategy=payload.conflict_strategy,
|
||||||
|
source_revision=source.etag,
|
||||||
|
metadata={
|
||||||
|
**dict(source.metadata),
|
||||||
|
**payload.metadata,
|
||||||
|
"connector_space_id": space.id,
|
||||||
|
"browse_name": source.name,
|
||||||
|
"browse_path": source_path,
|
||||||
|
"browse_modified_at": source.modified_at,
|
||||||
|
"browse_etag": source.etag,
|
||||||
|
"folder_sync": True,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
_source_path, downloaded, metadata = _download_connector_payload(
|
||||||
|
profile, file_payload, operation="sync"
|
||||||
|
)
|
||||||
|
stored, sync_action, previous_version_id = (
|
||||||
|
sync_file_asset_from_source(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
owner_type=space.owner_type,
|
||||||
|
owner_id=owner_id,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
filename=downloaded.filename,
|
||||||
|
data=downloaded.data,
|
||||||
|
display_path=target_path,
|
||||||
|
content_type=downloaded.content_type,
|
||||||
|
metadata=metadata,
|
||||||
|
conflict_strategy=payload.conflict_strategy,
|
||||||
|
is_admin=_is_admin(principal),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
_audit_connector_sync(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
stored.asset,
|
||||||
|
sync_action=sync_action,
|
||||||
|
previous_version_id=previous_version_id,
|
||||||
|
)
|
||||||
|
file_response = _asset_response(
|
||||||
|
session, stored.asset, include_shares=True
|
||||||
|
)
|
||||||
|
counts[sync_action] += 1
|
||||||
|
items.append(
|
||||||
|
FileConnectorFolderSyncItemResponse(
|
||||||
|
source_path=source_path,
|
||||||
|
target_path=target_path,
|
||||||
|
action=sync_action,
|
||||||
|
file=file_response,
|
||||||
|
previous_version_id=previous_version_id,
|
||||||
|
current_version_id=stored.version.id,
|
||||||
|
source_revision=file_response.source_revision,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
except ConnectorPolicyDenied as exc:
|
||||||
|
counts["policy_denied"] += 1
|
||||||
|
items.append(
|
||||||
|
FileConnectorFolderSyncItemResponse(
|
||||||
|
source_path=source_path,
|
||||||
|
target_path=target_path,
|
||||||
|
action="policy_denied",
|
||||||
|
source_revision=source.etag,
|
||||||
|
detail=str(exc),
|
||||||
|
policy_decision=exc.decision.to_dict(),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
except FileStorageError as exc:
|
||||||
|
detail = str(exc)
|
||||||
|
if detail.startswith("Skipped upload target:"):
|
||||||
|
action = "skipped"
|
||||||
|
counts["skipped"] += 1
|
||||||
|
elif detail.startswith("Target file already exists:"):
|
||||||
|
action = "conflict"
|
||||||
|
counts["conflicts"] += 1
|
||||||
|
else:
|
||||||
|
action = "failed"
|
||||||
|
counts["failed"] += 1
|
||||||
|
items.append(
|
||||||
|
FileConnectorFolderSyncItemResponse(
|
||||||
|
source_path=source_path,
|
||||||
|
target_path=target_path,
|
||||||
|
action=action,
|
||||||
|
source_revision=source.etag,
|
||||||
|
detail=detail,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
except (
|
||||||
|
ConnectorImportError,
|
||||||
|
UnsafeFilePathError,
|
||||||
|
OSError,
|
||||||
|
ValueError,
|
||||||
|
json.JSONDecodeError,
|
||||||
|
) as exc:
|
||||||
|
counts["failed"] += 1
|
||||||
|
items.append(
|
||||||
|
FileConnectorFolderSyncItemResponse(
|
||||||
|
source_path=source_path,
|
||||||
|
target_path=target_path,
|
||||||
|
action="failed",
|
||||||
|
source_revision=source.etag,
|
||||||
|
detail=str(exc),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
summary = FileConnectorFolderSyncSummary(
|
||||||
|
discovered=len(discovery.files),
|
||||||
|
**counts,
|
||||||
|
)
|
||||||
|
audit_from_principal(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
action="files.connector.folder_synced",
|
||||||
|
object_type="file_connector_space",
|
||||||
|
object_id=space.id,
|
||||||
|
details={
|
||||||
|
"connector_profile_id": profile.id,
|
||||||
|
"provider": profile.provider,
|
||||||
|
"library_id": space.library_id,
|
||||||
|
"remote_path": remote_path,
|
||||||
|
"target_folder": target_folder,
|
||||||
|
"recursive": payload.recursive,
|
||||||
|
"truncated": discovery.truncated,
|
||||||
|
"summary": summary.model_dump(),
|
||||||
|
"results": [
|
||||||
|
{
|
||||||
|
"source_path": item.source_path,
|
||||||
|
"target_path": item.target_path,
|
||||||
|
"action": item.action,
|
||||||
|
"file_id": item.file.id if item.file else None,
|
||||||
|
"current_version_id": item.current_version_id,
|
||||||
|
}
|
||||||
|
for item in items
|
||||||
|
],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
return FileConnectorFolderSyncResponse(
|
||||||
|
connector_space_id=space.id,
|
||||||
|
connector_profile_id=profile.id,
|
||||||
|
provider=profile.provider,
|
||||||
|
remote_path=remote_path,
|
||||||
|
target_folder=target_folder,
|
||||||
|
recursive=payload.recursive,
|
||||||
|
truncated=discovery.truncated,
|
||||||
|
summary=summary,
|
||||||
|
items=items,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post(
|
||||||
|
"/connector-spaces/{space_id}/write-back",
|
||||||
|
response_model=FileConnectorWriteResponse,
|
||||||
|
)
|
||||||
|
def write_back_connector_file(
|
||||||
|
space_id: str,
|
||||||
|
payload: FileConnectorWriteRequest,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(require_scope("files:connector:write")),
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
space = get_connector_space_for_user(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
space_id=space_id,
|
||||||
|
is_admin=_is_admin(principal),
|
||||||
|
)
|
||||||
|
if space.read_only:
|
||||||
|
raise FileStorageError("This connector space is read-only")
|
||||||
|
profile = _visible_connector_profile(
|
||||||
|
session, principal, space.connector_profile_id
|
||||||
|
)
|
||||||
|
requested_path = normalize_connector_browse_path(payload.remote_path)
|
||||||
|
remote_path = normalize_connector_browse_path(
|
||||||
|
"/".join(part for part in (space.remote_path, requested_path) if part)
|
||||||
|
)
|
||||||
|
decision = connector_policy_decision(
|
||||||
|
ConnectorAccessRequest(
|
||||||
|
connector_id=profile.id,
|
||||||
|
credential_id=profile.credential_profile_id,
|
||||||
|
provider=profile.provider,
|
||||||
|
external_path=remote_path,
|
||||||
|
external_url=connector_effective_endpoint_url(
|
||||||
|
provider=profile.provider,
|
||||||
|
endpoint_url=profile.endpoint_url,
|
||||||
|
metadata=profile.metadata,
|
||||||
|
),
|
||||||
|
operation="write",
|
||||||
|
),
|
||||||
|
profile.policy_sources,
|
||||||
|
)
|
||||||
|
if not decision.allowed:
|
||||||
|
raise ConnectorPolicyDenied(decision)
|
||||||
|
asset = get_asset_for_user(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
asset_id=payload.file_id,
|
||||||
|
require_write=True,
|
||||||
|
is_admin=_is_admin(principal),
|
||||||
|
)
|
||||||
|
data, version, blob = read_asset_bytes(session, asset)
|
||||||
|
connector_library_id = space.library_id
|
||||||
|
content_type = blob.content_type
|
||||||
|
# Close the read snapshot before the independent recovery transaction
|
||||||
|
# records authority for the external effect. This avoids upgrading an
|
||||||
|
# older SQLite read snapshot after the ledger commit.
|
||||||
|
session.commit()
|
||||||
|
result = write_connector_file(
|
||||||
|
profile,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
library_id=connector_library_id,
|
||||||
|
remote_path=remote_path,
|
||||||
|
data=data,
|
||||||
|
content_type=content_type,
|
||||||
|
expected_revision=payload.expected_revision,
|
||||||
|
idempotency_key=payload.idempotency_key,
|
||||||
|
)
|
||||||
|
audit_from_principal(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
action="files.connector.written",
|
||||||
|
object_type="file",
|
||||||
|
object_id=asset.id,
|
||||||
|
details={
|
||||||
|
"file_version_id": version.id,
|
||||||
|
"file_blob_id": blob.id,
|
||||||
|
"checksum_sha256": blob.checksum_sha256,
|
||||||
|
"connector_space_id": space.id,
|
||||||
|
"connector_profile_id": profile.id,
|
||||||
|
"remote_path": remote_path,
|
||||||
|
"recovery_operation_id": result.recovery_operation_id,
|
||||||
|
"recovery_status": result.status,
|
||||||
|
"revision": result.revision,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
return FileConnectorWriteResponse(
|
||||||
|
recovery_operation_id=result.recovery_operation_id,
|
||||||
|
status=result.status,
|
||||||
|
replayed=result.replayed,
|
||||||
|
provider=result.provider,
|
||||||
|
remote_path=result.remote_path,
|
||||||
|
revision=result.revision,
|
||||||
|
checksum_sha256=result.checksum_sha256,
|
||||||
|
size_bytes=result.size_bytes,
|
||||||
|
)
|
||||||
|
except ConnectorPolicyDenied as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _connector_policy_error(exc) from exc
|
||||||
|
except (FileStorageError, ConnectorBrowseError) as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _http_error(exc) from exc
|
||||||
|
|
||||||
|
|
||||||
|
@router.post(
|
||||||
|
"/connectors/profiles/{profile_id}/import", response_model=FileUploadResponse
|
||||||
|
)
|
||||||
|
def import_connector_file(
|
||||||
|
profile_id: str,
|
||||||
|
payload: FileConnectorImportRequest,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(require_scope("files:file:upload")),
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
if payload.campaign_id:
|
||||||
|
_ensure_campaign_file_access(session, principal, payload.campaign_id)
|
||||||
|
profile = _visible_connector_profile(
|
||||||
|
session, principal, profile_id, campaign_id=payload.campaign_id
|
||||||
|
)
|
||||||
|
_source_path, downloaded, metadata = _download_connector_payload(
|
||||||
|
profile, payload, operation="import"
|
||||||
|
)
|
||||||
|
target_owner = payload.owner_id or principal.user.id
|
||||||
|
stored = create_file_asset(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
owner_type=payload.owner_type,
|
||||||
|
owner_id=target_owner,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
filename=downloaded.filename,
|
||||||
|
data=downloaded.data,
|
||||||
|
folder=payload.target_folder,
|
||||||
|
display_path=payload.target_path,
|
||||||
|
content_type=downloaded.content_type,
|
||||||
|
metadata=metadata,
|
||||||
|
campaign_id=payload.campaign_id,
|
||||||
|
conflict_strategy=payload.conflict_strategy,
|
||||||
|
is_admin=_is_admin(principal),
|
||||||
|
)
|
||||||
|
_audit_connector_imports(session, principal, [stored.asset])
|
||||||
|
session.commit()
|
||||||
|
except ConnectorPolicyDenied as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _connector_policy_error(exc) from exc
|
||||||
|
except ConnectorImportUnsupported as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_501_NOT_IMPLEMENTED, detail=str(exc)
|
||||||
|
) from exc
|
||||||
|
except (
|
||||||
|
ConnectorImportError,
|
||||||
|
FileStorageError,
|
||||||
|
UnsafeFilePathError,
|
||||||
|
ValueError,
|
||||||
|
json.JSONDecodeError,
|
||||||
|
) as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _http_error(exc) from exc
|
||||||
|
return FileUploadResponse(
|
||||||
|
files=[_asset_response(session, stored.asset, include_shares=True)]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post(
|
||||||
|
"/connectors/profiles/{profile_id}/sync", response_model=FileConnectorSyncResponse
|
||||||
|
)
|
||||||
|
def sync_connector_file(
|
||||||
|
profile_id: str,
|
||||||
|
payload: FileConnectorImportRequest,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(require_scope("files:file:upload")),
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
if payload.campaign_id:
|
||||||
|
_ensure_campaign_file_access(session, principal, payload.campaign_id)
|
||||||
|
profile = _visible_connector_profile(
|
||||||
|
session, principal, profile_id, campaign_id=payload.campaign_id
|
||||||
|
)
|
||||||
|
_source_path, downloaded, metadata = _download_connector_payload(
|
||||||
|
profile, payload, operation="sync"
|
||||||
|
)
|
||||||
|
target_owner = payload.owner_id or principal.user.id
|
||||||
|
stored, sync_action, previous_version_id = sync_file_asset_from_source(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
owner_type=payload.owner_type,
|
||||||
|
owner_id=target_owner,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
filename=downloaded.filename,
|
||||||
|
data=downloaded.data,
|
||||||
|
folder=payload.target_folder,
|
||||||
|
display_path=payload.target_path,
|
||||||
|
content_type=downloaded.content_type,
|
||||||
|
metadata=metadata,
|
||||||
|
campaign_id=payload.campaign_id,
|
||||||
|
conflict_strategy=payload.conflict_strategy,
|
||||||
|
is_admin=_is_admin(principal),
|
||||||
|
)
|
||||||
|
_audit_connector_sync(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
stored.asset,
|
||||||
|
sync_action=sync_action,
|
||||||
|
previous_version_id=previous_version_id,
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
except ConnectorPolicyDenied as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _connector_policy_error(exc) from exc
|
||||||
|
except ConnectorImportUnsupported as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_501_NOT_IMPLEMENTED, detail=str(exc)
|
||||||
|
) from exc
|
||||||
|
except (
|
||||||
|
ConnectorImportError,
|
||||||
|
FileStorageError,
|
||||||
|
UnsafeFilePathError,
|
||||||
|
ValueError,
|
||||||
|
json.JSONDecodeError,
|
||||||
|
) as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _http_error(exc) from exc
|
||||||
|
return FileConnectorSyncResponse(
|
||||||
|
file=_asset_response(session, stored.asset, include_shares=True),
|
||||||
|
action=sync_action,
|
||||||
|
previous_version_id=previous_version_id,
|
||||||
|
current_version_id=stored.version.id,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get(
|
||||||
|
"/connectors/profiles/{profile_id}/browse",
|
||||||
|
response_model=FileConnectorBrowseResponse,
|
||||||
|
)
|
||||||
|
def browse_connector_profile_items(
|
||||||
|
profile_id: str,
|
||||||
|
path: str | None = None,
|
||||||
|
library_id: str | None = None,
|
||||||
|
continuation_token: str | None = None,
|
||||||
|
campaign_id: str | None = None,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(
|
||||||
|
require_any_scope(
|
||||||
|
"files:file:read",
|
||||||
|
"files:file:upload",
|
||||||
|
"files:file:download",
|
||||||
|
"files:file:admin",
|
||||||
|
"system:settings:read",
|
||||||
|
"admin:settings:read",
|
||||||
|
)
|
||||||
|
),
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
profile = _visible_connector_profile(
|
||||||
|
session, principal, profile_id, campaign_id=campaign_id
|
||||||
|
)
|
||||||
|
browse_path = normalize_connector_browse_path(path)
|
||||||
|
decision = connector_policy_decision(
|
||||||
|
ConnectorAccessRequest(
|
||||||
|
connector_id=profile.id,
|
||||||
|
credential_id=profile.credential_profile_id,
|
||||||
|
provider=profile.provider,
|
||||||
|
external_path=browse_path,
|
||||||
|
external_url=connector_effective_endpoint_url(
|
||||||
|
provider=profile.provider,
|
||||||
|
endpoint_url=profile.endpoint_url,
|
||||||
|
metadata=profile.metadata,
|
||||||
|
),
|
||||||
|
operation="browse",
|
||||||
|
),
|
||||||
|
profile.policy_sources,
|
||||||
|
)
|
||||||
|
if not decision.allowed:
|
||||||
|
raise ConnectorPolicyDenied(decision)
|
||||||
|
items = browse_connector_profile(
|
||||||
|
profile,
|
||||||
|
path=browse_path,
|
||||||
|
library_id=library_id,
|
||||||
|
continuation_token=continuation_token,
|
||||||
|
)
|
||||||
|
except ConnectorPolicyDenied as exc:
|
||||||
|
raise _connector_policy_error(exc) from exc
|
||||||
|
except ConnectorBrowseUnsupported as exc:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_501_NOT_IMPLEMENTED, detail=str(exc)
|
||||||
|
) from exc
|
||||||
|
except (ConnectorBrowseError, OSError, ValueError, json.JSONDecodeError) as exc:
|
||||||
|
raise _http_error(exc) from exc
|
||||||
|
return FileConnectorBrowseResponse(
|
||||||
|
profile_id=profile.id,
|
||||||
|
provider=profile.provider,
|
||||||
|
path=browse_path,
|
||||||
|
library_id=library_id,
|
||||||
|
next_continuation_token=_connector_browse_next_token(items),
|
||||||
|
has_more=any(bool(item.metadata.get("listing_truncated")) for item in items),
|
||||||
|
decision=decision.to_dict(),
|
||||||
|
items=[FileConnectorBrowseItem(**item.to_response()) for item in items],
|
||||||
|
)
|
||||||
@@ -0,0 +1,261 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from fastapi import APIRouter, Depends
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from govoplan_core.auth import ApiPrincipal, require_any_scope
|
||||||
|
from govoplan_files.backend.change_tracking import (
|
||||||
|
FILES_CONNECTOR_PROFILES_COLLECTION,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.schemas import (
|
||||||
|
FileConnectorProfileResponse,
|
||||||
|
FileConnectorProfileUpdateRequest,
|
||||||
|
)
|
||||||
|
from govoplan_core.db.session import get_session
|
||||||
|
from govoplan_files.backend.storage.common import FileStorageError
|
||||||
|
from govoplan_files.backend.storage.connector_credential_deletion import (
|
||||||
|
delete_connector_profile_row,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.storage.connector_deployment import (
|
||||||
|
connector_effective_endpoint_url,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.storage.connector_profile_store import (
|
||||||
|
connector_profile_from_row,
|
||||||
|
get_connector_profile_row,
|
||||||
|
update_connector_profile_row,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.storage.connector_policy import (
|
||||||
|
ConnectorPolicyDenied,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
from govoplan_files.backend.route_support import (
|
||||||
|
FILES_CONNECTOR_PROFILE_RESOURCE,
|
||||||
|
_can_read_disabled_connector_profiles,
|
||||||
|
_connector_policy_error,
|
||||||
|
_credential_row_for_profile,
|
||||||
|
_ensure_connector_configuration_allowed,
|
||||||
|
_ensure_connector_local_policy_allowed,
|
||||||
|
_http_error,
|
||||||
|
_record_connector_settings_change,
|
||||||
|
_require_connector_profile_write,
|
||||||
|
_visible_connector_profile,
|
||||||
|
)
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/files", tags=["files"])
|
||||||
|
|
||||||
|
|
||||||
|
@router.get(
|
||||||
|
"/connectors/profiles/{profile_id}", response_model=FileConnectorProfileResponse
|
||||||
|
)
|
||||||
|
def get_connector_profile(
|
||||||
|
profile_id: str,
|
||||||
|
campaign_id: str | None = None,
|
||||||
|
include_disabled: bool = False,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(
|
||||||
|
require_any_scope(
|
||||||
|
"files:file:read",
|
||||||
|
"files:file:upload",
|
||||||
|
"files:file:download",
|
||||||
|
"files:file:admin",
|
||||||
|
"system:settings:read",
|
||||||
|
"admin:settings:read",
|
||||||
|
)
|
||||||
|
),
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
profile = _visible_connector_profile(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
profile_id,
|
||||||
|
campaign_id=campaign_id,
|
||||||
|
include_disabled=include_disabled
|
||||||
|
and _can_read_disabled_connector_profiles(principal),
|
||||||
|
include_effective_policy=False,
|
||||||
|
)
|
||||||
|
except (OSError, ValueError, json.JSONDecodeError) as exc:
|
||||||
|
raise _http_error(exc) from exc
|
||||||
|
return FileConnectorProfileResponse(**profile.to_response())
|
||||||
|
|
||||||
|
|
||||||
|
@router.patch(
|
||||||
|
"/connectors/profiles/{profile_id}", response_model=FileConnectorProfileResponse
|
||||||
|
)
|
||||||
|
def update_connector_profile(
|
||||||
|
profile_id: str,
|
||||||
|
payload: FileConnectorProfileUpdateRequest,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(
|
||||||
|
require_any_scope(
|
||||||
|
"files:file:admin", "system:settings:write", "admin:settings:write"
|
||||||
|
)
|
||||||
|
),
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
row = get_connector_profile_row(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
profile_id=profile_id,
|
||||||
|
include_disabled=True,
|
||||||
|
)
|
||||||
|
except FileStorageError as exc:
|
||||||
|
raise _http_error(exc, not_found=True) from exc
|
||||||
|
_require_connector_profile_write(principal, row.scope_type)
|
||||||
|
credentials = payload.credentials
|
||||||
|
try:
|
||||||
|
credential_profile_id = (
|
||||||
|
payload.credential_profile_id
|
||||||
|
if payload.credential_profile_id is not None
|
||||||
|
else row.credential_profile_id
|
||||||
|
)
|
||||||
|
provider = payload.provider if payload.provider is not None else row.provider
|
||||||
|
credential_row = _credential_row_for_profile(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
credential_profile_id=credential_profile_id,
|
||||||
|
provider=provider,
|
||||||
|
profile_id=row.id,
|
||||||
|
scope_type=row.scope_type,
|
||||||
|
scope_id=row.scope_id,
|
||||||
|
include_disabled=True,
|
||||||
|
)
|
||||||
|
_ensure_connector_configuration_allowed(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
connector_id=row.id,
|
||||||
|
credential_id=credential_profile_id,
|
||||||
|
provider=provider,
|
||||||
|
endpoint_url=connector_effective_endpoint_url(
|
||||||
|
provider=provider,
|
||||||
|
endpoint_url=payload.endpoint_url
|
||||||
|
if payload.endpoint_url is not None
|
||||||
|
else row.endpoint_url,
|
||||||
|
metadata=payload.metadata
|
||||||
|
if payload.metadata is not None
|
||||||
|
else row.metadata_,
|
||||||
|
),
|
||||||
|
base_path=payload.base_path
|
||||||
|
if payload.base_path is not None
|
||||||
|
else row.base_path,
|
||||||
|
scope_type=row.scope_type,
|
||||||
|
scope_id=row.scope_id,
|
||||||
|
operation="configure",
|
||||||
|
)
|
||||||
|
if payload.policy is not None:
|
||||||
|
_ensure_connector_local_policy_allowed(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
scope_type=row.scope_type,
|
||||||
|
scope_id=row.scope_id,
|
||||||
|
policy=payload.policy,
|
||||||
|
)
|
||||||
|
update_connector_profile_row(
|
||||||
|
session,
|
||||||
|
row,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
label=payload.label,
|
||||||
|
provider=payload.provider,
|
||||||
|
endpoint_url=payload.endpoint_url,
|
||||||
|
base_path=payload.base_path,
|
||||||
|
enabled=payload.enabled,
|
||||||
|
credential_profile_id=payload.credential_profile_id,
|
||||||
|
credential_mode=payload.credential_mode,
|
||||||
|
username=credentials.username if credentials else None,
|
||||||
|
password=credentials.password if credentials else None,
|
||||||
|
token=credentials.token if credentials else None,
|
||||||
|
password_env=credentials.password_env if credentials else None,
|
||||||
|
token_env=credentials.token_env if credentials else None,
|
||||||
|
secret_ref=credentials.secret_ref if credentials else None,
|
||||||
|
clear_password=payload.clear_password,
|
||||||
|
clear_token=payload.clear_token,
|
||||||
|
capabilities=payload.capabilities,
|
||||||
|
policy=payload.policy,
|
||||||
|
metadata=payload.metadata,
|
||||||
|
)
|
||||||
|
_record_connector_settings_change(
|
||||||
|
session,
|
||||||
|
collection=FILES_CONNECTOR_PROFILES_COLLECTION,
|
||||||
|
resource_type=FILES_CONNECTOR_PROFILE_RESOURCE,
|
||||||
|
resource_id=row.id,
|
||||||
|
operation="updated",
|
||||||
|
principal=principal,
|
||||||
|
tenant_id=row.tenant_id,
|
||||||
|
payload={
|
||||||
|
"scope_type": row.scope_type,
|
||||||
|
"scope_id": row.scope_id,
|
||||||
|
"provider": row.provider,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
session.refresh(row)
|
||||||
|
return FileConnectorProfileResponse(
|
||||||
|
**connector_profile_from_row(
|
||||||
|
row, credential_row=credential_row
|
||||||
|
).to_response()
|
||||||
|
)
|
||||||
|
except ConnectorPolicyDenied as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _connector_policy_error(exc) from exc
|
||||||
|
except (FileStorageError, ValueError, json.JSONDecodeError) as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _http_error(exc) from exc
|
||||||
|
|
||||||
|
|
||||||
|
@router.delete(
|
||||||
|
"/connectors/profiles/{profile_id}", response_model=FileConnectorProfileResponse
|
||||||
|
)
|
||||||
|
def deactivate_connector_profile(
|
||||||
|
profile_id: str,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(
|
||||||
|
require_any_scope(
|
||||||
|
"files:file:admin", "system:settings:write", "admin:settings:write"
|
||||||
|
)
|
||||||
|
),
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
row = get_connector_profile_row(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
profile_id=profile_id,
|
||||||
|
include_disabled=True,
|
||||||
|
)
|
||||||
|
except FileStorageError as exc:
|
||||||
|
raise _http_error(exc, not_found=True) from exc
|
||||||
|
_require_connector_profile_write(principal, row.scope_type)
|
||||||
|
try:
|
||||||
|
changed = delete_connector_profile_row(
|
||||||
|
session,
|
||||||
|
row,
|
||||||
|
deletion_reason="api_delete",
|
||||||
|
user_id=principal.user.id,
|
||||||
|
api_key_id=principal.api_key.id if principal.api_key else None,
|
||||||
|
)
|
||||||
|
if changed:
|
||||||
|
_record_connector_settings_change(
|
||||||
|
session,
|
||||||
|
collection=FILES_CONNECTOR_PROFILES_COLLECTION,
|
||||||
|
resource_type=FILES_CONNECTOR_PROFILE_RESOURCE,
|
||||||
|
resource_id=row.id,
|
||||||
|
operation="deleted",
|
||||||
|
principal=principal,
|
||||||
|
tenant_id=row.tenant_id,
|
||||||
|
payload={
|
||||||
|
"scope_type": row.scope_type,
|
||||||
|
"scope_id": row.scope_id,
|
||||||
|
"provider": row.provider,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
session.refresh(row)
|
||||||
|
return FileConnectorProfileResponse(
|
||||||
|
**connector_profile_from_row(row).to_response()
|
||||||
|
)
|
||||||
|
except FileStorageError as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _http_error(exc) from exc
|
||||||
|
except Exception:
|
||||||
|
session.rollback()
|
||||||
|
raise
|
||||||
@@ -0,0 +1,852 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from typing import Literal
|
||||||
|
from fastapi import APIRouter, Depends, Query, status
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from govoplan_core.auth import ApiPrincipal, require_any_scope
|
||||||
|
from govoplan_files.backend.change_tracking import (
|
||||||
|
FILES_CONNECTOR_CREDENTIALS_COLLECTION,
|
||||||
|
FILES_CONNECTOR_POLICIES_COLLECTION,
|
||||||
|
FILES_CONNECTOR_PROFILES_COLLECTION,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.schemas import (
|
||||||
|
FileConnectorCredentialCreateRequest,
|
||||||
|
FileConnectorCredentialResponse,
|
||||||
|
FileConnectorCredentialsResponse,
|
||||||
|
FileConnectorCredentialUpdateRequest,
|
||||||
|
FileConnectorDiscoveryRequest,
|
||||||
|
FileConnectorDiscoveryResponse,
|
||||||
|
FileConnectorSettingsDeltaResponse,
|
||||||
|
FileConnectorPolicyEvaluateRequest,
|
||||||
|
FileConnectorPolicyEvaluateResponse,
|
||||||
|
FileConnectorPolicyResponse,
|
||||||
|
FileConnectorPolicyUpdateRequest,
|
||||||
|
FileConnectorProfileCreateRequest,
|
||||||
|
FileConnectorProfileResponse,
|
||||||
|
FileConnectorProfilesResponse,
|
||||||
|
FileConnectorProviderResponse,
|
||||||
|
FileConnectorProvidersResponse,
|
||||||
|
)
|
||||||
|
from govoplan_core.db.session import get_session
|
||||||
|
from govoplan_files.backend.storage.common import FileStorageError
|
||||||
|
from govoplan_files.backend.storage.connector_credential_store import (
|
||||||
|
create_connector_credential_row,
|
||||||
|
get_connector_credential_row,
|
||||||
|
list_database_connector_credentials,
|
||||||
|
update_connector_credential_row,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.storage.connector_credential_deletion import (
|
||||||
|
delete_connector_credential_row,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.storage.connector_browse import (
|
||||||
|
ConnectorBrowseError,
|
||||||
|
ConnectorBrowseUnsupported,
|
||||||
|
browse_connector_profile,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.storage.connector_deployment import (
|
||||||
|
connector_effective_endpoint_url,
|
||||||
|
reject_api_controlled_deployment_references,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.storage.connector_profile_store import (
|
||||||
|
connector_profile_from_row,
|
||||||
|
create_connector_profile_row,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.storage.connector_providers import (
|
||||||
|
connector_provider_descriptors,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.storage.connector_policy import (
|
||||||
|
ConnectorAccessRequest,
|
||||||
|
ConnectorPolicyDenied,
|
||||||
|
connector_policy_decision,
|
||||||
|
connector_policy_sources_from_payload,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.storage.connector_policy_store import (
|
||||||
|
connector_policy_response,
|
||||||
|
set_connector_policy,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
from govoplan_files.backend.route_support import (
|
||||||
|
FILES_CONNECTOR_CREDENTIAL_RESOURCE,
|
||||||
|
FILES_CONNECTOR_POLICY_RESOURCE,
|
||||||
|
FILES_CONNECTOR_PROFILE_RESOURCE,
|
||||||
|
_audit_connector_discovery_attempt,
|
||||||
|
_can_read_disabled_connector_profiles,
|
||||||
|
_connector_credential_response,
|
||||||
|
_connector_policy_error,
|
||||||
|
_credential_row_for_profile,
|
||||||
|
_discovery_profile_from_payload,
|
||||||
|
_ensure_campaign_file_access,
|
||||||
|
_ensure_connector_configuration_allowed,
|
||||||
|
_ensure_connector_credential_configuration_allowed,
|
||||||
|
_ensure_connector_local_policy_allowed,
|
||||||
|
_file_connector_policy_resource_id,
|
||||||
|
_file_connector_settings_entries,
|
||||||
|
_http_error,
|
||||||
|
_record_connector_settings_change,
|
||||||
|
_require_connector_credential_write,
|
||||||
|
_require_connector_policy_read,
|
||||||
|
_require_connector_profile_write,
|
||||||
|
_same_endpoint,
|
||||||
|
_visible_connector_profiles,
|
||||||
|
_webdav_discovery_candidates,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.services.connector_settings_delta import (
|
||||||
|
_full_file_connector_settings_delta_response,
|
||||||
|
_incremental_file_connector_settings_delta_response,
|
||||||
|
)
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/files", tags=["files"])
|
||||||
|
|
||||||
|
|
||||||
|
@router.post(
|
||||||
|
"/connector-policy/evaluate", response_model=FileConnectorPolicyEvaluateResponse
|
||||||
|
)
|
||||||
|
def evaluate_connector_policy(
|
||||||
|
payload: FileConnectorPolicyEvaluateRequest,
|
||||||
|
principal: ApiPrincipal = Depends(
|
||||||
|
require_any_scope("files:file:read", "files:file:upload", "files:file:download")
|
||||||
|
),
|
||||||
|
):
|
||||||
|
del principal
|
||||||
|
sources = connector_policy_sources_from_payload(
|
||||||
|
[item.model_dump(mode="json") for item in payload.policy_sources]
|
||||||
|
)
|
||||||
|
request = ConnectorAccessRequest.from_provenance(
|
||||||
|
payload.source_provenance.model_dump(mode="json", exclude_none=True),
|
||||||
|
operation=payload.operation,
|
||||||
|
)
|
||||||
|
return FileConnectorPolicyEvaluateResponse(
|
||||||
|
decision=connector_policy_decision(request, sources).to_dict()
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get(
|
||||||
|
"/connectors/settings/delta", response_model=FileConnectorSettingsDeltaResponse
|
||||||
|
)
|
||||||
|
def connector_settings_delta(
|
||||||
|
scope_type: str = Query(default="tenant"),
|
||||||
|
scope_id: str | None = Query(default=None),
|
||||||
|
provider: str | None = None,
|
||||||
|
campaign_id: str | None = None,
|
||||||
|
include_disabled: bool = False,
|
||||||
|
include_inactive: bool = False,
|
||||||
|
owner_type: Literal["user", "group"] | None = None,
|
||||||
|
owner_id: str | None = None,
|
||||||
|
since: str | None = None,
|
||||||
|
limit: int = Query(default=100, ge=1, le=500),
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(
|
||||||
|
require_any_scope(
|
||||||
|
"files:file:admin", "system:settings:read", "admin:settings:read"
|
||||||
|
)
|
||||||
|
),
|
||||||
|
):
|
||||||
|
scope_type = scope_type.strip().casefold()
|
||||||
|
_require_connector_policy_read(principal, scope_type)
|
||||||
|
try:
|
||||||
|
if since is None:
|
||||||
|
return _full_file_connector_settings_delta_response(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
scope_type=scope_type,
|
||||||
|
scope_id=scope_id,
|
||||||
|
provider=provider,
|
||||||
|
campaign_id=campaign_id,
|
||||||
|
include_disabled=include_disabled,
|
||||||
|
include_inactive=include_inactive,
|
||||||
|
owner_type=owner_type,
|
||||||
|
owner_id=owner_id,
|
||||||
|
)
|
||||||
|
entries, has_more = _file_connector_settings_entries(
|
||||||
|
session, tenant_id=principal.tenant_id, since=since, limit=limit
|
||||||
|
)
|
||||||
|
if entries is None:
|
||||||
|
return _full_file_connector_settings_delta_response(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
scope_type=scope_type,
|
||||||
|
scope_id=scope_id,
|
||||||
|
provider=provider,
|
||||||
|
campaign_id=campaign_id,
|
||||||
|
include_disabled=include_disabled,
|
||||||
|
include_inactive=include_inactive,
|
||||||
|
owner_type=owner_type,
|
||||||
|
owner_id=owner_id,
|
||||||
|
)
|
||||||
|
return _incremental_file_connector_settings_delta_response(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
entries=entries,
|
||||||
|
has_more=has_more,
|
||||||
|
scope_type=scope_type,
|
||||||
|
scope_id=scope_id,
|
||||||
|
provider=provider,
|
||||||
|
campaign_id=campaign_id,
|
||||||
|
include_disabled=include_disabled,
|
||||||
|
include_inactive=include_inactive,
|
||||||
|
owner_type=owner_type,
|
||||||
|
owner_id=owner_id,
|
||||||
|
)
|
||||||
|
except (FileStorageError, ValueError, json.JSONDecodeError) as exc:
|
||||||
|
raise _http_error(exc) from exc
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/connectors/providers", response_model=FileConnectorProvidersResponse)
|
||||||
|
def list_connector_providers(
|
||||||
|
principal: ApiPrincipal = Depends(
|
||||||
|
require_any_scope("files:file:read", "files:file:upload", "files:file:admin")
|
||||||
|
),
|
||||||
|
):
|
||||||
|
del principal
|
||||||
|
return FileConnectorProvidersResponse(
|
||||||
|
providers=[
|
||||||
|
FileConnectorProviderResponse(**item.to_response())
|
||||||
|
for item in connector_provider_descriptors()
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/connectors/discover", response_model=FileConnectorDiscoveryResponse)
|
||||||
|
def discover_connector_endpoint(
|
||||||
|
payload: FileConnectorDiscoveryRequest,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(
|
||||||
|
require_any_scope(
|
||||||
|
"files:file:admin", "system:settings:write", "admin:settings:write"
|
||||||
|
)
|
||||||
|
),
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
reject_api_controlled_deployment_references(
|
||||||
|
password_env=payload.credentials.password_env,
|
||||||
|
token_env=payload.credentials.token_env,
|
||||||
|
secret_ref=payload.credentials.secret_ref,
|
||||||
|
metadata=payload.metadata,
|
||||||
|
)
|
||||||
|
except ValueError as exc:
|
||||||
|
raise _http_error(exc) from exc
|
||||||
|
if payload.provider not in {"webdav", "nextcloud"}:
|
||||||
|
return FileConnectorDiscoveryResponse(
|
||||||
|
provider=payload.provider,
|
||||||
|
endpoint_url=None,
|
||||||
|
base_path=payload.base_path,
|
||||||
|
status="unsupported",
|
||||||
|
message=f"Discovery is not implemented for {payload.provider} connectors yet",
|
||||||
|
)
|
||||||
|
candidates: list[dict[str, str]] = []
|
||||||
|
for endpoint_url in _webdav_discovery_candidates(payload):
|
||||||
|
profile = _discovery_profile_from_payload(
|
||||||
|
payload, endpoint_url, principal=principal
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
_ensure_connector_configuration_allowed(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
connector_id=None,
|
||||||
|
credential_id=None,
|
||||||
|
provider=profile.provider,
|
||||||
|
endpoint_url=endpoint_url,
|
||||||
|
base_path=profile.base_path,
|
||||||
|
scope_type="tenant",
|
||||||
|
scope_id=principal.tenant_id,
|
||||||
|
operation="discover",
|
||||||
|
)
|
||||||
|
_audit_connector_discovery_attempt(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
provider=profile.provider,
|
||||||
|
endpoint_url=endpoint_url,
|
||||||
|
base_path=profile.base_path,
|
||||||
|
)
|
||||||
|
browse_connector_profile(profile, path=payload.base_path or "")
|
||||||
|
except ConnectorPolicyDenied as exc:
|
||||||
|
raise _connector_policy_error(exc) from exc
|
||||||
|
except (
|
||||||
|
ConnectorBrowseError,
|
||||||
|
ConnectorBrowseUnsupported,
|
||||||
|
OSError,
|
||||||
|
ValueError,
|
||||||
|
json.JSONDecodeError,
|
||||||
|
) as exc:
|
||||||
|
message = str(exc)
|
||||||
|
if "credentials were rejected" in message.casefold():
|
||||||
|
if payload.require_valid_credentials:
|
||||||
|
candidates.append(
|
||||||
|
{
|
||||||
|
"endpoint_url": endpoint_url,
|
||||||
|
"status": "credentials_rejected",
|
||||||
|
"message": "The endpoint exists, but the credentials were rejected.",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return FileConnectorDiscoveryResponse(
|
||||||
|
provider=payload.provider,
|
||||||
|
endpoint_url=endpoint_url,
|
||||||
|
base_path=payload.base_path,
|
||||||
|
status="credentials_rejected",
|
||||||
|
message="The endpoint was found, but login failed with these credentials.",
|
||||||
|
candidates=candidates,
|
||||||
|
metadata={"discovered_by": "webdav-auth-challenge"},
|
||||||
|
)
|
||||||
|
candidates.append(
|
||||||
|
{
|
||||||
|
"endpoint_url": endpoint_url,
|
||||||
|
"status": "found",
|
||||||
|
"message": "The endpoint exists, but credentials are required or were rejected.",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return FileConnectorDiscoveryResponse(
|
||||||
|
provider=payload.provider,
|
||||||
|
endpoint_url=endpoint_url,
|
||||||
|
base_path=payload.base_path,
|
||||||
|
status="found",
|
||||||
|
message="The endpoint was found. Add working credentials before saving or testing the connection.",
|
||||||
|
candidates=candidates,
|
||||||
|
metadata={"discovered_by": "webdav-auth-challenge"},
|
||||||
|
)
|
||||||
|
candidates.append(
|
||||||
|
{"endpoint_url": endpoint_url, "status": "failed", "message": message}
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
status_value = (
|
||||||
|
"usable" if _same_endpoint(endpoint_url, payload.endpoint_url) else "found"
|
||||||
|
)
|
||||||
|
message = (
|
||||||
|
"The supplied URL is directly usable."
|
||||||
|
if status_value == "usable"
|
||||||
|
else "A usable connector endpoint was discovered."
|
||||||
|
)
|
||||||
|
candidates.append(
|
||||||
|
{"endpoint_url": endpoint_url, "status": status_value, "message": message}
|
||||||
|
)
|
||||||
|
return FileConnectorDiscoveryResponse(
|
||||||
|
provider=payload.provider,
|
||||||
|
endpoint_url=endpoint_url,
|
||||||
|
base_path=payload.base_path,
|
||||||
|
status=status_value,
|
||||||
|
message=message,
|
||||||
|
candidates=candidates,
|
||||||
|
metadata={"discovered_by": "webdav-propfind"},
|
||||||
|
)
|
||||||
|
return FileConnectorDiscoveryResponse(
|
||||||
|
provider=payload.provider,
|
||||||
|
endpoint_url=None,
|
||||||
|
base_path=payload.base_path,
|
||||||
|
status="not_found",
|
||||||
|
message="No usable WebDAV endpoint was found for this server URL.",
|
||||||
|
candidates=candidates,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/connectors/credentials", response_model=FileConnectorCredentialsResponse)
|
||||||
|
def list_connector_credentials(
|
||||||
|
provider: str | None = None,
|
||||||
|
include_disabled: bool = False,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(
|
||||||
|
require_any_scope(
|
||||||
|
"files:file:admin", "system:settings:read", "admin:settings:read"
|
||||||
|
)
|
||||||
|
),
|
||||||
|
):
|
||||||
|
provider_norm = provider.strip().casefold() if provider else None
|
||||||
|
credentials = list_database_connector_credentials(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
include_disabled=include_disabled
|
||||||
|
and _can_read_disabled_connector_profiles(principal),
|
||||||
|
)
|
||||||
|
return FileConnectorCredentialsResponse(
|
||||||
|
credentials=[
|
||||||
|
FileConnectorCredentialResponse(**credential.to_response())
|
||||||
|
for credential in credentials
|
||||||
|
if provider_norm is None or credential.provider in {None, provider_norm}
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get(
|
||||||
|
"/connectors/policies/{scope_type}", response_model=FileConnectorPolicyResponse
|
||||||
|
)
|
||||||
|
def read_connector_policy(
|
||||||
|
scope_type: str,
|
||||||
|
scope_id: str | None = Query(default=None),
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(
|
||||||
|
require_any_scope(
|
||||||
|
"files:file:admin", "system:settings:read", "admin:settings:read"
|
||||||
|
)
|
||||||
|
),
|
||||||
|
):
|
||||||
|
_require_connector_policy_read(principal, scope_type)
|
||||||
|
try:
|
||||||
|
return FileConnectorPolicyResponse(
|
||||||
|
**connector_policy_response(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
scope_type=scope_type,
|
||||||
|
scope_id=scope_id,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
except (FileStorageError, ValueError, json.JSONDecodeError) as exc:
|
||||||
|
raise _http_error(exc, not_found=True) from exc
|
||||||
|
|
||||||
|
|
||||||
|
@router.put(
|
||||||
|
"/connectors/policies/{scope_type}", response_model=FileConnectorPolicyResponse
|
||||||
|
)
|
||||||
|
def write_connector_policy(
|
||||||
|
scope_type: str,
|
||||||
|
payload: FileConnectorPolicyUpdateRequest,
|
||||||
|
scope_id: str | None = Query(default=None),
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(
|
||||||
|
require_any_scope(
|
||||||
|
"files:file:admin", "system:settings:write", "admin:settings:write"
|
||||||
|
)
|
||||||
|
),
|
||||||
|
):
|
||||||
|
_require_connector_profile_write(principal, scope_type)
|
||||||
|
try:
|
||||||
|
set_connector_policy(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
scope_type=scope_type,
|
||||||
|
scope_id=scope_id,
|
||||||
|
policy=payload.policy,
|
||||||
|
)
|
||||||
|
_record_connector_settings_change(
|
||||||
|
session,
|
||||||
|
collection=FILES_CONNECTOR_POLICIES_COLLECTION,
|
||||||
|
resource_type=FILES_CONNECTOR_POLICY_RESOURCE,
|
||||||
|
resource_id=_file_connector_policy_resource_id(scope_type, scope_id),
|
||||||
|
operation="updated",
|
||||||
|
principal=principal,
|
||||||
|
tenant_id=None
|
||||||
|
if scope_type.strip().casefold() == "system"
|
||||||
|
else principal.tenant_id,
|
||||||
|
payload={"scope_type": scope_type.strip().casefold(), "scope_id": scope_id},
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
return FileConnectorPolicyResponse(
|
||||||
|
**connector_policy_response(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
scope_type=scope_type,
|
||||||
|
scope_id=scope_id,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
except (FileStorageError, ValueError, json.JSONDecodeError) as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _http_error(exc) from exc
|
||||||
|
|
||||||
|
|
||||||
|
@router.post(
|
||||||
|
"/connectors/credentials",
|
||||||
|
response_model=FileConnectorCredentialResponse,
|
||||||
|
status_code=status.HTTP_201_CREATED,
|
||||||
|
)
|
||||||
|
def create_connector_credential(
|
||||||
|
payload: FileConnectorCredentialCreateRequest,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(
|
||||||
|
require_any_scope(
|
||||||
|
"files:file:admin", "system:settings:write", "admin:settings:write"
|
||||||
|
)
|
||||||
|
),
|
||||||
|
):
|
||||||
|
_require_connector_credential_write(principal, payload.scope_type)
|
||||||
|
credentials = payload.credentials
|
||||||
|
try:
|
||||||
|
_ensure_connector_credential_configuration_allowed(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
credential_id=payload.id,
|
||||||
|
provider=payload.provider,
|
||||||
|
scope_type=payload.scope_type,
|
||||||
|
scope_id=payload.scope_id,
|
||||||
|
operation="configure_credentials",
|
||||||
|
)
|
||||||
|
_ensure_connector_local_policy_allowed(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
scope_type=payload.scope_type,
|
||||||
|
scope_id=payload.scope_id,
|
||||||
|
policy=payload.policy,
|
||||||
|
)
|
||||||
|
row = create_connector_credential_row(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
credential_id=payload.id,
|
||||||
|
label=payload.label,
|
||||||
|
provider=payload.provider,
|
||||||
|
scope_type=payload.scope_type,
|
||||||
|
scope_id=payload.scope_id,
|
||||||
|
enabled=payload.enabled,
|
||||||
|
credential_mode=payload.credential_mode,
|
||||||
|
username=credentials.username,
|
||||||
|
password=credentials.password,
|
||||||
|
token=credentials.token,
|
||||||
|
password_env=credentials.password_env,
|
||||||
|
token_env=credentials.token_env,
|
||||||
|
secret_ref=credentials.secret_ref,
|
||||||
|
policy=payload.policy,
|
||||||
|
metadata=payload.metadata,
|
||||||
|
)
|
||||||
|
_record_connector_settings_change(
|
||||||
|
session,
|
||||||
|
collection=FILES_CONNECTOR_CREDENTIALS_COLLECTION,
|
||||||
|
resource_type=FILES_CONNECTOR_CREDENTIAL_RESOURCE,
|
||||||
|
resource_id=row.id,
|
||||||
|
operation="created",
|
||||||
|
principal=principal,
|
||||||
|
tenant_id=row.tenant_id,
|
||||||
|
payload={
|
||||||
|
"scope_type": row.scope_type,
|
||||||
|
"scope_id": row.scope_id,
|
||||||
|
"provider": row.provider,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
session.refresh(row)
|
||||||
|
return _connector_credential_response(row)
|
||||||
|
except ConnectorPolicyDenied as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _connector_policy_error(exc) from exc
|
||||||
|
except (FileStorageError, ValueError, json.JSONDecodeError) as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _http_error(exc) from exc
|
||||||
|
|
||||||
|
|
||||||
|
@router.get(
|
||||||
|
"/connectors/credentials/{credential_id}",
|
||||||
|
response_model=FileConnectorCredentialResponse,
|
||||||
|
)
|
||||||
|
def get_connector_credential(
|
||||||
|
credential_id: str,
|
||||||
|
include_disabled: bool = False,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(
|
||||||
|
require_any_scope(
|
||||||
|
"files:file:admin", "system:settings:read", "admin:settings:read"
|
||||||
|
)
|
||||||
|
),
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
row = get_connector_credential_row(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
credential_id=credential_id,
|
||||||
|
include_disabled=include_disabled
|
||||||
|
and _can_read_disabled_connector_profiles(principal),
|
||||||
|
)
|
||||||
|
return _connector_credential_response(row)
|
||||||
|
except FileStorageError as exc:
|
||||||
|
raise _http_error(exc, not_found=True) from exc
|
||||||
|
|
||||||
|
|
||||||
|
@router.patch(
|
||||||
|
"/connectors/credentials/{credential_id}",
|
||||||
|
response_model=FileConnectorCredentialResponse,
|
||||||
|
)
|
||||||
|
def update_connector_credential(
|
||||||
|
credential_id: str,
|
||||||
|
payload: FileConnectorCredentialUpdateRequest,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(
|
||||||
|
require_any_scope(
|
||||||
|
"files:file:admin", "system:settings:write", "admin:settings:write"
|
||||||
|
)
|
||||||
|
),
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
row = get_connector_credential_row(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
credential_id=credential_id,
|
||||||
|
include_disabled=True,
|
||||||
|
)
|
||||||
|
except FileStorageError as exc:
|
||||||
|
raise _http_error(exc, not_found=True) from exc
|
||||||
|
_require_connector_credential_write(principal, row.scope_type)
|
||||||
|
credentials = payload.credentials
|
||||||
|
try:
|
||||||
|
provider = payload.provider if payload.provider is not None else row.provider
|
||||||
|
_ensure_connector_credential_configuration_allowed(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
credential_id=row.id,
|
||||||
|
provider=provider,
|
||||||
|
scope_type=row.scope_type,
|
||||||
|
scope_id=row.scope_id,
|
||||||
|
operation="configure_credentials",
|
||||||
|
)
|
||||||
|
if payload.policy is not None:
|
||||||
|
_ensure_connector_local_policy_allowed(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
scope_type=row.scope_type,
|
||||||
|
scope_id=row.scope_id,
|
||||||
|
policy=payload.policy,
|
||||||
|
)
|
||||||
|
update_connector_credential_row(
|
||||||
|
session,
|
||||||
|
row,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
label=payload.label,
|
||||||
|
provider=payload.provider,
|
||||||
|
enabled=payload.enabled,
|
||||||
|
credential_mode=payload.credential_mode,
|
||||||
|
username=credentials.username if credentials else None,
|
||||||
|
password=credentials.password if credentials else None,
|
||||||
|
token=credentials.token if credentials else None,
|
||||||
|
password_env=credentials.password_env if credentials else None,
|
||||||
|
token_env=credentials.token_env if credentials else None,
|
||||||
|
secret_ref=credentials.secret_ref if credentials else None,
|
||||||
|
clear_password=payload.clear_password,
|
||||||
|
clear_token=payload.clear_token,
|
||||||
|
policy=payload.policy,
|
||||||
|
metadata=payload.metadata,
|
||||||
|
)
|
||||||
|
_record_connector_settings_change(
|
||||||
|
session,
|
||||||
|
collection=FILES_CONNECTOR_CREDENTIALS_COLLECTION,
|
||||||
|
resource_type=FILES_CONNECTOR_CREDENTIAL_RESOURCE,
|
||||||
|
resource_id=row.id,
|
||||||
|
operation="updated",
|
||||||
|
principal=principal,
|
||||||
|
tenant_id=row.tenant_id,
|
||||||
|
payload={
|
||||||
|
"scope_type": row.scope_type,
|
||||||
|
"scope_id": row.scope_id,
|
||||||
|
"provider": row.provider,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
session.refresh(row)
|
||||||
|
return _connector_credential_response(row)
|
||||||
|
except ConnectorPolicyDenied as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _connector_policy_error(exc) from exc
|
||||||
|
except (FileStorageError, ValueError, json.JSONDecodeError) as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _http_error(exc) from exc
|
||||||
|
|
||||||
|
|
||||||
|
@router.delete(
|
||||||
|
"/connectors/credentials/{credential_id}",
|
||||||
|
response_model=FileConnectorCredentialResponse,
|
||||||
|
)
|
||||||
|
def deactivate_connector_credential(
|
||||||
|
credential_id: str,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(
|
||||||
|
require_any_scope(
|
||||||
|
"files:file:admin", "system:settings:write", "admin:settings:write"
|
||||||
|
)
|
||||||
|
),
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
row = get_connector_credential_row(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
credential_id=credential_id,
|
||||||
|
include_disabled=True,
|
||||||
|
)
|
||||||
|
except FileStorageError as exc:
|
||||||
|
raise _http_error(exc, not_found=True) from exc
|
||||||
|
_require_connector_credential_write(principal, row.scope_type)
|
||||||
|
try:
|
||||||
|
deletion = delete_connector_credential_row(
|
||||||
|
session,
|
||||||
|
row,
|
||||||
|
deletion_reason="api_delete",
|
||||||
|
user_id=principal.user.id,
|
||||||
|
api_key_id=principal.api_key.id if principal.api_key else None,
|
||||||
|
)
|
||||||
|
if deletion.changed:
|
||||||
|
_record_connector_settings_change(
|
||||||
|
session,
|
||||||
|
collection=FILES_CONNECTOR_CREDENTIALS_COLLECTION,
|
||||||
|
resource_type=FILES_CONNECTOR_CREDENTIAL_RESOURCE,
|
||||||
|
resource_id=row.id,
|
||||||
|
operation="deleted",
|
||||||
|
principal=principal,
|
||||||
|
tenant_id=row.tenant_id,
|
||||||
|
payload={
|
||||||
|
"scope_type": row.scope_type,
|
||||||
|
"scope_id": row.scope_id,
|
||||||
|
"provider": row.provider,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
for profile in deletion.affected_profiles:
|
||||||
|
_record_connector_settings_change(
|
||||||
|
session,
|
||||||
|
collection=FILES_CONNECTOR_PROFILES_COLLECTION,
|
||||||
|
resource_type=FILES_CONNECTOR_PROFILE_RESOURCE,
|
||||||
|
resource_id=profile.id,
|
||||||
|
operation="updated",
|
||||||
|
principal=principal,
|
||||||
|
tenant_id=profile.tenant_id,
|
||||||
|
payload={
|
||||||
|
"scope_type": profile.scope_type,
|
||||||
|
"scope_id": profile.scope_id,
|
||||||
|
"provider": profile.provider,
|
||||||
|
"reason": "credential_deleted",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
session.refresh(row)
|
||||||
|
return _connector_credential_response(row)
|
||||||
|
except FileStorageError as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _http_error(exc) from exc
|
||||||
|
except Exception:
|
||||||
|
session.rollback()
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/connectors/profiles", response_model=FileConnectorProfilesResponse)
|
||||||
|
def list_connector_profiles(
|
||||||
|
provider: str | None = None,
|
||||||
|
campaign_id: str | None = None,
|
||||||
|
include_disabled: bool = False,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(
|
||||||
|
require_any_scope(
|
||||||
|
"files:file:read",
|
||||||
|
"files:file:upload",
|
||||||
|
"files:file:download",
|
||||||
|
"files:file:admin",
|
||||||
|
"system:settings:read",
|
||||||
|
"admin:settings:read",
|
||||||
|
)
|
||||||
|
),
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
if campaign_id:
|
||||||
|
_ensure_campaign_file_access(session, principal, campaign_id)
|
||||||
|
profiles = _visible_connector_profiles(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
provider=provider,
|
||||||
|
campaign_id=campaign_id,
|
||||||
|
include_disabled=include_disabled
|
||||||
|
and _can_read_disabled_connector_profiles(principal),
|
||||||
|
include_admin_scopes=_can_read_disabled_connector_profiles(principal),
|
||||||
|
include_effective_policy=False,
|
||||||
|
)
|
||||||
|
except (OSError, ValueError, json.JSONDecodeError) as exc:
|
||||||
|
raise _http_error(exc) from exc
|
||||||
|
return FileConnectorProfilesResponse(
|
||||||
|
profiles=[
|
||||||
|
FileConnectorProfileResponse(**profile.to_response())
|
||||||
|
for profile in profiles
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post(
|
||||||
|
"/connectors/profiles",
|
||||||
|
response_model=FileConnectorProfileResponse,
|
||||||
|
status_code=status.HTTP_201_CREATED,
|
||||||
|
)
|
||||||
|
def create_connector_profile(
|
||||||
|
payload: FileConnectorProfileCreateRequest,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(
|
||||||
|
require_any_scope(
|
||||||
|
"files:file:admin", "system:settings:write", "admin:settings:write"
|
||||||
|
)
|
||||||
|
),
|
||||||
|
):
|
||||||
|
_require_connector_profile_write(principal, payload.scope_type)
|
||||||
|
credentials = payload.credentials
|
||||||
|
try:
|
||||||
|
credential_row = _credential_row_for_profile(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
credential_profile_id=payload.credential_profile_id,
|
||||||
|
provider=payload.provider,
|
||||||
|
profile_id=payload.id,
|
||||||
|
scope_type=payload.scope_type,
|
||||||
|
scope_id=payload.scope_id,
|
||||||
|
)
|
||||||
|
_ensure_connector_configuration_allowed(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
connector_id=payload.id,
|
||||||
|
credential_id=payload.credential_profile_id,
|
||||||
|
provider=payload.provider,
|
||||||
|
endpoint_url=connector_effective_endpoint_url(
|
||||||
|
provider=payload.provider,
|
||||||
|
endpoint_url=payload.endpoint_url,
|
||||||
|
metadata=payload.metadata,
|
||||||
|
),
|
||||||
|
base_path=payload.base_path,
|
||||||
|
scope_type=payload.scope_type,
|
||||||
|
scope_id=payload.scope_id,
|
||||||
|
operation="configure",
|
||||||
|
)
|
||||||
|
_ensure_connector_local_policy_allowed(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
scope_type=payload.scope_type,
|
||||||
|
scope_id=payload.scope_id,
|
||||||
|
policy=payload.policy,
|
||||||
|
)
|
||||||
|
row = create_connector_profile_row(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
profile_id=payload.id,
|
||||||
|
label=payload.label,
|
||||||
|
provider=payload.provider,
|
||||||
|
scope_type=payload.scope_type,
|
||||||
|
scope_id=payload.scope_id,
|
||||||
|
endpoint_url=payload.endpoint_url,
|
||||||
|
base_path=payload.base_path,
|
||||||
|
enabled=payload.enabled,
|
||||||
|
credential_profile_id=payload.credential_profile_id,
|
||||||
|
credential_mode=payload.credential_mode,
|
||||||
|
username=credentials.username,
|
||||||
|
password=credentials.password,
|
||||||
|
token=credentials.token,
|
||||||
|
password_env=credentials.password_env,
|
||||||
|
token_env=credentials.token_env,
|
||||||
|
secret_ref=credentials.secret_ref,
|
||||||
|
capabilities=payload.capabilities,
|
||||||
|
policy=payload.policy,
|
||||||
|
metadata=payload.metadata,
|
||||||
|
)
|
||||||
|
_record_connector_settings_change(
|
||||||
|
session,
|
||||||
|
collection=FILES_CONNECTOR_PROFILES_COLLECTION,
|
||||||
|
resource_type=FILES_CONNECTOR_PROFILE_RESOURCE,
|
||||||
|
resource_id=row.id,
|
||||||
|
operation="created",
|
||||||
|
principal=principal,
|
||||||
|
tenant_id=row.tenant_id,
|
||||||
|
payload={
|
||||||
|
"scope_type": row.scope_type,
|
||||||
|
"scope_id": row.scope_id,
|
||||||
|
"provider": row.provider,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
session.refresh(row)
|
||||||
|
return FileConnectorProfileResponse(
|
||||||
|
**connector_profile_from_row(
|
||||||
|
row, credential_row=credential_row
|
||||||
|
).to_response()
|
||||||
|
)
|
||||||
|
except ConnectorPolicyDenied as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _connector_policy_error(exc) from exc
|
||||||
|
except (FileStorageError, ValueError, json.JSONDecodeError) as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _http_error(exc) from exc
|
||||||
@@ -0,0 +1,151 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import Literal
|
||||||
|
from fastapi import APIRouter, Depends, Query
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from govoplan_core.auth import ApiPrincipal, require_scope
|
||||||
|
from govoplan_files.backend.schemas import (
|
||||||
|
FileFolderCreateRequest,
|
||||||
|
FileFolderDeleteRequest,
|
||||||
|
FileFolderDeleteResponse,
|
||||||
|
FileFolderResponse,
|
||||||
|
FileFoldersResponse,
|
||||||
|
)
|
||||||
|
from govoplan_core.db.session import get_session
|
||||||
|
from govoplan_files.backend.storage.paths import UnsafeFilePathError
|
||||||
|
from govoplan_files.backend.storage.common import FileStorageError
|
||||||
|
from govoplan_files.backend.storage.folders import (
|
||||||
|
create_folder,
|
||||||
|
list_folders_for_user,
|
||||||
|
list_folders_for_user_window,
|
||||||
|
soft_delete_folder,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
from govoplan_files.backend.route_support import (
|
||||||
|
_folder_response,
|
||||||
|
_http_error,
|
||||||
|
_is_admin,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.services.list_queries import (
|
||||||
|
FOLDERS_LIST_CURSOR_SCOPE,
|
||||||
|
_cursor_page_size,
|
||||||
|
_files_delta_watermark,
|
||||||
|
_folder_cursor_values,
|
||||||
|
_folders_list_fingerprint,
|
||||||
|
_next_folder_list_cursor,
|
||||||
|
)
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/files", tags=["files"])
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/folders", response_model=FileFoldersResponse)
|
||||||
|
def list_file_folders(
|
||||||
|
owner_type: Literal["user", "group"],
|
||||||
|
owner_id: str,
|
||||||
|
page_size: int | None = Query(default=None, ge=1, le=1000),
|
||||||
|
cursor: str | None = None,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(require_scope("files:file:read")),
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
watermark = _files_delta_watermark(session, principal.tenant_id)
|
||||||
|
effective_page_size = _cursor_page_size(
|
||||||
|
FOLDERS_LIST_CURSOR_SCOPE, cursor, page_size
|
||||||
|
)
|
||||||
|
if effective_page_size is None:
|
||||||
|
folders = list_folders_for_user(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
owner_type=owner_type,
|
||||||
|
owner_id=owner_id,
|
||||||
|
is_admin=_is_admin(principal),
|
||||||
|
)
|
||||||
|
return FileFoldersResponse(
|
||||||
|
folders=[_folder_response(folder) for folder in folders],
|
||||||
|
watermark=watermark,
|
||||||
|
)
|
||||||
|
fingerprint = _folders_list_fingerprint(
|
||||||
|
principal,
|
||||||
|
owner_type=owner_type,
|
||||||
|
owner_id=owner_id,
|
||||||
|
page_size=effective_page_size,
|
||||||
|
)
|
||||||
|
after_path, after_id = _folder_cursor_values(cursor, fingerprint=fingerprint)
|
||||||
|
folders, has_more = list_folders_for_user_window(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
owner_type=owner_type,
|
||||||
|
owner_id=owner_id,
|
||||||
|
is_admin=_is_admin(principal),
|
||||||
|
page_size=effective_page_size,
|
||||||
|
after_path=after_path,
|
||||||
|
after_id=after_id,
|
||||||
|
)
|
||||||
|
return FileFoldersResponse(
|
||||||
|
folders=[_folder_response(folder) for folder in folders],
|
||||||
|
cursor=cursor,
|
||||||
|
next_cursor=_next_folder_list_cursor(
|
||||||
|
principal,
|
||||||
|
folders,
|
||||||
|
owner_type=owner_type,
|
||||||
|
owner_id=owner_id,
|
||||||
|
page_size=effective_page_size,
|
||||||
|
has_more=has_more,
|
||||||
|
),
|
||||||
|
watermark=watermark,
|
||||||
|
)
|
||||||
|
except FileStorageError as exc:
|
||||||
|
raise _http_error(exc) from exc
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/folders", response_model=FileFolderResponse)
|
||||||
|
def create_file_folder(
|
||||||
|
payload: FileFolderCreateRequest,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(require_scope("files:file:organize")),
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
folder = create_folder(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
owner_type=payload.owner_type,
|
||||||
|
owner_id=payload.owner_id,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
path=payload.path,
|
||||||
|
is_admin=_is_admin(principal),
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
return _folder_response(folder)
|
||||||
|
except (FileStorageError, UnsafeFilePathError, ValueError) as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _http_error(exc) from exc
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/folders/delete", response_model=FileFolderDeleteResponse)
|
||||||
|
def delete_file_folder(
|
||||||
|
payload: FileFolderDeleteRequest,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(require_scope("files:file:delete")),
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
deleted_folders, deleted_files = soft_delete_folder(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
owner_type=payload.owner_type,
|
||||||
|
owner_id=payload.owner_id,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
path=payload.path,
|
||||||
|
recursive=payload.recursive,
|
||||||
|
is_admin=_is_admin(principal),
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
return FileFolderDeleteResponse(
|
||||||
|
deleted_folders=deleted_folders, deleted_files=deleted_files
|
||||||
|
)
|
||||||
|
except (FileStorageError, UnsafeFilePathError, ValueError) as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _http_error(exc) from exc
|
||||||
@@ -0,0 +1,172 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
import hashlib
|
||||||
|
|
||||||
|
from fastapi import (
|
||||||
|
APIRouter,
|
||||||
|
Depends,
|
||||||
|
File as FastAPIFile,
|
||||||
|
Header,
|
||||||
|
HTTPException,
|
||||||
|
UploadFile,
|
||||||
|
status,
|
||||||
|
)
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from govoplan_core.core.events import (
|
||||||
|
EventActorRef,
|
||||||
|
EventObjectRef,
|
||||||
|
EventTenantRef,
|
||||||
|
PlatformEvent,
|
||||||
|
emit_platform_event,
|
||||||
|
)
|
||||||
|
from govoplan_core.core.institutional import EvidenceReference
|
||||||
|
from govoplan_core.db.session import get_session
|
||||||
|
from govoplan_files.backend.db.models import FileFormEvidenceGrant
|
||||||
|
from govoplan_files.backend.route_support import _read_limited_upload
|
||||||
|
from govoplan_files.backend.storage.common import FileStorageError
|
||||||
|
from govoplan_files.backend.storage.files import create_file_asset
|
||||||
|
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/files", tags=["files"])
|
||||||
|
|
||||||
|
|
||||||
|
@router.post(
|
||||||
|
"/form-evidence/upload",
|
||||||
|
response_model=dict[str, object],
|
||||||
|
status_code=status.HTTP_201_CREATED,
|
||||||
|
)
|
||||||
|
def upload_form_evidence(
|
||||||
|
file: UploadFile = FastAPIFile(...),
|
||||||
|
x_form_evidence_token: str = Header(alias="X-Form-Evidence-Token"),
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
) -> dict[str, object]:
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
try:
|
||||||
|
grant = (
|
||||||
|
session.query(FileFormEvidenceGrant)
|
||||||
|
.filter(
|
||||||
|
FileFormEvidenceGrant.token_sha256
|
||||||
|
== _token_sha256(x_form_evidence_token)
|
||||||
|
)
|
||||||
|
.with_for_update()
|
||||||
|
.one_or_none()
|
||||||
|
)
|
||||||
|
if grant is None or grant.status != "issued" or _aware(grant.expires_at) <= now:
|
||||||
|
raise _upload_unavailable()
|
||||||
|
content_type = (
|
||||||
|
str(file.content_type or "application/octet-stream")
|
||||||
|
.split(";", 1)[0]
|
||||||
|
.strip()
|
||||||
|
.casefold()
|
||||||
|
)
|
||||||
|
if grant.allowed_content_types and content_type not in set(
|
||||||
|
grant.allowed_content_types
|
||||||
|
):
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_415_UNSUPPORTED_MEDIA_TYPE,
|
||||||
|
detail="This Form does not accept the uploaded document type.",
|
||||||
|
)
|
||||||
|
data = _read_limited_upload(file, max_bytes=grant.max_size_bytes)
|
||||||
|
stored = create_file_asset(
|
||||||
|
session,
|
||||||
|
tenant_id=grant.tenant_id,
|
||||||
|
owner_type="user",
|
||||||
|
owner_id=grant.custodian_user_id,
|
||||||
|
user_id=grant.custodian_user_id,
|
||||||
|
filename=file.filename or "form-attachment",
|
||||||
|
data=data,
|
||||||
|
folder=f"Form submissions/{grant.form_instance_id}",
|
||||||
|
content_type=content_type,
|
||||||
|
description="Managed attachment captured through Forms Runtime.",
|
||||||
|
metadata={
|
||||||
|
"form_evidence": {
|
||||||
|
"grant_id": grant.id,
|
||||||
|
"form_instance_id": grant.form_instance_id,
|
||||||
|
"form_definition_id": grant.form_definition_id,
|
||||||
|
"form_definition_revision": grant.form_definition_revision,
|
||||||
|
"purpose": grant.purpose,
|
||||||
|
},
|
||||||
|
"source_provenance": {
|
||||||
|
"source_type": "form_evidence",
|
||||||
|
"connector_id": "forms_runtime.evidence.files",
|
||||||
|
"provider": "forms_runtime",
|
||||||
|
"external_id": grant.id,
|
||||||
|
"revision": grant.form_definition_revision,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
conflict_strategy="rename",
|
||||||
|
is_admin=True,
|
||||||
|
)
|
||||||
|
grant.status = "uploaded"
|
||||||
|
grant.file_asset_id = stored.asset.id
|
||||||
|
grant.file_version_id = stored.version.id
|
||||||
|
grant.uploaded_at = now
|
||||||
|
session.add(grant)
|
||||||
|
evidence = EvidenceReference(
|
||||||
|
kind="document",
|
||||||
|
owner_module="files",
|
||||||
|
evidence_id=stored.asset.id,
|
||||||
|
tenant_id=grant.tenant_id,
|
||||||
|
version=stored.version.id,
|
||||||
|
checksum=stored.version.checksum_sha256,
|
||||||
|
source_ref=(f"form_submission:{grant.form_instance_id}:grant:{grant.id}"),
|
||||||
|
responsible_actor_ref=f"user:{grant.custodian_user_id}",
|
||||||
|
captured_at=now,
|
||||||
|
)
|
||||||
|
emit_platform_event(
|
||||||
|
session,
|
||||||
|
PlatformEvent(
|
||||||
|
type="files.form_evidence.uploaded",
|
||||||
|
module_id="files",
|
||||||
|
payload={
|
||||||
|
"grant_id": grant.id,
|
||||||
|
"form_instance_id": grant.form_instance_id,
|
||||||
|
"file_asset_id": stored.asset.id,
|
||||||
|
"file_version_id": stored.version.id,
|
||||||
|
"checksum_sha256": stored.version.checksum_sha256,
|
||||||
|
"size_bytes": stored.version.size_bytes,
|
||||||
|
},
|
||||||
|
occurred_at=now,
|
||||||
|
actor=EventActorRef(type="user", id=grant.custodian_user_id),
|
||||||
|
tenant=EventTenantRef(id=grant.tenant_id),
|
||||||
|
resource=EventObjectRef(type="file", id=stored.asset.id),
|
||||||
|
classification="confidential",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
except HTTPException:
|
||||||
|
session.rollback()
|
||||||
|
raise
|
||||||
|
except (FileStorageError, ValueError) as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
|
||||||
|
detail=str(exc),
|
||||||
|
) from exc
|
||||||
|
return {
|
||||||
|
"grant_id": grant.id,
|
||||||
|
"evidence": evidence.to_dict(include_inspection=True),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _token_sha256(token: str) -> str:
|
||||||
|
clean = str(token or "").strip()
|
||||||
|
if len(clean) < 32:
|
||||||
|
raise _upload_unavailable()
|
||||||
|
return hashlib.sha256(clean.encode("utf-8")).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def _upload_unavailable() -> HTTPException:
|
||||||
|
return HTTPException(
|
||||||
|
status_code=status.HTTP_404_NOT_FOUND,
|
||||||
|
detail="The Form evidence upload is unavailable.",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _aware(value: datetime) -> datetime:
|
||||||
|
return value if value.tzinfo is not None else value.replace(tzinfo=UTC)
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["router", "upload_form_evidence"]
|
||||||
@@ -0,0 +1,387 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Depends, HTTPException, Query, status
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from govoplan_core.auth import ApiPrincipal, require_scope
|
||||||
|
from govoplan_core.audit.logging import audit_from_principal
|
||||||
|
from govoplan_core.db.session import get_session
|
||||||
|
from govoplan_files.backend.db.models import (
|
||||||
|
FileIntegrityFinding,
|
||||||
|
FileIntegrityScan,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.schemas import (
|
||||||
|
FileIntegrityActionRequest,
|
||||||
|
FileIntegrityActionResponse,
|
||||||
|
FileIntegrityFindingResponse,
|
||||||
|
FileIntegrityFindingsResponse,
|
||||||
|
FileIntegrityScanCreateRequest,
|
||||||
|
FileIntegrityScanRunRequest,
|
||||||
|
FileIntegrityScanResponse,
|
||||||
|
FileIntegrityScansResponse,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.storage.backends import StorageBackendError
|
||||||
|
from govoplan_files.backend.storage.common import FileStorageError
|
||||||
|
from govoplan_files.backend.storage.integrity import (
|
||||||
|
cleanup_orphan_finding,
|
||||||
|
create_integrity_scan,
|
||||||
|
mark_integrity_scan_failed,
|
||||||
|
recheck_integrity_finding,
|
||||||
|
run_integrity_scan_batch,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/files/integrity", tags=["files-integrity"])
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/scans", response_model=FileIntegrityScansResponse)
|
||||||
|
def list_integrity_scans(
|
||||||
|
limit: int = Query(default=50, ge=1, le=200),
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(require_scope("files:file:admin")),
|
||||||
|
) -> FileIntegrityScansResponse:
|
||||||
|
rows = (
|
||||||
|
session.query(FileIntegrityScan)
|
||||||
|
.filter(FileIntegrityScan.tenant_id == principal.tenant_id)
|
||||||
|
.order_by(FileIntegrityScan.created_at.desc(), FileIntegrityScan.id.desc())
|
||||||
|
.limit(limit)
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
return FileIntegrityScansResponse(
|
||||||
|
scans=[_scan_response(row) for row in rows]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post(
|
||||||
|
"/scans",
|
||||||
|
response_model=FileIntegrityScanResponse,
|
||||||
|
status_code=status.HTTP_201_CREATED,
|
||||||
|
)
|
||||||
|
def create_scan(
|
||||||
|
payload: FileIntegrityScanCreateRequest,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(require_scope("files:file:admin")),
|
||||||
|
) -> FileIntegrityScanResponse:
|
||||||
|
try:
|
||||||
|
scan = create_integrity_scan(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
verify_checksums=payload.verify_checksums,
|
||||||
|
batch_size=payload.batch_size,
|
||||||
|
)
|
||||||
|
audit_from_principal(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
action="files.integrity.scan_created",
|
||||||
|
object_type="file_integrity_scan",
|
||||||
|
object_id=scan.id,
|
||||||
|
details={
|
||||||
|
"storage_backend": scan.storage_backend,
|
||||||
|
"verify_checksums": scan.verify_checksums,
|
||||||
|
"batch_size": scan.batch_size,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
return _scan_response(scan)
|
||||||
|
except (FileStorageError, StorageBackendError) as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _integrity_http_error(exc) from exc
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/scans/{scan_id}/run", response_model=FileIntegrityScanResponse)
|
||||||
|
def run_scan_batch(
|
||||||
|
scan_id: str,
|
||||||
|
payload: FileIntegrityScanRunRequest,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(require_scope("files:file:admin")),
|
||||||
|
) -> FileIntegrityScanResponse:
|
||||||
|
scan = _scan_for_tenant(
|
||||||
|
session,
|
||||||
|
scan_id,
|
||||||
|
principal.tenant_id,
|
||||||
|
for_update=True,
|
||||||
|
)
|
||||||
|
_assert_expected_revision(scan.revision, payload.expected_revision)
|
||||||
|
previous_status = scan.status
|
||||||
|
try:
|
||||||
|
run_integrity_scan_batch(session, scan)
|
||||||
|
if scan.status == "completed" and previous_status != "completed":
|
||||||
|
audit_from_principal(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
action="files.integrity.scan_completed",
|
||||||
|
object_type="file_integrity_scan",
|
||||||
|
object_id=scan.id,
|
||||||
|
details={
|
||||||
|
"verified_blob_count": scan.verified_blob_count,
|
||||||
|
"quarantined_blob_count": scan.quarantined_blob_count,
|
||||||
|
"orphan_object_count": scan.orphan_object_count,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
return _scan_response(scan)
|
||||||
|
except (FileStorageError, StorageBackendError) as exc:
|
||||||
|
session.rollback()
|
||||||
|
scan = _scan_for_tenant(
|
||||||
|
session,
|
||||||
|
scan_id,
|
||||||
|
principal.tenant_id,
|
||||||
|
for_update=True,
|
||||||
|
)
|
||||||
|
mark_integrity_scan_failed(scan, error=exc)
|
||||||
|
audit_from_principal(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
action="files.integrity.scan_failed",
|
||||||
|
object_type="file_integrity_scan",
|
||||||
|
object_id=scan.id,
|
||||||
|
details={"error_type": type(exc).__name__},
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
raise _integrity_http_error(exc) from exc
|
||||||
|
|
||||||
|
|
||||||
|
@router.get(
|
||||||
|
"/scans/{scan_id}/findings",
|
||||||
|
response_model=FileIntegrityFindingsResponse,
|
||||||
|
)
|
||||||
|
def list_integrity_findings(
|
||||||
|
scan_id: str,
|
||||||
|
state_filter: str | None = Query(default=None, alias="state"),
|
||||||
|
limit: int = Query(default=500, ge=1, le=1000),
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(require_scope("files:file:admin")),
|
||||||
|
) -> FileIntegrityFindingsResponse:
|
||||||
|
scan = _scan_for_tenant(session, scan_id, principal.tenant_id)
|
||||||
|
query = session.query(FileIntegrityFinding).filter(
|
||||||
|
FileIntegrityFinding.scan_id == scan.id,
|
||||||
|
FileIntegrityFinding.tenant_id == principal.tenant_id,
|
||||||
|
)
|
||||||
|
if state_filter:
|
||||||
|
query = query.filter(FileIntegrityFinding.state == state_filter)
|
||||||
|
rows = (
|
||||||
|
query.order_by(
|
||||||
|
FileIntegrityFinding.created_at.asc(),
|
||||||
|
FileIntegrityFinding.id.asc(),
|
||||||
|
)
|
||||||
|
.limit(limit)
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
return FileIntegrityFindingsResponse(
|
||||||
|
findings=[_finding_response(row) for row in rows]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post(
|
||||||
|
"/findings/{finding_id}/recheck",
|
||||||
|
response_model=FileIntegrityActionResponse,
|
||||||
|
)
|
||||||
|
def recheck_finding(
|
||||||
|
finding_id: str,
|
||||||
|
payload: FileIntegrityActionRequest,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(require_scope("files:file:admin")),
|
||||||
|
) -> FileIntegrityActionResponse:
|
||||||
|
finding = _finding_for_tenant(
|
||||||
|
session,
|
||||||
|
finding_id,
|
||||||
|
principal.tenant_id,
|
||||||
|
for_update=True,
|
||||||
|
)
|
||||||
|
_assert_expected_revision(finding.revision, payload.expected_revision)
|
||||||
|
try:
|
||||||
|
result = recheck_integrity_finding(
|
||||||
|
session,
|
||||||
|
finding,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
dry_run=payload.dry_run,
|
||||||
|
)
|
||||||
|
_audit_integrity_action(session, principal, result)
|
||||||
|
session.commit()
|
||||||
|
return _action_response(result)
|
||||||
|
except (FileStorageError, StorageBackendError) as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _integrity_http_error(exc) from exc
|
||||||
|
|
||||||
|
|
||||||
|
@router.post(
|
||||||
|
"/findings/{finding_id}/cleanup",
|
||||||
|
response_model=FileIntegrityActionResponse,
|
||||||
|
)
|
||||||
|
def cleanup_finding(
|
||||||
|
finding_id: str,
|
||||||
|
payload: FileIntegrityActionRequest,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(require_scope("files:file:admin")),
|
||||||
|
) -> FileIntegrityActionResponse:
|
||||||
|
finding = _finding_for_tenant(
|
||||||
|
session,
|
||||||
|
finding_id,
|
||||||
|
principal.tenant_id,
|
||||||
|
for_update=True,
|
||||||
|
)
|
||||||
|
_assert_expected_revision(finding.revision, payload.expected_revision)
|
||||||
|
try:
|
||||||
|
result = cleanup_orphan_finding(
|
||||||
|
session,
|
||||||
|
finding,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
dry_run=payload.dry_run,
|
||||||
|
)
|
||||||
|
_audit_integrity_action(session, principal, result)
|
||||||
|
session.commit()
|
||||||
|
return _action_response(result)
|
||||||
|
except (FileStorageError, StorageBackendError) as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _integrity_http_error(exc) from exc
|
||||||
|
|
||||||
|
|
||||||
|
def _scan_for_tenant(
|
||||||
|
session: Session,
|
||||||
|
scan_id: str,
|
||||||
|
tenant_id: str,
|
||||||
|
*,
|
||||||
|
for_update: bool = False,
|
||||||
|
) -> FileIntegrityScan:
|
||||||
|
query = session.query(FileIntegrityScan).filter(FileIntegrityScan.id == scan_id)
|
||||||
|
if for_update:
|
||||||
|
query = query.populate_existing().with_for_update()
|
||||||
|
scan = query.one_or_none()
|
||||||
|
if scan is None or scan.tenant_id != tenant_id:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_404_NOT_FOUND,
|
||||||
|
detail="Integrity scan not found",
|
||||||
|
)
|
||||||
|
return scan
|
||||||
|
|
||||||
|
|
||||||
|
def _finding_for_tenant(
|
||||||
|
session: Session,
|
||||||
|
finding_id: str,
|
||||||
|
tenant_id: str,
|
||||||
|
*,
|
||||||
|
for_update: bool = False,
|
||||||
|
) -> FileIntegrityFinding:
|
||||||
|
query = session.query(FileIntegrityFinding).filter(
|
||||||
|
FileIntegrityFinding.id == finding_id
|
||||||
|
)
|
||||||
|
if for_update:
|
||||||
|
query = query.populate_existing().with_for_update()
|
||||||
|
finding = query.one_or_none()
|
||||||
|
if finding is None or finding.tenant_id != tenant_id:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_404_NOT_FOUND,
|
||||||
|
detail="Integrity finding not found",
|
||||||
|
)
|
||||||
|
return finding
|
||||||
|
|
||||||
|
|
||||||
|
def _assert_expected_revision(current: int, expected: int) -> None:
|
||||||
|
if current != expected:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_409_CONFLICT,
|
||||||
|
detail=(
|
||||||
|
"The integrity record changed after it was loaded; reload before "
|
||||||
|
"performing this action."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _audit_integrity_action(session, principal, result) -> None:
|
||||||
|
audit_from_principal(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
action=f"files.integrity.{result.action}",
|
||||||
|
object_type="file_integrity_finding",
|
||||||
|
object_id=result.finding.id,
|
||||||
|
details={
|
||||||
|
"scan_id": result.finding.scan_id,
|
||||||
|
"kind": result.finding.kind,
|
||||||
|
"dry_run": result.dry_run,
|
||||||
|
"changed": result.changed,
|
||||||
|
"storage_key_sha256": hashlib.sha256(
|
||||||
|
result.finding.storage_key.encode("utf-8")
|
||||||
|
).hexdigest(),
|
||||||
|
"inspection": result.inspection.kind
|
||||||
|
if result.inspection
|
||||||
|
else None,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _scan_response(scan: FileIntegrityScan) -> FileIntegrityScanResponse:
|
||||||
|
return FileIntegrityScanResponse(
|
||||||
|
id=scan.id,
|
||||||
|
tenant_id=scan.tenant_id,
|
||||||
|
storage_backend=scan.storage_backend,
|
||||||
|
storage_prefix=scan.storage_prefix,
|
||||||
|
status=scan.status,
|
||||||
|
revision=scan.revision,
|
||||||
|
phase=scan.phase,
|
||||||
|
verify_checksums=scan.verify_checksums,
|
||||||
|
batch_size=scan.batch_size,
|
||||||
|
scanned_blob_count=scan.scanned_blob_count,
|
||||||
|
verified_blob_count=scan.verified_blob_count,
|
||||||
|
quarantined_blob_count=scan.quarantined_blob_count,
|
||||||
|
scanned_object_count=scan.scanned_object_count,
|
||||||
|
orphan_object_count=scan.orphan_object_count,
|
||||||
|
created_by_user_id=scan.created_by_user_id,
|
||||||
|
started_at=scan.started_at.isoformat() if scan.started_at else None,
|
||||||
|
completed_at=scan.completed_at.isoformat()
|
||||||
|
if scan.completed_at
|
||||||
|
else None,
|
||||||
|
last_error=scan.last_error,
|
||||||
|
created_at=scan.created_at.isoformat(),
|
||||||
|
updated_at=scan.updated_at.isoformat(),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _finding_response(
|
||||||
|
finding: FileIntegrityFinding,
|
||||||
|
) -> FileIntegrityFindingResponse:
|
||||||
|
return FileIntegrityFindingResponse(
|
||||||
|
id=finding.id,
|
||||||
|
scan_id=finding.scan_id,
|
||||||
|
tenant_id=finding.tenant_id,
|
||||||
|
kind=finding.kind,
|
||||||
|
state=finding.state,
|
||||||
|
revision=finding.revision,
|
||||||
|
blob_id=finding.blob_id,
|
||||||
|
storage_key=finding.storage_key,
|
||||||
|
expected_size_bytes=finding.expected_size_bytes,
|
||||||
|
observed_size_bytes=finding.observed_size_bytes,
|
||||||
|
expected_checksum_sha256=finding.expected_checksum_sha256,
|
||||||
|
observed_checksum_sha256=finding.observed_checksum_sha256,
|
||||||
|
resolved_at=finding.resolved_at.isoformat()
|
||||||
|
if finding.resolved_at
|
||||||
|
else None,
|
||||||
|
resolved_by_user_id=finding.resolved_by_user_id,
|
||||||
|
created_at=finding.created_at.isoformat(),
|
||||||
|
updated_at=finding.updated_at.isoformat(),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _action_response(result) -> FileIntegrityActionResponse:
|
||||||
|
return FileIntegrityActionResponse(
|
||||||
|
action=result.action,
|
||||||
|
changed=result.changed,
|
||||||
|
dry_run=result.dry_run,
|
||||||
|
finding=_finding_response(result.finding),
|
||||||
|
inspection_kind=result.inspection.kind if result.inspection else None,
|
||||||
|
inspection_valid=result.inspection.valid if result.inspection else None,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _integrity_http_error(exc: Exception) -> HTTPException:
|
||||||
|
if isinstance(exc, FileStorageError):
|
||||||
|
return HTTPException(
|
||||||
|
status_code=status.HTTP_409_CONFLICT,
|
||||||
|
detail=str(exc),
|
||||||
|
)
|
||||||
|
return HTTPException(
|
||||||
|
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||||
|
detail="The configured file storage backend could not complete the integrity operation",
|
||||||
|
)
|
||||||
@@ -0,0 +1,297 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Depends
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from govoplan_core.auth import ApiPrincipal, require_scope
|
||||||
|
from govoplan_core.audit.logging import audit_from_principal
|
||||||
|
from govoplan_core.db.session import get_session
|
||||||
|
from govoplan_files.backend.route_support import (
|
||||||
|
_asset_response,
|
||||||
|
_connector_space_response,
|
||||||
|
_http_error,
|
||||||
|
_is_admin,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.schemas import (
|
||||||
|
FileAssetResponse,
|
||||||
|
FileBlobGcRequest,
|
||||||
|
FileBlobGcResponse,
|
||||||
|
FileConnectorSpaceResponse,
|
||||||
|
FileFolderDeleteRequest,
|
||||||
|
FileFolderRestoreResponse,
|
||||||
|
FileLifecycleUpdateRequest,
|
||||||
|
FilePurgeExecuteRequest,
|
||||||
|
FilePurgePreviewItem,
|
||||||
|
FilePurgePreviewRequest,
|
||||||
|
FilePurgePreviewResponse,
|
||||||
|
FilePurgeResponse,
|
||||||
|
FileRestoreResponse,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.storage.common import FileStorageError
|
||||||
|
from govoplan_files.backend.storage.lifecycle import (
|
||||||
|
execute_asset_purge,
|
||||||
|
garbage_collect_unreferenced_blobs,
|
||||||
|
get_asset_for_lifecycle,
|
||||||
|
preview_asset_purge,
|
||||||
|
restore_asset,
|
||||||
|
restore_connector_space,
|
||||||
|
restore_folder,
|
||||||
|
set_asset_lifecycle,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/files", tags=["files-lifecycle"])
|
||||||
|
|
||||||
|
|
||||||
|
@router.patch("/{file_id}/lifecycle", response_model=FileAssetResponse)
|
||||||
|
def update_file_lifecycle(
|
||||||
|
file_id: str,
|
||||||
|
payload: FileLifecycleUpdateRequest,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(require_scope("files:file:retention")),
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
asset = get_asset_for_lifecycle(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
asset_id=file_id,
|
||||||
|
is_admin=_is_admin(principal),
|
||||||
|
)
|
||||||
|
set_asset_lifecycle(
|
||||||
|
session,
|
||||||
|
asset,
|
||||||
|
retained_until=payload.retained_until,
|
||||||
|
legal_hold=payload.legal_hold,
|
||||||
|
reason=payload.reason,
|
||||||
|
expected_revision=payload.expected_revision,
|
||||||
|
)
|
||||||
|
audit_from_principal(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
action="files.lifecycle.updated",
|
||||||
|
object_type="file",
|
||||||
|
object_id=asset.id,
|
||||||
|
details={
|
||||||
|
"retained_until": (
|
||||||
|
asset.retained_until.isoformat() if asset.retained_until else None
|
||||||
|
),
|
||||||
|
"legal_hold": asset.legal_hold,
|
||||||
|
"lifecycle_revision": asset.lifecycle_revision,
|
||||||
|
"reason": asset.lifecycle_reason,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
return _asset_response(session, asset, include_shares=True)
|
||||||
|
except FileStorageError as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _http_error(exc, not_found=True) from exc
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/assets/{file_id}/restore", response_model=FileRestoreResponse)
|
||||||
|
def restore_file(
|
||||||
|
file_id: str,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(require_scope("files:file:restore")),
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
asset = get_asset_for_lifecycle(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
asset_id=file_id,
|
||||||
|
is_admin=_is_admin(principal),
|
||||||
|
)
|
||||||
|
changed = restore_asset(session, asset)
|
||||||
|
if changed:
|
||||||
|
audit_from_principal(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
action="files.file.restored",
|
||||||
|
object_type="file",
|
||||||
|
object_id=asset.id,
|
||||||
|
details={"display_path": asset.display_path},
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
return FileRestoreResponse(restored_count=1 if changed else 0)
|
||||||
|
except FileStorageError as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _http_error(exc, not_found=True) from exc
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/folders/restore", response_model=FileFolderRestoreResponse)
|
||||||
|
def restore_file_folder(
|
||||||
|
payload: FileFolderDeleteRequest,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(require_scope("files:file:restore")),
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
restored_folders, restored_files = restore_folder(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
owner_type=payload.owner_type,
|
||||||
|
owner_id=payload.owner_id,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
path=payload.path,
|
||||||
|
recursive=payload.recursive,
|
||||||
|
is_admin=_is_admin(principal),
|
||||||
|
)
|
||||||
|
audit_from_principal(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
action="files.folder.restored",
|
||||||
|
object_type="file_folder",
|
||||||
|
object_id=payload.path,
|
||||||
|
details={
|
||||||
|
"restored_folders": restored_folders,
|
||||||
|
"restored_files": restored_files,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
return FileFolderRestoreResponse(
|
||||||
|
restored_folders=restored_folders, restored_files=restored_files
|
||||||
|
)
|
||||||
|
except FileStorageError as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _http_error(exc, not_found=True) from exc
|
||||||
|
|
||||||
|
|
||||||
|
@router.post(
|
||||||
|
"/connector-spaces/{space_id}/restore",
|
||||||
|
response_model=FileConnectorSpaceResponse,
|
||||||
|
)
|
||||||
|
def restore_file_connector_space(
|
||||||
|
space_id: str,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(require_scope("files:file:restore")),
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
space = restore_connector_space(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
space_id=space_id,
|
||||||
|
is_admin=_is_admin(principal),
|
||||||
|
)
|
||||||
|
audit_from_principal(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
action="files.connector_space.restored",
|
||||||
|
object_type="file_connector_space",
|
||||||
|
object_id=space.id,
|
||||||
|
details={"label": space.label},
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
return _connector_space_response(space)
|
||||||
|
except FileStorageError as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _http_error(exc, not_found=True) from exc
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/purge/preview", response_model=FilePurgePreviewResponse)
|
||||||
|
def preview_file_purge(
|
||||||
|
payload: FilePurgePreviewRequest,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(require_scope("files:file:purge")),
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
preview = preview_asset_purge(
|
||||||
|
session, tenant_id=principal.tenant_id, file_ids=payload.file_ids
|
||||||
|
)
|
||||||
|
return FilePurgePreviewResponse(
|
||||||
|
preview_sha256=preview.preview_sha256,
|
||||||
|
eligible=preview.eligible,
|
||||||
|
items=[
|
||||||
|
FilePurgePreviewItem(
|
||||||
|
file_id=item.file_id,
|
||||||
|
filename=item.filename,
|
||||||
|
lifecycle_revision=item.lifecycle_revision,
|
||||||
|
deleted_at=(item.deleted_at.isoformat() if item.deleted_at else None),
|
||||||
|
retained_until=(
|
||||||
|
item.retained_until.isoformat() if item.retained_until else None
|
||||||
|
),
|
||||||
|
legal_hold=item.legal_hold,
|
||||||
|
blockers=list(item.blockers),
|
||||||
|
blob_ids=list(item.blob_ids),
|
||||||
|
)
|
||||||
|
for item in preview.items
|
||||||
|
],
|
||||||
|
)
|
||||||
|
except FileStorageError as exc:
|
||||||
|
raise _http_error(exc) from exc
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/purge/execute", response_model=FilePurgeResponse)
|
||||||
|
def execute_file_purge(
|
||||||
|
payload: FilePurgeExecuteRequest,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(require_scope("files:file:purge")),
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
result = execute_asset_purge(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
file_ids=payload.file_ids,
|
||||||
|
preview_sha256=payload.preview_sha256,
|
||||||
|
idempotency_key=payload.idempotency_key,
|
||||||
|
approval_reference=payload.approval_reference,
|
||||||
|
before_commit=lambda assets: audit_from_principal(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
action="files.files.purged",
|
||||||
|
object_type="file_batch",
|
||||||
|
object_id=payload.preview_sha256,
|
||||||
|
details={
|
||||||
|
"file_ids": [asset.id for asset in assets],
|
||||||
|
"approval_reference": payload.approval_reference,
|
||||||
|
"preview_sha256": payload.preview_sha256,
|
||||||
|
},
|
||||||
|
),
|
||||||
|
)
|
||||||
|
return FilePurgeResponse(
|
||||||
|
recovery_operation_id=result.recovery_operation_id,
|
||||||
|
status=result.status,
|
||||||
|
replayed=result.replayed,
|
||||||
|
purged_files=result.purged_files,
|
||||||
|
released_blobs=result.released_blobs,
|
||||||
|
)
|
||||||
|
except FileStorageError as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _http_error(exc) from exc
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/purge/blobs", response_model=FileBlobGcResponse)
|
||||||
|
def garbage_collect_file_blobs(
|
||||||
|
payload: FileBlobGcRequest,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(require_scope("files:file:purge")),
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
result = garbage_collect_unreferenced_blobs(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
limit=payload.limit,
|
||||||
|
approval_reference=payload.approval_reference,
|
||||||
|
)
|
||||||
|
audit_from_principal(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
action="files.blobs.garbage_collected",
|
||||||
|
object_type="file_blob_batch",
|
||||||
|
object_id=principal.tenant_id,
|
||||||
|
details={
|
||||||
|
"inspected_blobs": result.inspected_blobs,
|
||||||
|
"deleted_blobs": result.deleted_blobs,
|
||||||
|
"unresolved_operation_ids": list(result.unresolved_operation_ids),
|
||||||
|
"approval_reference": payload.approval_reference,
|
||||||
|
},
|
||||||
|
commit=True,
|
||||||
|
)
|
||||||
|
return FileBlobGcResponse(
|
||||||
|
inspected_blobs=result.inspected_blobs,
|
||||||
|
deleted_blobs=result.deleted_blobs,
|
||||||
|
unresolved_operation_ids=list(result.unresolved_operation_ids),
|
||||||
|
)
|
||||||
|
except FileStorageError as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _http_error(exc) from exc
|
||||||
@@ -0,0 +1,194 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import Literal
|
||||||
|
from fastapi import APIRouter, Depends, HTTPException, Query, status
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from govoplan_core.auth import ApiPrincipal, require_scope
|
||||||
|
from govoplan_files.backend.schemas import (
|
||||||
|
FileDeltaResponse,
|
||||||
|
FileListResponse,
|
||||||
|
)
|
||||||
|
from govoplan_core.db.session import get_session
|
||||||
|
from govoplan_files.backend.storage.files import (
|
||||||
|
count_assets_for_user,
|
||||||
|
list_assets_for_user,
|
||||||
|
list_assets_for_user_window,
|
||||||
|
list_recent_assets_for_user,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
from govoplan_files.backend.route_support import (
|
||||||
|
_asset_list_response,
|
||||||
|
_ensure_campaign_file_access,
|
||||||
|
_ensure_list_owner_access,
|
||||||
|
_is_admin,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.services.list_queries import (
|
||||||
|
FILES_LIST_CURSOR_SCOPE,
|
||||||
|
_cursor_page_size,
|
||||||
|
_file_cursor_values,
|
||||||
|
_files_delta_response,
|
||||||
|
_files_delta_watermark,
|
||||||
|
_files_list_fingerprint,
|
||||||
|
_full_file_delta_response,
|
||||||
|
_next_file_list_cursor,
|
||||||
|
)
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/files", tags=["files"])
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/delta", response_model=FileDeltaResponse)
|
||||||
|
def files_delta(
|
||||||
|
owner_type: Literal["user", "group"] | None = None,
|
||||||
|
owner_id: str | None = None,
|
||||||
|
campaign_id: str | None = None,
|
||||||
|
path_prefix: str | None = None,
|
||||||
|
since: str | None = None,
|
||||||
|
limit: int = Query(default=500, ge=1, le=1000),
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(require_scope("files:file:read")),
|
||||||
|
):
|
||||||
|
_ensure_list_owner_access(session, principal, owner_type, owner_id)
|
||||||
|
_ensure_campaign_file_access(session, principal, campaign_id)
|
||||||
|
if since is None:
|
||||||
|
return _full_file_delta_response(
|
||||||
|
session,
|
||||||
|
principal=principal,
|
||||||
|
owner_type=owner_type,
|
||||||
|
owner_id=owner_id,
|
||||||
|
campaign_id=campaign_id,
|
||||||
|
path_prefix=path_prefix,
|
||||||
|
)
|
||||||
|
return _files_delta_response(
|
||||||
|
session,
|
||||||
|
principal=principal,
|
||||||
|
owner_type=owner_type,
|
||||||
|
owner_id=owner_id,
|
||||||
|
campaign_id=campaign_id,
|
||||||
|
path_prefix=path_prefix,
|
||||||
|
since=since,
|
||||||
|
limit=limit,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("", response_model=FileListResponse)
|
||||||
|
def list_files(
|
||||||
|
owner_type: Literal["user", "group"] | None = None,
|
||||||
|
owner_id: str | None = None,
|
||||||
|
campaign_id: str | None = None,
|
||||||
|
path_prefix: str | None = None,
|
||||||
|
campaign_usage: Literal["linked", "unlinked"] | None = None,
|
||||||
|
audit_relevant: bool | None = None,
|
||||||
|
sort: Literal["path", "recent"] = "path",
|
||||||
|
page_size: int | None = Query(default=None, ge=1, le=1000),
|
||||||
|
cursor: str | None = None,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(require_scope("files:file:read")),
|
||||||
|
):
|
||||||
|
_ensure_list_owner_access(session, principal, owner_type, owner_id)
|
||||||
|
_ensure_campaign_file_access(session, principal, campaign_id)
|
||||||
|
watermark = _files_delta_watermark(session, principal.tenant_id)
|
||||||
|
total = count_assets_for_user(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
owner_type=owner_type,
|
||||||
|
owner_id=owner_id,
|
||||||
|
campaign_id=campaign_id,
|
||||||
|
path_prefix=path_prefix,
|
||||||
|
campaign_usage=campaign_usage,
|
||||||
|
audit_relevant=audit_relevant,
|
||||||
|
is_admin=_is_admin(principal),
|
||||||
|
)
|
||||||
|
if sort == "recent":
|
||||||
|
if cursor:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
|
||||||
|
detail="Recent file projections do not accept a path-order cursor.",
|
||||||
|
)
|
||||||
|
recent_limit = page_size or 25
|
||||||
|
assets = list_recent_assets_for_user(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
limit=recent_limit,
|
||||||
|
owner_type=owner_type,
|
||||||
|
owner_id=owner_id,
|
||||||
|
campaign_id=campaign_id,
|
||||||
|
path_prefix=path_prefix,
|
||||||
|
campaign_usage=campaign_usage,
|
||||||
|
audit_relevant=audit_relevant,
|
||||||
|
is_admin=_is_admin(principal),
|
||||||
|
)
|
||||||
|
return FileListResponse(
|
||||||
|
files=_asset_list_response(session, assets, include_shares=True),
|
||||||
|
total=total,
|
||||||
|
watermark=watermark,
|
||||||
|
)
|
||||||
|
effective_page_size = _cursor_page_size(FILES_LIST_CURSOR_SCOPE, cursor, page_size)
|
||||||
|
if effective_page_size is not None:
|
||||||
|
fingerprint = _files_list_fingerprint(
|
||||||
|
principal,
|
||||||
|
owner_type=owner_type,
|
||||||
|
owner_id=owner_id,
|
||||||
|
campaign_id=campaign_id,
|
||||||
|
path_prefix=path_prefix,
|
||||||
|
campaign_usage=campaign_usage,
|
||||||
|
audit_relevant=audit_relevant,
|
||||||
|
page_size=effective_page_size,
|
||||||
|
)
|
||||||
|
after_display_path, after_updated_at, after_id = _file_cursor_values(
|
||||||
|
cursor, fingerprint=fingerprint
|
||||||
|
)
|
||||||
|
assets, has_more = list_assets_for_user_window(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
owner_type=owner_type,
|
||||||
|
owner_id=owner_id,
|
||||||
|
campaign_id=campaign_id,
|
||||||
|
path_prefix=path_prefix,
|
||||||
|
campaign_usage=campaign_usage,
|
||||||
|
audit_relevant=audit_relevant,
|
||||||
|
is_admin=_is_admin(principal),
|
||||||
|
page_size=effective_page_size,
|
||||||
|
after_display_path=after_display_path,
|
||||||
|
after_updated_at=after_updated_at,
|
||||||
|
after_id=after_id,
|
||||||
|
)
|
||||||
|
return FileListResponse(
|
||||||
|
files=_asset_list_response(session, assets, include_shares=True),
|
||||||
|
total=total,
|
||||||
|
cursor=cursor,
|
||||||
|
next_cursor=_next_file_list_cursor(
|
||||||
|
principal,
|
||||||
|
assets,
|
||||||
|
owner_type=owner_type,
|
||||||
|
owner_id=owner_id,
|
||||||
|
campaign_id=campaign_id,
|
||||||
|
path_prefix=path_prefix,
|
||||||
|
campaign_usage=campaign_usage,
|
||||||
|
audit_relevant=audit_relevant,
|
||||||
|
page_size=effective_page_size,
|
||||||
|
has_more=has_more,
|
||||||
|
),
|
||||||
|
watermark=watermark,
|
||||||
|
)
|
||||||
|
assets = list_assets_for_user(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
owner_type=owner_type,
|
||||||
|
owner_id=owner_id,
|
||||||
|
campaign_id=campaign_id,
|
||||||
|
path_prefix=path_prefix,
|
||||||
|
campaign_usage=campaign_usage,
|
||||||
|
audit_relevant=audit_relevant,
|
||||||
|
is_admin=_is_admin(principal),
|
||||||
|
)
|
||||||
|
return FileListResponse(
|
||||||
|
files=_asset_list_response(session, assets, include_shares=True),
|
||||||
|
total=total,
|
||||||
|
watermark=watermark,
|
||||||
|
)
|
||||||
@@ -0,0 +1,330 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Depends, HTTPException, Query, status
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from govoplan_core.api.v1.schemas import (
|
||||||
|
ReferenceOptionListResponse,
|
||||||
|
ReferenceOptionResponse,
|
||||||
|
)
|
||||||
|
from govoplan_core.auth import ApiPrincipal, require_scope
|
||||||
|
from govoplan_core.audit.logging import audit_from_principal
|
||||||
|
from govoplan_core.core.references import (
|
||||||
|
access_scope_reference_page,
|
||||||
|
access_scope_reference_provider_available,
|
||||||
|
)
|
||||||
|
from govoplan_core.db.session import get_session
|
||||||
|
from govoplan_files.backend.runtime import get_registry
|
||||||
|
from govoplan_files.backend.schemas import (
|
||||||
|
BulkFileShareRequest,
|
||||||
|
BulkFileShareResponse,
|
||||||
|
FileShareRequest,
|
||||||
|
FileShareResponse,
|
||||||
|
FileSharesResponse,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.storage.common import FileStorageError
|
||||||
|
from govoplan_files.backend.storage.files import (
|
||||||
|
current_file_share_for_target,
|
||||||
|
get_asset_for_share_management,
|
||||||
|
list_file_shares,
|
||||||
|
revoke_file_share,
|
||||||
|
share_file,
|
||||||
|
share_files,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.route_support import (
|
||||||
|
_file_share_response,
|
||||||
|
_http_error,
|
||||||
|
_is_admin,
|
||||||
|
)
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/files", tags=["files"])
|
||||||
|
|
||||||
|
|
||||||
|
@router.get(
|
||||||
|
"/{file_id}/share-target-options",
|
||||||
|
response_model=ReferenceOptionListResponse,
|
||||||
|
)
|
||||||
|
def search_share_targets(
|
||||||
|
file_id: str,
|
||||||
|
target_type: str,
|
||||||
|
q: str = "",
|
||||||
|
selected: list[str] = Query(default=[]),
|
||||||
|
limit: int = Query(default=50, ge=1, le=200),
|
||||||
|
cursor: str | None = None,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(require_scope("files:file:share")),
|
||||||
|
) -> ReferenceOptionListResponse:
|
||||||
|
if target_type not in {"user", "group"}:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
|
||||||
|
detail="Share target type must be user or group",
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
get_asset_for_share_management(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
asset_id=file_id,
|
||||||
|
is_admin=_is_admin(principal),
|
||||||
|
)
|
||||||
|
registry = get_registry()
|
||||||
|
page = access_scope_reference_page(
|
||||||
|
registry,
|
||||||
|
principal,
|
||||||
|
scope_type=target_type,
|
||||||
|
reference_kind="membership" if target_type == "user" else "group",
|
||||||
|
query=q,
|
||||||
|
selected_values=selected,
|
||||||
|
limit=limit,
|
||||||
|
cursor=cursor,
|
||||||
|
administrative=True,
|
||||||
|
session=session,
|
||||||
|
)
|
||||||
|
except ValueError as exc:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
|
||||||
|
detail=str(exc),
|
||||||
|
) from exc
|
||||||
|
except FileStorageError as exc:
|
||||||
|
raise _http_error(exc) from exc
|
||||||
|
return ReferenceOptionListResponse(
|
||||||
|
options=[
|
||||||
|
ReferenceOptionResponse(**option.to_dict()) for option in page.options
|
||||||
|
],
|
||||||
|
provider_available=access_scope_reference_provider_available(registry),
|
||||||
|
next_cursor=page.next_cursor,
|
||||||
|
has_more=page.has_more,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/{file_id}/shares", response_model=FileSharesResponse)
|
||||||
|
def list_shares(
|
||||||
|
file_id: str,
|
||||||
|
include_inactive: bool = False,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(require_scope("files:file:share")),
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
asset = get_asset_for_share_management(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
asset_id=file_id,
|
||||||
|
is_admin=_is_admin(principal),
|
||||||
|
)
|
||||||
|
return FileSharesResponse(
|
||||||
|
shares=[
|
||||||
|
_file_share_response(share)
|
||||||
|
for share in list_file_shares(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
asset_id=asset.id,
|
||||||
|
include_inactive=include_inactive,
|
||||||
|
)
|
||||||
|
]
|
||||||
|
)
|
||||||
|
except FileStorageError as exc:
|
||||||
|
raise _http_error(exc) from exc
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/{file_id}/shares", response_model=FileShareResponse)
|
||||||
|
def create_share(
|
||||||
|
file_id: str,
|
||||||
|
payload: FileShareRequest,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(require_scope("files:file:share")),
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
asset = get_asset_for_share_management(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
asset_id=file_id,
|
||||||
|
is_admin=_is_admin(principal),
|
||||||
|
)
|
||||||
|
previous = current_file_share_for_target(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
asset_id=asset.id,
|
||||||
|
target_type=payload.target_type,
|
||||||
|
target_id=payload.target_id,
|
||||||
|
)
|
||||||
|
previous_permission = previous.permission if previous else None
|
||||||
|
previous_expiry = previous.expires_at if previous else None
|
||||||
|
share = share_file(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
asset=asset,
|
||||||
|
target_type=payload.target_type,
|
||||||
|
target_id=payload.target_id,
|
||||||
|
permission=payload.permission,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
expires_at=payload.expires_at,
|
||||||
|
)
|
||||||
|
session.flush()
|
||||||
|
action = _share_audit_action(
|
||||||
|
existed=previous is not None,
|
||||||
|
previous_permission=previous_permission,
|
||||||
|
permission=share.permission,
|
||||||
|
previous_expiry=previous_expiry,
|
||||||
|
expiry=share.expires_at,
|
||||||
|
)
|
||||||
|
if action:
|
||||||
|
_audit_share_change(session, principal, share, action=action)
|
||||||
|
session.commit()
|
||||||
|
return _file_share_response(share)
|
||||||
|
except FileStorageError as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _http_error(exc) from exc
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/bulk-shares", response_model=BulkFileShareResponse)
|
||||||
|
def create_bulk_shares(
|
||||||
|
payload: BulkFileShareRequest,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(require_scope("files:file:share")),
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
file_ids = list(dict.fromkeys(payload.file_ids))
|
||||||
|
assets = [
|
||||||
|
get_asset_for_share_management(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
asset_id=file_id,
|
||||||
|
is_admin=_is_admin(principal),
|
||||||
|
)
|
||||||
|
for file_id in file_ids
|
||||||
|
]
|
||||||
|
previous_by_asset = {
|
||||||
|
asset.id: current_file_share_for_target(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
asset_id=asset.id,
|
||||||
|
target_type=payload.target_type,
|
||||||
|
target_id=payload.target_id,
|
||||||
|
)
|
||||||
|
for asset in assets
|
||||||
|
}
|
||||||
|
previous_values = {
|
||||||
|
asset_id: (
|
||||||
|
share.permission if share else None,
|
||||||
|
share.expires_at if share else None,
|
||||||
|
)
|
||||||
|
for asset_id, share in previous_by_asset.items()
|
||||||
|
}
|
||||||
|
shares = share_files(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
assets=assets,
|
||||||
|
target_type=payload.target_type,
|
||||||
|
target_id=payload.target_id,
|
||||||
|
permission=payload.permission,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
expires_at=payload.expires_at,
|
||||||
|
)
|
||||||
|
session.flush()
|
||||||
|
for share in shares:
|
||||||
|
previous_permission, previous_expiry = previous_values[share.file_asset_id]
|
||||||
|
action = _share_audit_action(
|
||||||
|
existed=previous_by_asset[share.file_asset_id] is not None,
|
||||||
|
previous_permission=previous_permission,
|
||||||
|
permission=share.permission,
|
||||||
|
previous_expiry=previous_expiry,
|
||||||
|
expiry=share.expires_at,
|
||||||
|
)
|
||||||
|
if action:
|
||||||
|
_audit_share_change(session, principal, share, action=action)
|
||||||
|
session.commit()
|
||||||
|
return BulkFileShareResponse(
|
||||||
|
shared_count=len(shares),
|
||||||
|
shares=[_file_share_response(share) for share in shares],
|
||||||
|
)
|
||||||
|
except FileStorageError as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _http_error(exc) from exc
|
||||||
|
|
||||||
|
|
||||||
|
@router.delete("/{file_id}/shares/{share_id}", response_model=FileShareResponse)
|
||||||
|
def revoke_share(
|
||||||
|
file_id: str,
|
||||||
|
share_id: str,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(require_scope("files:file:share")),
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
asset = get_asset_for_share_management(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
asset_id=file_id,
|
||||||
|
is_admin=_is_admin(principal),
|
||||||
|
)
|
||||||
|
share, changed = revoke_file_share(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
asset_id=asset.id,
|
||||||
|
share_id=share_id,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
)
|
||||||
|
if changed:
|
||||||
|
_audit_share_change(
|
||||||
|
session, principal, share, action="files.share.revoked"
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
return _file_share_response(share)
|
||||||
|
except FileStorageError as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _http_error(exc) from exc
|
||||||
|
|
||||||
|
|
||||||
|
def _share_audit_action(
|
||||||
|
*,
|
||||||
|
existed: bool,
|
||||||
|
previous_permission: str | None,
|
||||||
|
permission: str,
|
||||||
|
previous_expiry: datetime | None,
|
||||||
|
expiry: datetime | None,
|
||||||
|
) -> str | None:
|
||||||
|
if not existed:
|
||||||
|
return "files.share.granted"
|
||||||
|
permission_changed = previous_permission != permission
|
||||||
|
expiry_changed = _normalized_expiry(previous_expiry) != _normalized_expiry(expiry)
|
||||||
|
if permission_changed:
|
||||||
|
return "files.share.changed"
|
||||||
|
if expiry_changed:
|
||||||
|
return "files.share.expiry_changed"
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _audit_share_change(
|
||||||
|
session: Session,
|
||||||
|
principal: ApiPrincipal,
|
||||||
|
share,
|
||||||
|
*,
|
||||||
|
action: str,
|
||||||
|
) -> None:
|
||||||
|
audit_from_principal(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
action=action,
|
||||||
|
object_type="file_share",
|
||||||
|
object_id=share.id,
|
||||||
|
details={
|
||||||
|
"file_asset_id": share.file_asset_id,
|
||||||
|
"target_type": share.target_type,
|
||||||
|
"target_id": share.target_id,
|
||||||
|
"permission": share.permission,
|
||||||
|
"expires_at": _normalized_expiry(share.expires_at),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _normalized_expiry(value: datetime | None) -> str | None:
|
||||||
|
if value is None:
|
||||||
|
return None
|
||||||
|
if value.tzinfo is None:
|
||||||
|
value = value.replace(tzinfo=timezone.utc)
|
||||||
|
return value.astimezone(timezone.utc).isoformat()
|
||||||
@@ -0,0 +1,308 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from typing import Literal
|
||||||
|
from fastapi import APIRouter, Depends, HTTPException, status
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from govoplan_core.auth import ApiPrincipal, require_scope
|
||||||
|
from govoplan_files.backend.change_tracking import (
|
||||||
|
FILES_CONNECTOR_SPACES_COLLECTION,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.schemas import (
|
||||||
|
FileConnectorSpaceCreateRequest,
|
||||||
|
FileConnectorSpaceResponse,
|
||||||
|
FileConnectorSpacesResponse,
|
||||||
|
FileConnectorSpaceUpdateRequest,
|
||||||
|
FileSpaceResponse,
|
||||||
|
FileSpacesResponse,
|
||||||
|
)
|
||||||
|
from govoplan_core.db.session import get_session
|
||||||
|
from govoplan_files.backend.storage.access import group_refs_for_ids, user_group_ids
|
||||||
|
from govoplan_files.backend.storage.common import FileStorageError
|
||||||
|
from govoplan_files.backend.storage.connector_spaces import (
|
||||||
|
connector_space_owner_id,
|
||||||
|
create_connector_space,
|
||||||
|
get_connector_space_for_user,
|
||||||
|
list_connector_spaces_for_user,
|
||||||
|
soft_delete_connector_space,
|
||||||
|
update_connector_space,
|
||||||
|
validate_connector_space_write_mode,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.storage.connector_policy import (
|
||||||
|
ConnectorPolicyDenied,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
from govoplan_files.backend.route_support import (
|
||||||
|
FILES_CONNECTOR_SPACE_RESOURCE,
|
||||||
|
_connector_policy_error,
|
||||||
|
_connector_space_file_space_response,
|
||||||
|
_connector_space_policy_decision,
|
||||||
|
_connector_space_response,
|
||||||
|
_http_error,
|
||||||
|
_is_admin,
|
||||||
|
_record_connector_settings_change,
|
||||||
|
_visible_connector_profile,
|
||||||
|
)
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/files", tags=["files"])
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/spaces", response_model=FileSpacesResponse)
|
||||||
|
def list_file_spaces(
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(require_scope("files:file:read")),
|
||||||
|
):
|
||||||
|
spaces = [
|
||||||
|
FileSpaceResponse(
|
||||||
|
id=f"user:{principal.user.id}",
|
||||||
|
label="My files",
|
||||||
|
owner_type="user",
|
||||||
|
owner_id=principal.user.id,
|
||||||
|
description="Files owned by your user account.",
|
||||||
|
)
|
||||||
|
]
|
||||||
|
group_ids = user_group_ids(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
include_admin_groups=_is_admin(principal),
|
||||||
|
)
|
||||||
|
if group_ids:
|
||||||
|
groups = group_refs_for_ids(tenant_id=principal.tenant_id, group_ids=group_ids)
|
||||||
|
spaces.extend(
|
||||||
|
FileSpaceResponse(
|
||||||
|
id=f"group:{group.id}",
|
||||||
|
label=f"{group.name} files",
|
||||||
|
owner_type="group",
|
||||||
|
owner_id=group.id,
|
||||||
|
description="Files owned by this group.",
|
||||||
|
)
|
||||||
|
for group in groups
|
||||||
|
)
|
||||||
|
connector_spaces = list_connector_spaces_for_user(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
is_admin=_is_admin(principal),
|
||||||
|
)
|
||||||
|
spaces.extend(
|
||||||
|
_connector_space_file_space_response(space) for space in connector_spaces
|
||||||
|
)
|
||||||
|
return FileSpacesResponse(spaces=spaces)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/connector-spaces", response_model=FileConnectorSpacesResponse)
|
||||||
|
def list_file_connector_spaces(
|
||||||
|
owner_type: Literal["user", "group"] | None = None,
|
||||||
|
owner_id: str | None = None,
|
||||||
|
include_inactive: bool = False,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(require_scope("files:file:read")),
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
spaces = list_connector_spaces_for_user(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
owner_type=owner_type,
|
||||||
|
owner_id=owner_id,
|
||||||
|
include_inactive=include_inactive and _is_admin(principal),
|
||||||
|
is_admin=_is_admin(principal),
|
||||||
|
)
|
||||||
|
return FileConnectorSpacesResponse(
|
||||||
|
spaces=[_connector_space_response(space) for space in spaces]
|
||||||
|
)
|
||||||
|
except FileStorageError as exc:
|
||||||
|
raise _http_error(exc) from exc
|
||||||
|
|
||||||
|
|
||||||
|
@router.post(
|
||||||
|
"/connector-spaces",
|
||||||
|
response_model=FileConnectorSpaceResponse,
|
||||||
|
status_code=status.HTTP_201_CREATED,
|
||||||
|
)
|
||||||
|
def create_file_connector_space(
|
||||||
|
payload: FileConnectorSpaceCreateRequest,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(require_scope("files:file:organize")),
|
||||||
|
):
|
||||||
|
if payload.owner_type == "group" and not payload.owner_id:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_400_BAD_REQUEST,
|
||||||
|
detail="owner_id is required for group connector spaces",
|
||||||
|
)
|
||||||
|
target_owner = payload.owner_id or principal.user.id
|
||||||
|
try:
|
||||||
|
profile = _visible_connector_profile(
|
||||||
|
session, principal, payload.connector_profile_id
|
||||||
|
)
|
||||||
|
decision = _connector_space_policy_decision(
|
||||||
|
profile,
|
||||||
|
library_id=payload.library_id,
|
||||||
|
remote_path=payload.remote_path,
|
||||||
|
operation="link",
|
||||||
|
)
|
||||||
|
if not decision.allowed:
|
||||||
|
raise ConnectorPolicyDenied(decision)
|
||||||
|
validate_connector_space_write_mode(profile, read_only=payload.read_only)
|
||||||
|
space = create_connector_space(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
owner_type=payload.owner_type,
|
||||||
|
owner_id=target_owner,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
label=payload.label,
|
||||||
|
profile=profile,
|
||||||
|
library_id=payload.library_id,
|
||||||
|
remote_path=payload.remote_path,
|
||||||
|
sync_mode=payload.sync_mode,
|
||||||
|
read_only=payload.read_only,
|
||||||
|
metadata=payload.metadata,
|
||||||
|
is_admin=_is_admin(principal),
|
||||||
|
)
|
||||||
|
_record_connector_settings_change(
|
||||||
|
session,
|
||||||
|
collection=FILES_CONNECTOR_SPACES_COLLECTION,
|
||||||
|
resource_type=FILES_CONNECTOR_SPACE_RESOURCE,
|
||||||
|
resource_id=space.id,
|
||||||
|
operation="created",
|
||||||
|
principal=principal,
|
||||||
|
tenant_id=space.tenant_id,
|
||||||
|
payload={
|
||||||
|
"owner_type": space.owner_type,
|
||||||
|
"owner_id": connector_space_owner_id(space),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
return _connector_space_response(space)
|
||||||
|
except ConnectorPolicyDenied as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _connector_policy_error(exc) from exc
|
||||||
|
except (FileStorageError, ValueError, json.JSONDecodeError) as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _http_error(exc) from exc
|
||||||
|
|
||||||
|
|
||||||
|
@router.patch("/connector-spaces/{space_id}", response_model=FileConnectorSpaceResponse)
|
||||||
|
def update_file_connector_space(
|
||||||
|
space_id: str,
|
||||||
|
payload: FileConnectorSpaceUpdateRequest,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(require_scope("files:file:organize")),
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
space = get_connector_space_for_user(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
space_id=space_id,
|
||||||
|
include_inactive=_is_admin(principal),
|
||||||
|
is_admin=_is_admin(principal),
|
||||||
|
)
|
||||||
|
except FileStorageError as exc:
|
||||||
|
raise _http_error(exc, not_found=True) from exc
|
||||||
|
try:
|
||||||
|
if (
|
||||||
|
payload.library_id is not None
|
||||||
|
or payload.remote_path is not None
|
||||||
|
or payload.read_only is not None
|
||||||
|
):
|
||||||
|
profile = _visible_connector_profile(
|
||||||
|
session, principal, space.connector_profile_id
|
||||||
|
)
|
||||||
|
decision = _connector_space_policy_decision(
|
||||||
|
profile,
|
||||||
|
library_id=payload.library_id
|
||||||
|
if payload.library_id is not None
|
||||||
|
else space.library_id,
|
||||||
|
remote_path=payload.remote_path
|
||||||
|
if payload.remote_path is not None
|
||||||
|
else space.remote_path,
|
||||||
|
operation="link",
|
||||||
|
)
|
||||||
|
if not decision.allowed:
|
||||||
|
raise ConnectorPolicyDenied(decision)
|
||||||
|
validate_connector_space_write_mode(
|
||||||
|
profile,
|
||||||
|
read_only=(
|
||||||
|
payload.read_only
|
||||||
|
if payload.read_only is not None
|
||||||
|
else space.read_only
|
||||||
|
),
|
||||||
|
)
|
||||||
|
update_connector_space(
|
||||||
|
session,
|
||||||
|
space,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
label=payload.label,
|
||||||
|
library_id=payload.library_id,
|
||||||
|
remote_path=payload.remote_path,
|
||||||
|
sync_mode=payload.sync_mode,
|
||||||
|
read_only=payload.read_only,
|
||||||
|
is_active=payload.is_active,
|
||||||
|
metadata=payload.metadata,
|
||||||
|
is_admin=_is_admin(principal),
|
||||||
|
)
|
||||||
|
_record_connector_settings_change(
|
||||||
|
session,
|
||||||
|
collection=FILES_CONNECTOR_SPACES_COLLECTION,
|
||||||
|
resource_type=FILES_CONNECTOR_SPACE_RESOURCE,
|
||||||
|
resource_id=space.id,
|
||||||
|
operation="updated",
|
||||||
|
principal=principal,
|
||||||
|
tenant_id=space.tenant_id,
|
||||||
|
payload={
|
||||||
|
"owner_type": space.owner_type,
|
||||||
|
"owner_id": connector_space_owner_id(space),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
return _connector_space_response(space)
|
||||||
|
except ConnectorPolicyDenied as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _connector_policy_error(exc) from exc
|
||||||
|
except (FileStorageError, ValueError, json.JSONDecodeError) as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _http_error(exc) from exc
|
||||||
|
|
||||||
|
|
||||||
|
@router.delete(
|
||||||
|
"/connector-spaces/{space_id}", response_model=FileConnectorSpaceResponse
|
||||||
|
)
|
||||||
|
def delete_file_connector_space(
|
||||||
|
space_id: str,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(require_scope("files:file:organize")),
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
space = get_connector_space_for_user(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
space_id=space_id,
|
||||||
|
include_inactive=True,
|
||||||
|
is_admin=_is_admin(principal),
|
||||||
|
)
|
||||||
|
soft_delete_connector_space(
|
||||||
|
session, space, user_id=principal.user.id, is_admin=_is_admin(principal)
|
||||||
|
)
|
||||||
|
_record_connector_settings_change(
|
||||||
|
session,
|
||||||
|
collection=FILES_CONNECTOR_SPACES_COLLECTION,
|
||||||
|
resource_type=FILES_CONNECTOR_SPACE_RESOURCE,
|
||||||
|
resource_id=space.id,
|
||||||
|
operation="deleted",
|
||||||
|
principal=principal,
|
||||||
|
tenant_id=space.tenant_id,
|
||||||
|
payload={
|
||||||
|
"owner_type": space.owner_type,
|
||||||
|
"owner_id": connector_space_owner_id(space),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
return _connector_space_response(space)
|
||||||
|
except FileStorageError as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _http_error(exc, not_found=True) from exc
|
||||||
@@ -0,0 +1,213 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import tempfile
|
||||||
|
from fastapi import APIRouter, Depends
|
||||||
|
from fastapi.responses import FileResponse
|
||||||
|
from starlette.background import BackgroundTask
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from govoplan_core.auth import ApiPrincipal, require_scope
|
||||||
|
from govoplan_files.backend.schemas import (
|
||||||
|
ArchiveRequest,
|
||||||
|
PatternMatchResponse,
|
||||||
|
PatternResolveRequest,
|
||||||
|
PatternResolveResponse,
|
||||||
|
RenamePreviewItem,
|
||||||
|
RenameRequest,
|
||||||
|
RenameResponse,
|
||||||
|
TransferRequest,
|
||||||
|
TransferResponse,
|
||||||
|
_conflict_resolutions,
|
||||||
|
)
|
||||||
|
from govoplan_core.db.session import get_session
|
||||||
|
from govoplan_files.backend.storage.paths import (
|
||||||
|
UnsafeFilePathError,
|
||||||
|
filename_from_path,
|
||||||
|
normalize_logical_path,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.storage.archives import create_zip_file
|
||||||
|
from govoplan_files.backend.storage.common import FileStorageError
|
||||||
|
from govoplan_files.backend.storage.files import (
|
||||||
|
get_asset_for_user,
|
||||||
|
list_assets_for_user,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.storage.search import resolve_patterns
|
||||||
|
from govoplan_files.backend.storage.transfers import (
|
||||||
|
rename_selection,
|
||||||
|
transfer_selection,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
from govoplan_files.backend.route_support import (
|
||||||
|
_asset_response,
|
||||||
|
_attachment_disposition,
|
||||||
|
_audit_connector_access,
|
||||||
|
_cleanup_temp_file,
|
||||||
|
_ensure_campaign_file_access,
|
||||||
|
_ensure_list_owner_access,
|
||||||
|
_http_error,
|
||||||
|
_is_admin,
|
||||||
|
)
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/files", tags=["files"])
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/bulk-rename", response_model=RenameResponse)
|
||||||
|
def bulk_rename(
|
||||||
|
payload: RenameRequest,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(require_scope("files:file:organize")),
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
plan = rename_selection(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
file_ids=payload.file_ids,
|
||||||
|
folder_paths=payload.folder_paths,
|
||||||
|
owner_type=payload.owner_type,
|
||||||
|
owner_id=payload.owner_id,
|
||||||
|
mode=payload.mode,
|
||||||
|
new_name=payload.new_name,
|
||||||
|
find=payload.find,
|
||||||
|
replacement=payload.replacement,
|
||||||
|
prefix=payload.prefix,
|
||||||
|
suffix=payload.suffix,
|
||||||
|
recursive=payload.recursive,
|
||||||
|
dry_run=payload.dry_run,
|
||||||
|
is_admin=_is_admin(principal),
|
||||||
|
)
|
||||||
|
if not payload.dry_run:
|
||||||
|
session.commit()
|
||||||
|
return RenameResponse(
|
||||||
|
dry_run=payload.dry_run,
|
||||||
|
items=[
|
||||||
|
RenamePreviewItem(
|
||||||
|
kind=item.kind,
|
||||||
|
id=item.id,
|
||||||
|
file_id=item.id if item.kind == "file" else None,
|
||||||
|
folder_path=item.old_path if item.kind == "folder" else None,
|
||||||
|
old_path=item.old_path,
|
||||||
|
new_path=item.new_path,
|
||||||
|
)
|
||||||
|
for item in plan
|
||||||
|
],
|
||||||
|
)
|
||||||
|
except (FileStorageError, UnsafeFilePathError, ValueError) as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _http_error(exc) from exc
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/transfer", response_model=TransferResponse)
|
||||||
|
def transfer_files(
|
||||||
|
payload: TransferRequest,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(require_scope("files:file:organize")),
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
files, folders = transfer_selection(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
operation=payload.operation,
|
||||||
|
file_ids=payload.file_ids,
|
||||||
|
folder_paths=payload.folder_paths,
|
||||||
|
source_owner_type=payload.source_owner_type,
|
||||||
|
source_owner_id=payload.source_owner_id,
|
||||||
|
target_owner_type=payload.target_owner_type,
|
||||||
|
target_owner_id=payload.target_owner_id,
|
||||||
|
target_folder=payload.target_folder,
|
||||||
|
conflict_strategy=payload.conflict_strategy,
|
||||||
|
conflict_resolutions=_conflict_resolutions(payload.conflict_resolutions),
|
||||||
|
is_admin=_is_admin(principal),
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
return TransferResponse(
|
||||||
|
operation=payload.operation, files=files, folders=folders
|
||||||
|
)
|
||||||
|
except (FileStorageError, UnsafeFilePathError, ValueError) as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _http_error(exc) from exc
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/archive.zip")
|
||||||
|
def download_archive(
|
||||||
|
payload: ArchiveRequest,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(require_scope("files:file:download")),
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
assets = [
|
||||||
|
get_asset_for_user(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
asset_id=file_id,
|
||||||
|
is_admin=_is_admin(principal),
|
||||||
|
)
|
||||||
|
for file_id in payload.file_ids
|
||||||
|
]
|
||||||
|
tmp = tempfile.NamedTemporaryFile(
|
||||||
|
prefix="govoplan-files-", suffix=".zip", delete=False
|
||||||
|
)
|
||||||
|
tmp_path = tmp.name
|
||||||
|
tmp.close()
|
||||||
|
try:
|
||||||
|
create_zip_file(session, assets, tmp_path)
|
||||||
|
except Exception:
|
||||||
|
_cleanup_temp_file(tmp_path)
|
||||||
|
raise
|
||||||
|
_audit_connector_access(session, principal, assets, operation="archive")
|
||||||
|
except FileStorageError as exc:
|
||||||
|
raise _http_error(exc) from exc
|
||||||
|
filename = filename_from_path(
|
||||||
|
normalize_logical_path(payload.filename, fallback_filename="files.zip")
|
||||||
|
)
|
||||||
|
headers = {"Content-Disposition": _attachment_disposition(filename)}
|
||||||
|
return FileResponse(
|
||||||
|
tmp_path,
|
||||||
|
media_type="application/zip",
|
||||||
|
headers=headers,
|
||||||
|
background=BackgroundTask(_cleanup_temp_file, tmp_path),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/resolve-patterns", response_model=PatternResolveResponse)
|
||||||
|
def resolve_file_patterns(
|
||||||
|
payload: PatternResolveRequest,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(require_scope("files:file:read")),
|
||||||
|
):
|
||||||
|
_ensure_list_owner_access(session, principal, payload.owner_type, payload.owner_id)
|
||||||
|
_ensure_campaign_file_access(session, principal, payload.campaign_id)
|
||||||
|
try:
|
||||||
|
assets = list_assets_for_user(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
owner_type=payload.owner_type,
|
||||||
|
owner_id=payload.owner_id,
|
||||||
|
campaign_id=payload.campaign_id,
|
||||||
|
path_prefix=payload.path_prefix,
|
||||||
|
is_admin=_is_admin(principal),
|
||||||
|
)
|
||||||
|
resolved, unmatched = resolve_patterns(
|
||||||
|
assets,
|
||||||
|
payload.patterns,
|
||||||
|
base_path=payload.path_prefix,
|
||||||
|
case_sensitive=payload.case_sensitive,
|
||||||
|
)
|
||||||
|
return PatternResolveResponse(
|
||||||
|
patterns=[
|
||||||
|
PatternMatchResponse(
|
||||||
|
pattern=item.pattern,
|
||||||
|
matches=[_asset_response(session, asset) for asset in item.matches],
|
||||||
|
)
|
||||||
|
for item in resolved
|
||||||
|
],
|
||||||
|
unmatched=[_asset_response(session, asset) for asset in unmatched]
|
||||||
|
if payload.include_unmatched
|
||||||
|
else [],
|
||||||
|
)
|
||||||
|
except (FileStorageError, UnsafeFilePathError, ValueError) as exc:
|
||||||
|
raise _http_error(exc) from exc
|
||||||
@@ -0,0 +1,487 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import hmac
|
||||||
|
import json
|
||||||
|
from datetime import UTC, datetime, timedelta
|
||||||
|
from typing import Literal
|
||||||
|
from fastapi import APIRouter, Depends, File as FastAPIFile, Form, UploadFile
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from govoplan_core.auth import ApiPrincipal, require_scope
|
||||||
|
from govoplan_core.security.secrets import (
|
||||||
|
TransientPayloadError,
|
||||||
|
open_transient_payload,
|
||||||
|
seal_transient_payload,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.schemas import (
|
||||||
|
ArchiveEntryResponse,
|
||||||
|
ArchivePreviewResponse,
|
||||||
|
ConflictResolutionRequest,
|
||||||
|
FileUploadResponse,
|
||||||
|
_conflict_resolutions,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.db.models import FileAsset
|
||||||
|
from govoplan_core.db.session import get_session
|
||||||
|
from govoplan_files.backend.runtime import settings
|
||||||
|
from govoplan_files.backend.storage.paths import (
|
||||||
|
UnsafeFilePathError,
|
||||||
|
normalize_folder,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.storage.archives import (
|
||||||
|
archive_format_for_filename,
|
||||||
|
extract_archive_upload,
|
||||||
|
extract_zip_upload,
|
||||||
|
inspect_archive,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.storage.common import FileStorageError
|
||||||
|
from govoplan_files.backend.storage.connector_policy import (
|
||||||
|
ConnectorPolicyDenied,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.storage.files import (
|
||||||
|
create_file_asset,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
from govoplan_files.backend.route_support import (
|
||||||
|
_asset_response,
|
||||||
|
_audit_connector_imports,
|
||||||
|
_cleanup_temp_file,
|
||||||
|
_connector_policy_error,
|
||||||
|
_enforce_connector_policy,
|
||||||
|
_http_error,
|
||||||
|
_is_admin,
|
||||||
|
_read_limited_upload,
|
||||||
|
_source_metadata_from_form,
|
||||||
|
_spool_limited_upload_to_temp,
|
||||||
|
)
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/files", tags=["files"])
|
||||||
|
_ARCHIVE_PREVIEW_PURPOSE = "files.archive-preview.v1"
|
||||||
|
|
||||||
|
|
||||||
|
def _archive_suffix(filename: str) -> str:
|
||||||
|
lowered = filename.casefold()
|
||||||
|
for suffix in (".tar.bz2", ".tar.gz", ".tar.xz", ".tbz2", ".tgz", ".txz", ".tar", ".zip"):
|
||||||
|
if lowered.endswith(suffix):
|
||||||
|
return suffix
|
||||||
|
return ".archive"
|
||||||
|
|
||||||
|
|
||||||
|
def _archive_sha256(path: str) -> str:
|
||||||
|
digest = hashlib.sha256()
|
||||||
|
with open(path, "rb") as source:
|
||||||
|
while chunk := source.read(1024 * 1024):
|
||||||
|
digest.update(chunk)
|
||||||
|
return digest.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def _archive_selected_paths(value: str) -> list[str]:
|
||||||
|
parsed = json.loads(value)
|
||||||
|
if not isinstance(parsed, list) or not parsed:
|
||||||
|
raise FileStorageError("Select at least one archive file or folder")
|
||||||
|
if len(parsed) > settings.file_archive_max_entries:
|
||||||
|
raise FileStorageError(
|
||||||
|
"Archive selection exceeds the configured entry limit"
|
||||||
|
)
|
||||||
|
selected: list[str] = []
|
||||||
|
for item in parsed:
|
||||||
|
if not isinstance(item, str) or not item.strip():
|
||||||
|
raise FileStorageError("Archive selection contains an invalid path")
|
||||||
|
if len(item) > 4096:
|
||||||
|
raise FileStorageError("Archive selection path is too long")
|
||||||
|
selected.append(item)
|
||||||
|
return selected
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_archive_preview_token(
|
||||||
|
token: str,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
user_id: str,
|
||||||
|
owner_type: str,
|
||||||
|
owner_id: str,
|
||||||
|
path: str,
|
||||||
|
campaign_id: str | None,
|
||||||
|
archive_format: str,
|
||||||
|
archive_sha256: str,
|
||||||
|
) -> None:
|
||||||
|
payload = open_transient_payload(
|
||||||
|
token,
|
||||||
|
ttl_seconds=settings.file_archive_preview_ttl_seconds,
|
||||||
|
)
|
||||||
|
expected = {
|
||||||
|
"purpose": _ARCHIVE_PREVIEW_PURPOSE,
|
||||||
|
"tenant_id": tenant_id,
|
||||||
|
"user_id": user_id,
|
||||||
|
"owner_type": owner_type,
|
||||||
|
"owner_id": owner_id,
|
||||||
|
"path": path,
|
||||||
|
"campaign_id": campaign_id or "",
|
||||||
|
"archive_format": archive_format,
|
||||||
|
}
|
||||||
|
if any(payload.get(key) != value for key, value in expected.items()):
|
||||||
|
raise FileStorageError(
|
||||||
|
"Archive preview does not match this upload destination"
|
||||||
|
)
|
||||||
|
token_digest = payload.get("archive_sha256")
|
||||||
|
if not isinstance(token_digest, str) or not hmac.compare_digest(
|
||||||
|
token_digest, archive_sha256
|
||||||
|
):
|
||||||
|
raise FileStorageError(
|
||||||
|
"Archive contents changed after preview; preview it again"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/archive-preview", response_model=ArchivePreviewResponse)
|
||||||
|
def preview_archive_upload(
|
||||||
|
file: UploadFile = FastAPIFile(...),
|
||||||
|
owner_type: Literal["user", "group"] = Form(default="user"),
|
||||||
|
owner_id: str | None = Form(default=None),
|
||||||
|
path: str = Form(default=""),
|
||||||
|
campaign_id: str | None = Form(default=None),
|
||||||
|
password: str | None = Form(default=None),
|
||||||
|
principal: ApiPrincipal = Depends(require_scope("files:file:upload")),
|
||||||
|
):
|
||||||
|
target_owner = owner_id or principal.user.id
|
||||||
|
archive_path: str | None = None
|
||||||
|
filename = file.filename or "archive"
|
||||||
|
try:
|
||||||
|
archive_format = archive_format_for_filename(filename)
|
||||||
|
archive_path = _spool_limited_upload_to_temp(
|
||||||
|
file,
|
||||||
|
max_bytes=settings.file_upload_zip_max_bytes,
|
||||||
|
suffix=_archive_suffix(filename),
|
||||||
|
)
|
||||||
|
inspection = inspect_archive(
|
||||||
|
archive_path,
|
||||||
|
filename=filename,
|
||||||
|
password=password,
|
||||||
|
max_entries=settings.file_archive_max_entries,
|
||||||
|
max_expanded_bytes=settings.file_archive_max_expanded_bytes,
|
||||||
|
max_expansion_ratio=settings.file_archive_max_expansion_ratio,
|
||||||
|
)
|
||||||
|
digest = _archive_sha256(archive_path)
|
||||||
|
normalized_path = normalize_folder(path)
|
||||||
|
preview_token = seal_transient_payload(
|
||||||
|
{
|
||||||
|
"purpose": _ARCHIVE_PREVIEW_PURPOSE,
|
||||||
|
"tenant_id": principal.tenant_id,
|
||||||
|
"user_id": principal.user.id,
|
||||||
|
"owner_type": owner_type,
|
||||||
|
"owner_id": target_owner,
|
||||||
|
"path": normalized_path,
|
||||||
|
"campaign_id": campaign_id or "",
|
||||||
|
"archive_format": archive_format,
|
||||||
|
"archive_sha256": digest,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
expires_at = datetime.now(UTC) + timedelta(
|
||||||
|
seconds=settings.file_archive_preview_ttl_seconds
|
||||||
|
)
|
||||||
|
return ArchivePreviewResponse(
|
||||||
|
preview_token=preview_token,
|
||||||
|
archive_format=inspection.archive_format,
|
||||||
|
entries=[
|
||||||
|
ArchiveEntryResponse(
|
||||||
|
path=entry.path,
|
||||||
|
kind=entry.kind,
|
||||||
|
size_bytes=entry.size_bytes,
|
||||||
|
compressed_size_bytes=entry.compressed_size_bytes,
|
||||||
|
encrypted=entry.encrypted,
|
||||||
|
)
|
||||||
|
for entry in inspection.entries
|
||||||
|
],
|
||||||
|
file_count=inspection.file_count,
|
||||||
|
directory_count=inspection.directory_count,
|
||||||
|
expanded_size_bytes=inspection.expanded_size_bytes,
|
||||||
|
compressed_size_bytes=inspection.compressed_size_bytes,
|
||||||
|
requires_password=inspection.requires_password,
|
||||||
|
password_verified=inspection.password_verified,
|
||||||
|
expires_at=expires_at.isoformat(),
|
||||||
|
)
|
||||||
|
except (FileStorageError, UnsafeFilePathError, ValueError) as exc:
|
||||||
|
raise _http_error(exc) from exc
|
||||||
|
finally:
|
||||||
|
if archive_path:
|
||||||
|
_cleanup_temp_file(archive_path)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/archive-confirm", response_model=FileUploadResponse)
|
||||||
|
def confirm_archive_upload(
|
||||||
|
file: UploadFile = FastAPIFile(...),
|
||||||
|
preview_token: str = Form(...),
|
||||||
|
selected_paths_json: str = Form(...),
|
||||||
|
owner_type: Literal["user", "group"] = Form(default="user"),
|
||||||
|
owner_id: str | None = Form(default=None),
|
||||||
|
path: str = Form(default=""),
|
||||||
|
campaign_id: str | None = Form(default=None),
|
||||||
|
password: str | None = Form(default=None),
|
||||||
|
conflict_strategy: Literal["reject", "overwrite", "rename"] = Form(
|
||||||
|
default="reject"
|
||||||
|
),
|
||||||
|
conflict_resolutions_json: str | None = Form(default=None),
|
||||||
|
source_provenance_json: str | None = Form(default=None),
|
||||||
|
source_revision: str | None = Form(default=None),
|
||||||
|
connector_policy_json: str | None = Form(default=None),
|
||||||
|
encryption_vault_id: str | None = Form(default=None),
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(require_scope("files:file:upload")),
|
||||||
|
):
|
||||||
|
target_owner = owner_id or principal.user.id
|
||||||
|
archive_path: str | None = None
|
||||||
|
filename = file.filename or "archive"
|
||||||
|
try:
|
||||||
|
raw_resolutions = (
|
||||||
|
json.loads(conflict_resolutions_json)
|
||||||
|
if conflict_resolutions_json
|
||||||
|
else []
|
||||||
|
)
|
||||||
|
upload_resolutions = _conflict_resolutions(
|
||||||
|
[ConflictResolutionRequest(**item) for item in raw_resolutions]
|
||||||
|
)
|
||||||
|
selected_paths = _archive_selected_paths(selected_paths_json)
|
||||||
|
_enforce_connector_policy(
|
||||||
|
source_provenance_json,
|
||||||
|
connector_policy_json,
|
||||||
|
operation="import",
|
||||||
|
)
|
||||||
|
metadata = _source_metadata_from_form(
|
||||||
|
source_provenance_json, source_revision
|
||||||
|
)
|
||||||
|
archive_format = archive_format_for_filename(filename)
|
||||||
|
normalized_path = normalize_folder(path)
|
||||||
|
archive_path = _spool_limited_upload_to_temp(
|
||||||
|
file,
|
||||||
|
max_bytes=settings.file_upload_zip_max_bytes,
|
||||||
|
suffix=_archive_suffix(filename),
|
||||||
|
)
|
||||||
|
digest = _archive_sha256(archive_path)
|
||||||
|
_validate_archive_preview_token(
|
||||||
|
preview_token,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
owner_type=owner_type,
|
||||||
|
owner_id=target_owner,
|
||||||
|
path=normalized_path,
|
||||||
|
campaign_id=campaign_id,
|
||||||
|
archive_format=archive_format,
|
||||||
|
archive_sha256=digest,
|
||||||
|
)
|
||||||
|
extracted = extract_archive_upload(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
owner_type=owner_type,
|
||||||
|
owner_id=target_owner,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
archive_data=archive_path,
|
||||||
|
filename=filename,
|
||||||
|
folder=normalized_path,
|
||||||
|
campaign_id=campaign_id,
|
||||||
|
selected_paths=selected_paths,
|
||||||
|
password=password,
|
||||||
|
conflict_strategy=conflict_strategy,
|
||||||
|
conflict_resolutions=upload_resolutions,
|
||||||
|
metadata=metadata,
|
||||||
|
is_admin=_is_admin(principal),
|
||||||
|
encryption_vault_id=encryption_vault_id,
|
||||||
|
max_entries=settings.file_archive_max_entries,
|
||||||
|
max_file_bytes=settings.file_upload_max_bytes,
|
||||||
|
max_expanded_bytes=settings.file_archive_max_expanded_bytes,
|
||||||
|
max_expansion_ratio=settings.file_archive_max_expansion_ratio,
|
||||||
|
)
|
||||||
|
uploaded_assets = [item.asset for item in extracted]
|
||||||
|
_audit_connector_imports(session, principal, uploaded_assets)
|
||||||
|
session.commit()
|
||||||
|
return FileUploadResponse(
|
||||||
|
files=[
|
||||||
|
_asset_response(session, asset, include_shares=True)
|
||||||
|
for asset in uploaded_assets
|
||||||
|
]
|
||||||
|
)
|
||||||
|
except ConnectorPolicyDenied as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _connector_policy_error(exc) from exc
|
||||||
|
except (
|
||||||
|
FileStorageError,
|
||||||
|
TransientPayloadError,
|
||||||
|
UnsafeFilePathError,
|
||||||
|
ValueError,
|
||||||
|
json.JSONDecodeError,
|
||||||
|
) as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _http_error(exc) from exc
|
||||||
|
finally:
|
||||||
|
if archive_path:
|
||||||
|
_cleanup_temp_file(archive_path)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/upload", response_model=FileUploadResponse)
|
||||||
|
def upload_files(
|
||||||
|
files: list[UploadFile] = FastAPIFile(...),
|
||||||
|
owner_type: Literal["user", "group"] = Form(default="user"),
|
||||||
|
owner_id: str | None = Form(default=None),
|
||||||
|
path: str = Form(default=""),
|
||||||
|
campaign_id: str | None = Form(default=None),
|
||||||
|
unpack_zip: bool = Form(default=False),
|
||||||
|
conflict_strategy: Literal["reject", "overwrite", "rename"] = Form(
|
||||||
|
default="reject"
|
||||||
|
),
|
||||||
|
conflict_resolutions_json: str | None = Form(default=None),
|
||||||
|
source_provenance_json: str | None = Form(default=None),
|
||||||
|
source_revision: str | None = Form(default=None),
|
||||||
|
connector_policy_json: str | None = Form(default=None),
|
||||||
|
encryption_vault_id: str | None = Form(default=None),
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(require_scope("files:file:upload")),
|
||||||
|
):
|
||||||
|
target_owner = owner_id or principal.user.id
|
||||||
|
uploaded_assets: list[FileAsset] = []
|
||||||
|
try:
|
||||||
|
raw_resolutions = (
|
||||||
|
json.loads(conflict_resolutions_json) if conflict_resolutions_json else []
|
||||||
|
)
|
||||||
|
upload_resolutions = _conflict_resolutions(
|
||||||
|
[ConflictResolutionRequest(**item) for item in raw_resolutions]
|
||||||
|
)
|
||||||
|
_enforce_connector_policy(
|
||||||
|
source_provenance_json, connector_policy_json, operation="import"
|
||||||
|
)
|
||||||
|
metadata = _source_metadata_from_form(source_provenance_json, source_revision)
|
||||||
|
for upload in files:
|
||||||
|
filename = upload.filename or "file"
|
||||||
|
content_type = upload.content_type or None
|
||||||
|
upload_limit = (
|
||||||
|
settings.file_upload_zip_max_bytes
|
||||||
|
if unpack_zip and filename.lower().endswith(".zip")
|
||||||
|
else settings.file_upload_max_bytes
|
||||||
|
)
|
||||||
|
if unpack_zip and filename.lower().endswith(".zip"):
|
||||||
|
zip_path = _spool_limited_upload_to_temp(
|
||||||
|
upload, max_bytes=upload_limit, suffix=".zip"
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
extracted = extract_zip_upload(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
owner_type=owner_type,
|
||||||
|
owner_id=target_owner,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
zip_data=zip_path,
|
||||||
|
folder=path,
|
||||||
|
campaign_id=campaign_id,
|
||||||
|
conflict_strategy=conflict_strategy,
|
||||||
|
conflict_resolutions=upload_resolutions,
|
||||||
|
metadata=metadata,
|
||||||
|
is_admin=_is_admin(principal),
|
||||||
|
encryption_vault_id=encryption_vault_id,
|
||||||
|
max_file_bytes=settings.file_upload_max_bytes,
|
||||||
|
max_total_bytes=settings.file_upload_zip_max_bytes,
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
_cleanup_temp_file(zip_path)
|
||||||
|
uploaded_assets.extend(item.asset for item in extracted)
|
||||||
|
continue
|
||||||
|
data = _read_limited_upload(upload, max_bytes=upload_limit)
|
||||||
|
stored = create_file_asset(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
owner_type=owner_type,
|
||||||
|
owner_id=target_owner,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
filename=filename,
|
||||||
|
data=data,
|
||||||
|
folder=path,
|
||||||
|
content_type=content_type,
|
||||||
|
campaign_id=campaign_id,
|
||||||
|
conflict_strategy=conflict_strategy,
|
||||||
|
conflict_resolutions=upload_resolutions,
|
||||||
|
metadata=metadata,
|
||||||
|
is_admin=_is_admin(principal),
|
||||||
|
encryption_vault_id=encryption_vault_id,
|
||||||
|
)
|
||||||
|
uploaded_assets.append(stored.asset)
|
||||||
|
_audit_connector_imports(session, principal, uploaded_assets)
|
||||||
|
session.commit()
|
||||||
|
except ConnectorPolicyDenied as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _connector_policy_error(exc) from exc
|
||||||
|
except (FileStorageError, UnsafeFilePathError, ValueError) as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _http_error(exc) from exc
|
||||||
|
return FileUploadResponse(
|
||||||
|
files=[
|
||||||
|
_asset_response(session, asset, include_shares=True)
|
||||||
|
for asset in uploaded_assets
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/upload-zip", response_model=FileUploadResponse)
|
||||||
|
def upload_zip(
|
||||||
|
file: UploadFile = FastAPIFile(...),
|
||||||
|
owner_type: Literal["user", "group"] = Form(default="user"),
|
||||||
|
owner_id: str | None = Form(default=None),
|
||||||
|
path: str = Form(default=""),
|
||||||
|
campaign_id: str | None = Form(default=None),
|
||||||
|
conflict_strategy: Literal["reject", "overwrite", "rename"] = Form(
|
||||||
|
default="reject"
|
||||||
|
),
|
||||||
|
conflict_resolutions_json: str | None = Form(default=None),
|
||||||
|
source_provenance_json: str | None = Form(default=None),
|
||||||
|
source_revision: str | None = Form(default=None),
|
||||||
|
connector_policy_json: str | None = Form(default=None),
|
||||||
|
encryption_vault_id: str | None = Form(default=None),
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(require_scope("files:file:upload")),
|
||||||
|
):
|
||||||
|
target_owner = owner_id or principal.user.id
|
||||||
|
zip_path: str | None = None
|
||||||
|
try:
|
||||||
|
raw_resolutions = (
|
||||||
|
json.loads(conflict_resolutions_json) if conflict_resolutions_json else []
|
||||||
|
)
|
||||||
|
upload_resolutions = _conflict_resolutions(
|
||||||
|
[ConflictResolutionRequest(**item) for item in raw_resolutions]
|
||||||
|
)
|
||||||
|
_enforce_connector_policy(
|
||||||
|
source_provenance_json, connector_policy_json, operation="import"
|
||||||
|
)
|
||||||
|
metadata = _source_metadata_from_form(source_provenance_json, source_revision)
|
||||||
|
zip_path = _spool_limited_upload_to_temp(
|
||||||
|
file, max_bytes=settings.file_upload_zip_max_bytes, suffix=".zip"
|
||||||
|
)
|
||||||
|
extracted = extract_zip_upload(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
owner_type=owner_type,
|
||||||
|
owner_id=target_owner,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
zip_data=zip_path,
|
||||||
|
folder=path,
|
||||||
|
campaign_id=campaign_id,
|
||||||
|
conflict_strategy=conflict_strategy,
|
||||||
|
conflict_resolutions=upload_resolutions,
|
||||||
|
metadata=metadata,
|
||||||
|
is_admin=_is_admin(principal),
|
||||||
|
encryption_vault_id=encryption_vault_id,
|
||||||
|
max_file_bytes=settings.file_upload_max_bytes,
|
||||||
|
max_total_bytes=settings.file_upload_zip_max_bytes,
|
||||||
|
)
|
||||||
|
_audit_connector_imports(session, principal, [item.asset for item in extracted])
|
||||||
|
session.commit()
|
||||||
|
except ConnectorPolicyDenied as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _connector_policy_error(exc) from exc
|
||||||
|
except (FileStorageError, UnsafeFilePathError, ValueError) as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _http_error(exc) from exc
|
||||||
|
finally:
|
||||||
|
if zip_path:
|
||||||
|
_cleanup_temp_file(zip_path)
|
||||||
|
return FileUploadResponse(
|
||||||
|
files=[
|
||||||
|
_asset_response(session, item.asset, include_shares=True)
|
||||||
|
for item in extracted
|
||||||
|
]
|
||||||
|
)
|
||||||
@@ -1,5 +1,6 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
from typing import Any, Literal
|
from typing import Any, Literal
|
||||||
|
|
||||||
from pydantic import BaseModel, Field
|
from pydantic import BaseModel, Field
|
||||||
@@ -36,6 +37,7 @@ class FileConnectorSpaceCreateRequest(BaseModel):
|
|||||||
library_id: str | None = None
|
library_id: str | None = None
|
||||||
remote_path: str = ""
|
remote_path: str = ""
|
||||||
sync_mode: Literal["manual"] = "manual"
|
sync_mode: Literal["manual"] = "manual"
|
||||||
|
read_only: bool = True
|
||||||
metadata: dict[str, Any] = Field(default_factory=dict)
|
metadata: dict[str, Any] = Field(default_factory=dict)
|
||||||
|
|
||||||
|
|
||||||
@@ -44,6 +46,7 @@ class FileConnectorSpaceUpdateRequest(BaseModel):
|
|||||||
library_id: str | None = None
|
library_id: str | None = None
|
||||||
remote_path: str | None = None
|
remote_path: str | None = None
|
||||||
sync_mode: Literal["manual"] | None = None
|
sync_mode: Literal["manual"] | None = None
|
||||||
|
read_only: bool | None = None
|
||||||
is_active: bool | None = None
|
is_active: bool | None = None
|
||||||
metadata: dict[str, Any] | None = None
|
metadata: dict[str, Any] | None = None
|
||||||
|
|
||||||
@@ -73,11 +76,93 @@ class FileConnectorSpacesResponse(BaseModel):
|
|||||||
|
|
||||||
class FileShareResponse(BaseModel):
|
class FileShareResponse(BaseModel):
|
||||||
id: str
|
id: str
|
||||||
|
file_asset_id: str
|
||||||
target_type: str
|
target_type: str
|
||||||
target_id: str
|
target_id: str
|
||||||
permission: str
|
permission: str
|
||||||
|
created_by_user_id: str | None = None
|
||||||
created_at: str
|
created_at: str
|
||||||
|
expires_at: str | None = None
|
||||||
revoked_at: str | None = None
|
revoked_at: str | None = None
|
||||||
|
revoked_by_user_id: str | None = None
|
||||||
|
active: bool
|
||||||
|
|
||||||
|
|
||||||
|
class FileSharesResponse(BaseModel):
|
||||||
|
shares: list[FileShareResponse] = Field(default_factory=list)
|
||||||
|
|
||||||
|
|
||||||
|
class FileIntegrityScanCreateRequest(BaseModel):
|
||||||
|
verify_checksums: bool = True
|
||||||
|
batch_size: int = Field(default=100, ge=1, le=1000)
|
||||||
|
|
||||||
|
|
||||||
|
class FileIntegrityScanResponse(BaseModel):
|
||||||
|
id: str
|
||||||
|
tenant_id: str
|
||||||
|
storage_backend: str
|
||||||
|
storage_prefix: str
|
||||||
|
status: str
|
||||||
|
revision: int
|
||||||
|
phase: str
|
||||||
|
verify_checksums: bool
|
||||||
|
batch_size: int
|
||||||
|
scanned_blob_count: int
|
||||||
|
verified_blob_count: int
|
||||||
|
quarantined_blob_count: int
|
||||||
|
scanned_object_count: int
|
||||||
|
orphan_object_count: int
|
||||||
|
created_by_user_id: str | None = None
|
||||||
|
started_at: str | None = None
|
||||||
|
completed_at: str | None = None
|
||||||
|
last_error: str | None = None
|
||||||
|
created_at: str
|
||||||
|
updated_at: str
|
||||||
|
|
||||||
|
|
||||||
|
class FileIntegrityScansResponse(BaseModel):
|
||||||
|
scans: list[FileIntegrityScanResponse] = Field(default_factory=list)
|
||||||
|
|
||||||
|
|
||||||
|
class FileIntegrityFindingResponse(BaseModel):
|
||||||
|
id: str
|
||||||
|
scan_id: str
|
||||||
|
tenant_id: str
|
||||||
|
kind: str
|
||||||
|
state: str
|
||||||
|
revision: int
|
||||||
|
blob_id: str | None = None
|
||||||
|
storage_key: str
|
||||||
|
expected_size_bytes: int | None = None
|
||||||
|
observed_size_bytes: int | None = None
|
||||||
|
expected_checksum_sha256: str | None = None
|
||||||
|
observed_checksum_sha256: str | None = None
|
||||||
|
resolved_at: str | None = None
|
||||||
|
resolved_by_user_id: str | None = None
|
||||||
|
created_at: str
|
||||||
|
updated_at: str
|
||||||
|
|
||||||
|
|
||||||
|
class FileIntegrityFindingsResponse(BaseModel):
|
||||||
|
findings: list[FileIntegrityFindingResponse] = Field(default_factory=list)
|
||||||
|
|
||||||
|
|
||||||
|
class FileIntegrityActionRequest(BaseModel):
|
||||||
|
dry_run: bool = True
|
||||||
|
expected_revision: int = Field(ge=1)
|
||||||
|
|
||||||
|
|
||||||
|
class FileIntegrityScanRunRequest(BaseModel):
|
||||||
|
expected_revision: int = Field(ge=1)
|
||||||
|
|
||||||
|
|
||||||
|
class FileIntegrityActionResponse(BaseModel):
|
||||||
|
action: str
|
||||||
|
changed: bool
|
||||||
|
dry_run: bool
|
||||||
|
finding: FileIntegrityFindingResponse
|
||||||
|
inspection_kind: str | None = None
|
||||||
|
inspection_valid: bool | None = None
|
||||||
|
|
||||||
|
|
||||||
class FileSourceProvenance(BaseModel):
|
class FileSourceProvenance(BaseModel):
|
||||||
@@ -109,6 +194,10 @@ class FileAssetResponse(BaseModel):
|
|||||||
created_at: str
|
created_at: str
|
||||||
updated_at: str
|
updated_at: str
|
||||||
deleted_at: str | None = None
|
deleted_at: str | None = None
|
||||||
|
retained_until: str | None = None
|
||||||
|
legal_hold: bool = False
|
||||||
|
lifecycle_revision: int = 1
|
||||||
|
lifecycle_reason: str | None = None
|
||||||
audit_relevant: bool = False
|
audit_relevant: bool = False
|
||||||
metadata: dict[str, Any] | None = None
|
metadata: dict[str, Any] | None = None
|
||||||
source_provenance: FileSourceProvenance | None = None
|
source_provenance: FileSourceProvenance | None = None
|
||||||
@@ -154,6 +243,7 @@ class FileFolderDeleteResponse(BaseModel):
|
|||||||
|
|
||||||
class FileListResponse(BaseModel):
|
class FileListResponse(BaseModel):
|
||||||
files: list[FileAssetResponse]
|
files: list[FileAssetResponse]
|
||||||
|
total: int
|
||||||
cursor: str | None = None
|
cursor: str | None = None
|
||||||
next_cursor: str | None = None
|
next_cursor: str | None = None
|
||||||
watermark: str | None = None
|
watermark: str | None = None
|
||||||
@@ -172,6 +262,27 @@ class FileUploadResponse(BaseModel):
|
|||||||
files: list[FileAssetResponse]
|
files: list[FileAssetResponse]
|
||||||
|
|
||||||
|
|
||||||
|
class ArchiveEntryResponse(BaseModel):
|
||||||
|
path: str
|
||||||
|
kind: Literal["file", "directory"]
|
||||||
|
size_bytes: int
|
||||||
|
compressed_size_bytes: int | None = None
|
||||||
|
encrypted: bool = False
|
||||||
|
|
||||||
|
|
||||||
|
class ArchivePreviewResponse(BaseModel):
|
||||||
|
preview_token: str
|
||||||
|
archive_format: str
|
||||||
|
entries: list[ArchiveEntryResponse] = Field(default_factory=list)
|
||||||
|
file_count: int
|
||||||
|
directory_count: int
|
||||||
|
expanded_size_bytes: int
|
||||||
|
compressed_size_bytes: int
|
||||||
|
requires_password: bool
|
||||||
|
password_verified: bool
|
||||||
|
expires_at: str
|
||||||
|
|
||||||
|
|
||||||
class FileConnectorPolicySource(BaseModel):
|
class FileConnectorPolicySource(BaseModel):
|
||||||
scope_type: Literal["system", "tenant", "user", "group", "campaign"] = "system"
|
scope_type: Literal["system", "tenant", "user", "group", "campaign"] = "system"
|
||||||
scope_id: str | None = None
|
scope_id: str | None = None
|
||||||
@@ -372,6 +483,7 @@ class FileConnectorProviderResponse(BaseModel):
|
|||||||
installed: bool
|
installed: bool
|
||||||
browse_supported: bool
|
browse_supported: bool
|
||||||
import_supported: bool
|
import_supported: bool
|
||||||
|
write_supported: bool = False
|
||||||
optional_dependency: str | None = None
|
optional_dependency: str | None = None
|
||||||
permission_model: str
|
permission_model: str
|
||||||
sync_strategy: str
|
sync_strategy: str
|
||||||
@@ -418,7 +530,7 @@ class FileConnectorImportRequest(BaseModel):
|
|||||||
target_folder: str | None = None
|
target_folder: str | None = None
|
||||||
target_path: str | None = None
|
target_path: str | None = None
|
||||||
campaign_id: str | None = None
|
campaign_id: str | None = None
|
||||||
conflict_strategy: Literal["reject", "overwrite", "rename"] = "reject"
|
conflict_strategy: Literal["reject", "overwrite", "rename", "skip"] = "reject"
|
||||||
source_revision: str | None = None
|
source_revision: str | None = None
|
||||||
metadata: dict[str, Any] = Field(default_factory=dict)
|
metadata: dict[str, Any] = Field(default_factory=dict)
|
||||||
|
|
||||||
@@ -430,6 +542,77 @@ class FileConnectorSyncResponse(BaseModel):
|
|||||||
current_version_id: str
|
current_version_id: str
|
||||||
|
|
||||||
|
|
||||||
|
class FileConnectorFolderSyncRequest(BaseModel):
|
||||||
|
path: str = ""
|
||||||
|
target_folder: str | None = None
|
||||||
|
recursive: bool = True
|
||||||
|
conflict_strategy: Literal["reject", "overwrite", "rename", "skip"] = "skip"
|
||||||
|
max_files: int = Field(default=100, ge=1, le=500)
|
||||||
|
max_depth: int = Field(default=12, ge=0, le=50)
|
||||||
|
metadata: dict[str, Any] = Field(default_factory=dict)
|
||||||
|
|
||||||
|
|
||||||
|
class FileConnectorFolderSyncItemResponse(BaseModel):
|
||||||
|
source_path: str
|
||||||
|
target_path: str | None = None
|
||||||
|
action: Literal[
|
||||||
|
"created",
|
||||||
|
"updated",
|
||||||
|
"unchanged",
|
||||||
|
"skipped",
|
||||||
|
"conflict",
|
||||||
|
"policy_denied",
|
||||||
|
"failed",
|
||||||
|
]
|
||||||
|
file: FileAssetResponse | None = None
|
||||||
|
previous_version_id: str | None = None
|
||||||
|
current_version_id: str | None = None
|
||||||
|
source_revision: str | None = None
|
||||||
|
detail: str | None = None
|
||||||
|
policy_decision: dict[str, Any] | None = None
|
||||||
|
|
||||||
|
|
||||||
|
class FileConnectorFolderSyncSummary(BaseModel):
|
||||||
|
discovered: int = 0
|
||||||
|
created: int = 0
|
||||||
|
updated: int = 0
|
||||||
|
unchanged: int = 0
|
||||||
|
skipped: int = 0
|
||||||
|
conflicts: int = 0
|
||||||
|
policy_denied: int = 0
|
||||||
|
failed: int = 0
|
||||||
|
|
||||||
|
|
||||||
|
class FileConnectorFolderSyncResponse(BaseModel):
|
||||||
|
connector_space_id: str
|
||||||
|
connector_profile_id: str
|
||||||
|
provider: str
|
||||||
|
remote_path: str
|
||||||
|
target_folder: str
|
||||||
|
recursive: bool
|
||||||
|
truncated: bool = False
|
||||||
|
summary: FileConnectorFolderSyncSummary
|
||||||
|
items: list[FileConnectorFolderSyncItemResponse] = Field(default_factory=list)
|
||||||
|
|
||||||
|
|
||||||
|
class FileConnectorWriteRequest(BaseModel):
|
||||||
|
file_id: str
|
||||||
|
remote_path: str
|
||||||
|
idempotency_key: str = Field(min_length=8, max_length=160)
|
||||||
|
expected_revision: str | None = Field(default=None, max_length=500)
|
||||||
|
|
||||||
|
|
||||||
|
class FileConnectorWriteResponse(BaseModel):
|
||||||
|
recovery_operation_id: str
|
||||||
|
status: str
|
||||||
|
replayed: bool = False
|
||||||
|
provider: str
|
||||||
|
remote_path: str
|
||||||
|
revision: str | None = None
|
||||||
|
checksum_sha256: str
|
||||||
|
size_bytes: int
|
||||||
|
|
||||||
|
|
||||||
class BulkDeleteRequest(BaseModel):
|
class BulkDeleteRequest(BaseModel):
|
||||||
file_ids: list[str]
|
file_ids: list[str]
|
||||||
|
|
||||||
@@ -438,6 +621,69 @@ class BulkDeleteResponse(BaseModel):
|
|||||||
deleted_count: int
|
deleted_count: int
|
||||||
|
|
||||||
|
|
||||||
|
class FileLifecycleUpdateRequest(BaseModel):
|
||||||
|
retained_until: datetime | None = None
|
||||||
|
legal_hold: bool = False
|
||||||
|
reason: str = Field(min_length=1, max_length=500)
|
||||||
|
expected_revision: int = Field(ge=1)
|
||||||
|
|
||||||
|
|
||||||
|
class FileRestoreResponse(BaseModel):
|
||||||
|
restored_count: int
|
||||||
|
|
||||||
|
|
||||||
|
class FileFolderRestoreResponse(BaseModel):
|
||||||
|
restored_folders: int
|
||||||
|
restored_files: int
|
||||||
|
|
||||||
|
|
||||||
|
class FilePurgePreviewRequest(BaseModel):
|
||||||
|
file_ids: list[str] = Field(min_length=1, max_length=100)
|
||||||
|
|
||||||
|
|
||||||
|
class FilePurgePreviewItem(BaseModel):
|
||||||
|
file_id: str
|
||||||
|
filename: str
|
||||||
|
lifecycle_revision: int
|
||||||
|
deleted_at: str | None = None
|
||||||
|
retained_until: str | None = None
|
||||||
|
legal_hold: bool = False
|
||||||
|
blockers: list[str] = Field(default_factory=list)
|
||||||
|
blob_ids: list[str] = Field(default_factory=list)
|
||||||
|
|
||||||
|
|
||||||
|
class FilePurgePreviewResponse(BaseModel):
|
||||||
|
preview_sha256: str
|
||||||
|
eligible: bool
|
||||||
|
items: list[FilePurgePreviewItem]
|
||||||
|
|
||||||
|
|
||||||
|
class FilePurgeExecuteRequest(FilePurgePreviewRequest):
|
||||||
|
preview_sha256: str = Field(min_length=64, max_length=64)
|
||||||
|
idempotency_key: str = Field(min_length=8, max_length=160)
|
||||||
|
approval_reference: str = Field(min_length=3, max_length=1000)
|
||||||
|
confirmation: Literal["PURGE"]
|
||||||
|
|
||||||
|
|
||||||
|
class FilePurgeResponse(BaseModel):
|
||||||
|
recovery_operation_id: str
|
||||||
|
status: str
|
||||||
|
replayed: bool = False
|
||||||
|
purged_files: int = 0
|
||||||
|
released_blobs: int = 0
|
||||||
|
|
||||||
|
|
||||||
|
class FileBlobGcRequest(BaseModel):
|
||||||
|
limit: int = Field(default=25, ge=1, le=100)
|
||||||
|
approval_reference: str = Field(min_length=3, max_length=1000)
|
||||||
|
|
||||||
|
|
||||||
|
class FileBlobGcResponse(BaseModel):
|
||||||
|
inspected_blobs: int
|
||||||
|
deleted_blobs: int
|
||||||
|
unresolved_operation_ids: list[str] = Field(default_factory=list)
|
||||||
|
|
||||||
|
|
||||||
class ConflictResolutionRequest(BaseModel):
|
class ConflictResolutionRequest(BaseModel):
|
||||||
target_path: str
|
target_path: str
|
||||||
action: Literal["overwrite", "rename", "skip"]
|
action: Literal["overwrite", "rename", "skip"]
|
||||||
@@ -452,6 +698,7 @@ class FileShareRequest(BaseModel):
|
|||||||
target_type: Literal["user", "group", "campaign", "tenant"]
|
target_type: Literal["user", "group", "campaign", "tenant"]
|
||||||
target_id: str
|
target_id: str
|
||||||
permission: Literal["read", "write", "manage"] = "read"
|
permission: Literal["read", "write", "manage"] = "read"
|
||||||
|
expires_at: datetime | None = None
|
||||||
|
|
||||||
|
|
||||||
class BulkFileShareRequest(BaseModel):
|
class BulkFileShareRequest(BaseModel):
|
||||||
@@ -459,6 +706,7 @@ class BulkFileShareRequest(BaseModel):
|
|||||||
target_type: Literal["user", "group", "campaign", "tenant"]
|
target_type: Literal["user", "group", "campaign", "tenant"]
|
||||||
target_id: str
|
target_id: str
|
||||||
permission: Literal["read", "write", "manage"] = "read"
|
permission: Literal["read", "write", "manage"] = "read"
|
||||||
|
expires_at: datetime | None = None
|
||||||
|
|
||||||
|
|
||||||
class BulkFileShareResponse(BaseModel):
|
class BulkFileShareResponse(BaseModel):
|
||||||
|
|||||||
@@ -0,0 +1,329 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from collections.abc import Mapping, Sequence
|
||||||
|
from urllib.parse import quote
|
||||||
|
|
||||||
|
from sqlalchemy import func, or_, select
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from govoplan_core.auth import ApiPrincipal
|
||||||
|
from govoplan_core.core.events import PlatformEvent
|
||||||
|
from govoplan_core.core.modules import ModuleContext
|
||||||
|
from govoplan_core.core.search import (
|
||||||
|
SearchAuthorizationRequest,
|
||||||
|
SearchBackfillPage,
|
||||||
|
SearchBackfillRequest,
|
||||||
|
SearchDocument,
|
||||||
|
SearchIndexChange,
|
||||||
|
SearchResourceReference,
|
||||||
|
SearchResourceType,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.db.models import FileAsset, FileFolder, FileShare
|
||||||
|
from govoplan_files.backend.storage.share_state import effective_file_share_clause
|
||||||
|
|
||||||
|
|
||||||
|
PROVIDER_ID = "files.objects"
|
||||||
|
RESOURCE_MODELS = {
|
||||||
|
"file": FileAsset,
|
||||||
|
"folder": FileFolder,
|
||||||
|
}
|
||||||
|
READ_SCOPE = "files:file:read"
|
||||||
|
ADMIN_SCOPE = "files:file:admin"
|
||||||
|
|
||||||
|
|
||||||
|
class FilesSearchSource:
|
||||||
|
def resource_types(self) -> Sequence[SearchResourceType]:
|
||||||
|
return (
|
||||||
|
SearchResourceType(
|
||||||
|
provider_id=PROVIDER_ID,
|
||||||
|
module_id="files",
|
||||||
|
resource_type="file",
|
||||||
|
label="Files",
|
||||||
|
requires_authorization_recheck=True,
|
||||||
|
),
|
||||||
|
SearchResourceType(
|
||||||
|
provider_id=PROVIDER_ID,
|
||||||
|
module_id="files",
|
||||||
|
resource_type="folder",
|
||||||
|
label="File folders",
|
||||||
|
requires_authorization_recheck=True,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
def backfill(
|
||||||
|
self,
|
||||||
|
session: object,
|
||||||
|
*,
|
||||||
|
request: SearchBackfillRequest,
|
||||||
|
) -> SearchBackfillPage:
|
||||||
|
db = _session(session)
|
||||||
|
model = _model(request.provider_id, request.resource_type)
|
||||||
|
statement = select(model).where(
|
||||||
|
model.tenant_id == request.tenant_id,
|
||||||
|
model.deleted_at.is_(None),
|
||||||
|
)
|
||||||
|
if request.cursor:
|
||||||
|
statement = statement.where(model.id > request.cursor)
|
||||||
|
rows = list(
|
||||||
|
db.scalars(
|
||||||
|
statement.order_by(model.id).limit(request.limit + 1)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
has_more = len(rows) > request.limit
|
||||||
|
selected = rows[: request.limit]
|
||||||
|
shares = (
|
||||||
|
_shares_by_asset(db, selected)
|
||||||
|
if request.resource_type == "file"
|
||||||
|
else {}
|
||||||
|
)
|
||||||
|
high_watermark = db.scalar(
|
||||||
|
select(func.max(model.updated_at)).where(
|
||||||
|
model.tenant_id == request.tenant_id,
|
||||||
|
model.deleted_at.is_(None),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return SearchBackfillPage(
|
||||||
|
documents=tuple(
|
||||||
|
_document(
|
||||||
|
row,
|
||||||
|
resource_type=request.resource_type,
|
||||||
|
shares=shares.get(row.id, ()),
|
||||||
|
)
|
||||||
|
for row in selected
|
||||||
|
),
|
||||||
|
next_cursor=selected[-1].id if has_more and selected else None,
|
||||||
|
complete=not has_more,
|
||||||
|
high_watermark=(
|
||||||
|
high_watermark.isoformat()
|
||||||
|
if high_watermark is not None
|
||||||
|
else None
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
def authorize(
|
||||||
|
self,
|
||||||
|
session: object,
|
||||||
|
principal: object,
|
||||||
|
*,
|
||||||
|
requests: Sequence[SearchAuthorizationRequest],
|
||||||
|
) -> Mapping[str, bool]:
|
||||||
|
decisions = {item.reference.key: False for item in requests}
|
||||||
|
if not isinstance(principal, ApiPrincipal) or not (
|
||||||
|
principal.has(READ_SCOPE) or principal.has(ADMIN_SCOPE)
|
||||||
|
):
|
||||||
|
return decisions
|
||||||
|
db = _session(session)
|
||||||
|
for request in requests:
|
||||||
|
reference = request.reference
|
||||||
|
if (
|
||||||
|
reference.tenant_id != principal.tenant_id
|
||||||
|
or reference.module_id != "files"
|
||||||
|
or reference.resource_type not in RESOURCE_MODELS
|
||||||
|
):
|
||||||
|
continue
|
||||||
|
decisions[reference.key] = _can_read(
|
||||||
|
db,
|
||||||
|
principal,
|
||||||
|
reference=reference,
|
||||||
|
)
|
||||||
|
return decisions
|
||||||
|
|
||||||
|
def index_changes_for_event(
|
||||||
|
self,
|
||||||
|
session: object,
|
||||||
|
*,
|
||||||
|
event: PlatformEvent,
|
||||||
|
delivery_key: str,
|
||||||
|
) -> Sequence[SearchIndexChange]:
|
||||||
|
if (
|
||||||
|
event.module_id != "files"
|
||||||
|
or event.tenant is None
|
||||||
|
or event.resource is None
|
||||||
|
or event.resource.id is None
|
||||||
|
or event.resource.type not in RESOURCE_MODELS
|
||||||
|
):
|
||||||
|
return ()
|
||||||
|
db = _session(session)
|
||||||
|
reference = SearchResourceReference(
|
||||||
|
tenant_id=event.tenant.id,
|
||||||
|
module_id="files",
|
||||||
|
resource_type=event.resource.type,
|
||||||
|
resource_id=event.resource.id,
|
||||||
|
)
|
||||||
|
model = RESOURCE_MODELS[event.resource.type]
|
||||||
|
row = db.get(model, event.resource.id)
|
||||||
|
deleted = row is None or row.tenant_id != event.tenant.id or row.deleted_at is not None
|
||||||
|
cursor = event.event_id
|
||||||
|
document = None
|
||||||
|
if not deleted:
|
||||||
|
shares = (
|
||||||
|
tuple(_active_shares(db, row.id))
|
||||||
|
if event.resource.type == "file"
|
||||||
|
else ()
|
||||||
|
)
|
||||||
|
document = _document(
|
||||||
|
row,
|
||||||
|
resource_type=event.resource.type,
|
||||||
|
shares=shares,
|
||||||
|
change_cursor=cursor,
|
||||||
|
)
|
||||||
|
return (
|
||||||
|
SearchIndexChange(
|
||||||
|
change_id=f"{delivery_key}:{PROVIDER_ID}:{event.resource.type}",
|
||||||
|
provider_id=PROVIDER_ID,
|
||||||
|
kind="delete" if deleted else "upsert",
|
||||||
|
reference=reference,
|
||||||
|
source_revision=(
|
||||||
|
document.source_revision if document is not None else cursor
|
||||||
|
),
|
||||||
|
cursor=cursor,
|
||||||
|
document=document,
|
||||||
|
occurred_at=event.occurred_at,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def create_files_search_source(_context: ModuleContext) -> FilesSearchSource:
|
||||||
|
return FilesSearchSource()
|
||||||
|
|
||||||
|
|
||||||
|
def _model(provider_id: str, resource_type: str):
|
||||||
|
if provider_id != PROVIDER_ID or resource_type not in RESOURCE_MODELS:
|
||||||
|
raise ValueError("Unsupported Files search source.")
|
||||||
|
return RESOURCE_MODELS[resource_type]
|
||||||
|
|
||||||
|
|
||||||
|
def _document(
|
||||||
|
row: FileAsset | FileFolder,
|
||||||
|
*,
|
||||||
|
resource_type: str,
|
||||||
|
shares: Sequence[FileShare] = (),
|
||||||
|
change_cursor: str | None = None,
|
||||||
|
) -> SearchDocument:
|
||||||
|
is_file = isinstance(row, FileAsset)
|
||||||
|
title = row.filename if is_file else (row.path.rsplit("/", 1)[-1] or row.path)
|
||||||
|
path = row.display_path if is_file else row.path
|
||||||
|
owner_id = row.owner_user_id if row.owner_type == "user" else row.owner_group_id
|
||||||
|
tokens = [f"scope:{READ_SCOPE}", f"scope:{ADMIN_SCOPE}"]
|
||||||
|
if owner_id:
|
||||||
|
tokens.append(
|
||||||
|
f"membership:{owner_id}"
|
||||||
|
if row.owner_type == "user"
|
||||||
|
else f"group:{owner_id}"
|
||||||
|
)
|
||||||
|
for share in shares:
|
||||||
|
prefix = "membership" if share.target_type == "user" else share.target_type
|
||||||
|
if prefix in {"membership", "group", "tenant"}:
|
||||||
|
tokens.append(f"{prefix}:{share.target_id}")
|
||||||
|
updated_at = row.updated_at or row.created_at
|
||||||
|
revision = (
|
||||||
|
f"{row.current_version_id or 'none'}:{updated_at.isoformat()}"
|
||||||
|
if is_file
|
||||||
|
else updated_at.isoformat()
|
||||||
|
)
|
||||||
|
return SearchDocument(
|
||||||
|
tenant_id=row.tenant_id,
|
||||||
|
module_id="files",
|
||||||
|
provider_id=PROVIDER_ID,
|
||||||
|
resource_type=resource_type,
|
||||||
|
resource_id=row.id,
|
||||||
|
title=title,
|
||||||
|
url=f"/files?{resource_type}Id={quote(row.id, safe='')}",
|
||||||
|
summary=((row.description or "") if is_file else path)[:4000] or None,
|
||||||
|
body=" ".join(
|
||||||
|
value for value in (path, row.description if is_file else None) if value
|
||||||
|
)[:200_000],
|
||||||
|
keywords=(path[:200], row.owner_type[:200]),
|
||||||
|
visibility="restricted",
|
||||||
|
acl_tokens=tuple(dict.fromkeys(tokens)),
|
||||||
|
metadata={
|
||||||
|
"path": path,
|
||||||
|
"owner_type": row.owner_type,
|
||||||
|
"current_version_id": row.current_version_id if is_file else None,
|
||||||
|
},
|
||||||
|
source_revision=revision,
|
||||||
|
change_cursor=change_cursor,
|
||||||
|
source_updated_at=updated_at,
|
||||||
|
requires_authorization_recheck=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _can_read(
|
||||||
|
session: Session,
|
||||||
|
principal: ApiPrincipal,
|
||||||
|
*,
|
||||||
|
reference: SearchResourceReference,
|
||||||
|
) -> bool:
|
||||||
|
model = RESOURCE_MODELS[reference.resource_type]
|
||||||
|
row = session.get(model, reference.resource_id)
|
||||||
|
if row is None or row.tenant_id != principal.tenant_id or row.deleted_at is not None:
|
||||||
|
return False
|
||||||
|
if principal.has(ADMIN_SCOPE):
|
||||||
|
return True
|
||||||
|
user_id = str(getattr(principal.user, "id", "") or principal.membership_id or "")
|
||||||
|
if row.owner_type == "user" and row.owner_user_id == user_id:
|
||||||
|
return True
|
||||||
|
if row.owner_type == "group" and row.owner_group_id in principal.group_ids:
|
||||||
|
return True
|
||||||
|
if reference.resource_type != "file":
|
||||||
|
return False
|
||||||
|
target_clauses = [
|
||||||
|
(FileShare.target_type == "user") & (FileShare.target_id == user_id),
|
||||||
|
(FileShare.target_type == "tenant")
|
||||||
|
& (FileShare.target_id == principal.tenant_id),
|
||||||
|
]
|
||||||
|
if principal.group_ids:
|
||||||
|
target_clauses.append(
|
||||||
|
(FileShare.target_type == "group")
|
||||||
|
& (FileShare.target_id.in_(tuple(principal.group_ids)))
|
||||||
|
)
|
||||||
|
return session.scalar(
|
||||||
|
select(FileShare.id).where(
|
||||||
|
FileShare.tenant_id == principal.tenant_id,
|
||||||
|
FileShare.file_asset_id == row.id,
|
||||||
|
effective_file_share_clause(),
|
||||||
|
or_(*target_clauses),
|
||||||
|
).limit(1)
|
||||||
|
) is not None
|
||||||
|
|
||||||
|
|
||||||
|
def _active_shares(session: Session, asset_id: str) -> Sequence[FileShare]:
|
||||||
|
return tuple(
|
||||||
|
session.scalars(
|
||||||
|
select(FileShare).where(
|
||||||
|
FileShare.file_asset_id == asset_id,
|
||||||
|
effective_file_share_clause(),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _shares_by_asset(
|
||||||
|
session: Session,
|
||||||
|
rows: Sequence[FileAsset | FileFolder],
|
||||||
|
) -> dict[str, tuple[FileShare, ...]]:
|
||||||
|
asset_ids = [row.id for row in rows if isinstance(row, FileAsset)]
|
||||||
|
grouped: dict[str, list[FileShare]] = {asset_id: [] for asset_id in asset_ids}
|
||||||
|
if not asset_ids:
|
||||||
|
return {}
|
||||||
|
for share in session.scalars(
|
||||||
|
select(FileShare).where(
|
||||||
|
FileShare.file_asset_id.in_(asset_ids),
|
||||||
|
effective_file_share_clause(),
|
||||||
|
)
|
||||||
|
):
|
||||||
|
grouped.setdefault(share.file_asset_id, []).append(share)
|
||||||
|
return {key: tuple(value) for key, value in grouped.items()}
|
||||||
|
|
||||||
|
|
||||||
|
def _session(value: object) -> Session:
|
||||||
|
if not isinstance(value, Session):
|
||||||
|
raise TypeError("Files search requires a SQLAlchemy session.")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"FilesSearchSource",
|
||||||
|
"PROVIDER_ID",
|
||||||
|
"create_files_search_source",
|
||||||
|
]
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
"""Query and response assembly services used by Files routes."""
|
||||||
@@ -0,0 +1,292 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import Literal
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from govoplan_core.auth import ApiPrincipal
|
||||||
|
from govoplan_core.api.v1.schemas import DeltaDeletedItem
|
||||||
|
from govoplan_core.core.change_sequence import (
|
||||||
|
ChangeSequenceEntry,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.change_tracking import (
|
||||||
|
FILES_CONNECTOR_CREDENTIALS_COLLECTION,
|
||||||
|
FILES_CONNECTOR_POLICIES_COLLECTION,
|
||||||
|
FILES_CONNECTOR_PROFILES_COLLECTION,
|
||||||
|
FILES_CONNECTOR_SPACES_COLLECTION,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.schemas import (
|
||||||
|
FileConnectorCredentialResponse,
|
||||||
|
FileConnectorSettingsDeltaResponse,
|
||||||
|
FileConnectorPolicyResponse,
|
||||||
|
FileConnectorProfileResponse,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.db.models import FileConnectorSpace
|
||||||
|
from govoplan_files.backend.storage.connector_credential_store import (
|
||||||
|
ConnectorCredential,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.storage.connector_profiles import ConnectorProfile
|
||||||
|
from govoplan_files.backend.storage.connector_policy_store import (
|
||||||
|
connector_policy_response,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
from govoplan_files.backend.route_support import (
|
||||||
|
FILES_CONNECTOR_CREDENTIAL_RESOURCE,
|
||||||
|
FILES_CONNECTOR_PROFILE_RESOURCE,
|
||||||
|
FILES_CONNECTOR_SPACE_RESOURCE,
|
||||||
|
_can_read_disabled_connector_profiles,
|
||||||
|
_connector_deleted_item,
|
||||||
|
_connector_space_response,
|
||||||
|
_ensure_campaign_file_access,
|
||||||
|
_file_connector_settings_response_watermark,
|
||||||
|
_file_connector_settings_watermark,
|
||||||
|
_visible_connector_credentials,
|
||||||
|
_visible_connector_profiles,
|
||||||
|
_visible_connector_spaces,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _full_file_connector_settings_delta_response(
|
||||||
|
session: Session,
|
||||||
|
principal: ApiPrincipal,
|
||||||
|
*,
|
||||||
|
scope_type: str,
|
||||||
|
scope_id: str | None,
|
||||||
|
provider: str | None,
|
||||||
|
campaign_id: str | None,
|
||||||
|
include_disabled: bool,
|
||||||
|
include_inactive: bool,
|
||||||
|
owner_type: Literal["user", "group"] | None,
|
||||||
|
owner_id: str | None,
|
||||||
|
) -> FileConnectorSettingsDeltaResponse:
|
||||||
|
if campaign_id:
|
||||||
|
_ensure_campaign_file_access(session, principal, campaign_id)
|
||||||
|
profiles = _visible_connector_profiles(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
provider=provider,
|
||||||
|
campaign_id=campaign_id,
|
||||||
|
include_disabled=include_disabled
|
||||||
|
and _can_read_disabled_connector_profiles(principal),
|
||||||
|
include_admin_scopes=_can_read_disabled_connector_profiles(principal),
|
||||||
|
include_effective_policy=False,
|
||||||
|
)
|
||||||
|
credentials = _visible_connector_credentials(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
provider=provider,
|
||||||
|
include_disabled=include_disabled,
|
||||||
|
)
|
||||||
|
spaces = _visible_connector_spaces(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
owner_type=owner_type,
|
||||||
|
owner_id=owner_id,
|
||||||
|
include_inactive=include_inactive,
|
||||||
|
)
|
||||||
|
return FileConnectorSettingsDeltaResponse(
|
||||||
|
profiles=[
|
||||||
|
FileConnectorProfileResponse(**profile.to_response())
|
||||||
|
for profile in profiles
|
||||||
|
],
|
||||||
|
credentials=[
|
||||||
|
FileConnectorCredentialResponse(**credential.to_response())
|
||||||
|
for credential in credentials
|
||||||
|
],
|
||||||
|
spaces=[_connector_space_response(space) for space in spaces],
|
||||||
|
policy=FileConnectorPolicyResponse(
|
||||||
|
**connector_policy_response(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
scope_type=scope_type,
|
||||||
|
scope_id=scope_id,
|
||||||
|
)
|
||||||
|
),
|
||||||
|
changed_sections=["profiles", "credentials", "spaces", "policy"],
|
||||||
|
deleted=[],
|
||||||
|
watermark=_file_connector_settings_watermark(
|
||||||
|
session, tenant_id=principal.tenant_id
|
||||||
|
),
|
||||||
|
has_more=False,
|
||||||
|
full=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _changed_file_connector_setting_ids(
|
||||||
|
entries: list[ChangeSequenceEntry],
|
||||||
|
) -> tuple[set[str], set[str], set[str], bool]:
|
||||||
|
changed_profile_ids = {
|
||||||
|
entry.resource_id
|
||||||
|
for entry in entries
|
||||||
|
if entry.collection == FILES_CONNECTOR_PROFILES_COLLECTION
|
||||||
|
and entry.resource_type == FILES_CONNECTOR_PROFILE_RESOURCE
|
||||||
|
}
|
||||||
|
changed_credential_ids = {
|
||||||
|
entry.resource_id
|
||||||
|
for entry in entries
|
||||||
|
if entry.collection == FILES_CONNECTOR_CREDENTIALS_COLLECTION
|
||||||
|
and entry.resource_type == FILES_CONNECTOR_CREDENTIAL_RESOURCE
|
||||||
|
}
|
||||||
|
changed_space_ids = {
|
||||||
|
entry.resource_id
|
||||||
|
for entry in entries
|
||||||
|
if entry.collection == FILES_CONNECTOR_SPACES_COLLECTION
|
||||||
|
and entry.resource_type == FILES_CONNECTOR_SPACE_RESOURCE
|
||||||
|
}
|
||||||
|
policy_changed = any(
|
||||||
|
entry.collection == FILES_CONNECTOR_POLICIES_COLLECTION for entry in entries
|
||||||
|
)
|
||||||
|
return (
|
||||||
|
changed_profile_ids,
|
||||||
|
changed_credential_ids,
|
||||||
|
changed_space_ids,
|
||||||
|
policy_changed,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _file_connector_settings_changed_sections(
|
||||||
|
*,
|
||||||
|
changed_profile_ids: set[str],
|
||||||
|
changed_credential_ids: set[str],
|
||||||
|
changed_space_ids: set[str],
|
||||||
|
policy_changed: bool,
|
||||||
|
profiles: list[ConnectorProfile],
|
||||||
|
) -> list[str]:
|
||||||
|
changed_sections = []
|
||||||
|
if changed_profile_ids or any(
|
||||||
|
profile.credential_profile_id
|
||||||
|
and profile.credential_profile_id in changed_credential_ids
|
||||||
|
for profile in profiles
|
||||||
|
):
|
||||||
|
changed_sections.append("profiles")
|
||||||
|
if changed_credential_ids:
|
||||||
|
changed_sections.append("credentials")
|
||||||
|
if changed_space_ids:
|
||||||
|
changed_sections.append("spaces")
|
||||||
|
if policy_changed:
|
||||||
|
changed_sections.append("policy")
|
||||||
|
return changed_sections
|
||||||
|
|
||||||
|
|
||||||
|
def _file_connector_settings_deleted_items(
|
||||||
|
entries: list[ChangeSequenceEntry],
|
||||||
|
*,
|
||||||
|
visible_profiles: dict[str, ConnectorProfile],
|
||||||
|
visible_credentials: dict[str, ConnectorCredential],
|
||||||
|
visible_spaces: dict[str, FileConnectorSpace],
|
||||||
|
) -> list[DeltaDeletedItem]:
|
||||||
|
return [
|
||||||
|
_connector_deleted_item(entry)
|
||||||
|
for entry in entries
|
||||||
|
if (
|
||||||
|
entry.resource_type == FILES_CONNECTOR_PROFILE_RESOURCE
|
||||||
|
and entry.resource_id not in visible_profiles
|
||||||
|
)
|
||||||
|
or (
|
||||||
|
entry.resource_type == FILES_CONNECTOR_CREDENTIAL_RESOURCE
|
||||||
|
and entry.resource_id not in visible_credentials
|
||||||
|
)
|
||||||
|
or (
|
||||||
|
entry.resource_type == FILES_CONNECTOR_SPACE_RESOURCE
|
||||||
|
and entry.resource_id not in visible_spaces
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _incremental_file_connector_settings_delta_response(
|
||||||
|
session: Session,
|
||||||
|
principal: ApiPrincipal,
|
||||||
|
*,
|
||||||
|
entries: list[ChangeSequenceEntry],
|
||||||
|
has_more: bool,
|
||||||
|
scope_type: str,
|
||||||
|
scope_id: str | None,
|
||||||
|
provider: str | None,
|
||||||
|
campaign_id: str | None,
|
||||||
|
include_disabled: bool,
|
||||||
|
include_inactive: bool,
|
||||||
|
owner_type: Literal["user", "group"] | None,
|
||||||
|
owner_id: str | None,
|
||||||
|
) -> FileConnectorSettingsDeltaResponse:
|
||||||
|
changed_profile_ids, changed_credential_ids, changed_space_ids, policy_changed = (
|
||||||
|
_changed_file_connector_setting_ids(entries)
|
||||||
|
)
|
||||||
|
profiles = _visible_connector_profiles(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
provider=provider,
|
||||||
|
campaign_id=campaign_id,
|
||||||
|
include_disabled=include_disabled
|
||||||
|
and _can_read_disabled_connector_profiles(principal),
|
||||||
|
include_admin_scopes=_can_read_disabled_connector_profiles(principal),
|
||||||
|
include_effective_policy=False,
|
||||||
|
)
|
||||||
|
visible_profiles = {
|
||||||
|
profile.id: profile
|
||||||
|
for profile in profiles
|
||||||
|
if profile.id in changed_profile_ids
|
||||||
|
or (
|
||||||
|
profile.credential_profile_id
|
||||||
|
and profile.credential_profile_id in changed_credential_ids
|
||||||
|
)
|
||||||
|
}
|
||||||
|
credentials = _visible_connector_credentials(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
provider=provider,
|
||||||
|
include_disabled=include_disabled,
|
||||||
|
)
|
||||||
|
visible_credentials = {
|
||||||
|
credential.id: credential
|
||||||
|
for credential in credentials
|
||||||
|
if credential.id in changed_credential_ids
|
||||||
|
}
|
||||||
|
spaces = _visible_connector_spaces(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
owner_type=owner_type,
|
||||||
|
owner_id=owner_id,
|
||||||
|
include_inactive=include_inactive,
|
||||||
|
)
|
||||||
|
visible_spaces = {
|
||||||
|
space.id: space for space in spaces if space.id in changed_space_ids
|
||||||
|
}
|
||||||
|
return FileConnectorSettingsDeltaResponse(
|
||||||
|
profiles=[
|
||||||
|
FileConnectorProfileResponse(**profile.to_response())
|
||||||
|
for profile in visible_profiles.values()
|
||||||
|
],
|
||||||
|
credentials=[
|
||||||
|
FileConnectorCredentialResponse(**credential.to_response())
|
||||||
|
for credential in visible_credentials.values()
|
||||||
|
],
|
||||||
|
spaces=[_connector_space_response(space) for space in visible_spaces.values()],
|
||||||
|
policy=FileConnectorPolicyResponse(
|
||||||
|
**connector_policy_response(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
scope_type=scope_type,
|
||||||
|
scope_id=scope_id,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if policy_changed
|
||||||
|
else None,
|
||||||
|
changed_sections=_file_connector_settings_changed_sections(
|
||||||
|
changed_profile_ids=changed_profile_ids,
|
||||||
|
changed_credential_ids=changed_credential_ids,
|
||||||
|
changed_space_ids=changed_space_ids,
|
||||||
|
policy_changed=policy_changed,
|
||||||
|
profiles=profiles,
|
||||||
|
),
|
||||||
|
deleted=_file_connector_settings_deleted_items(
|
||||||
|
entries,
|
||||||
|
visible_profiles=visible_profiles,
|
||||||
|
visible_credentials=visible_credentials,
|
||||||
|
visible_spaces=visible_spaces,
|
||||||
|
),
|
||||||
|
watermark=_file_connector_settings_response_watermark(
|
||||||
|
session, tenant_id=principal.tenant_id, entries=entries, has_more=has_more
|
||||||
|
),
|
||||||
|
has_more=has_more,
|
||||||
|
full=False,
|
||||||
|
)
|
||||||
@@ -0,0 +1,620 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
from typing import Literal
|
||||||
|
from fastapi import HTTPException, status
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from govoplan_core.auth import ApiPrincipal
|
||||||
|
from govoplan_core.api.v1.schemas import DeltaDeletedItem
|
||||||
|
from govoplan_core.core.change_sequence import (
|
||||||
|
ChangeSequenceEntry,
|
||||||
|
decode_sequence_watermark,
|
||||||
|
encode_sequence_watermark,
|
||||||
|
max_sequence_id,
|
||||||
|
sequence_entries_since,
|
||||||
|
sequence_watermark_is_expired,
|
||||||
|
)
|
||||||
|
from govoplan_core.core.pagination import (
|
||||||
|
KeysetCursorError,
|
||||||
|
decode_keyset_cursor,
|
||||||
|
encode_keyset_cursor,
|
||||||
|
keyset_query_fingerprint,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.change_tracking import (
|
||||||
|
FILES_ASSETS_COLLECTION,
|
||||||
|
FILES_FOLDERS_COLLECTION,
|
||||||
|
FILES_MODULE_ID,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.schemas import (
|
||||||
|
FileDeltaResponse,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.db.models import FileAsset, FileFolder, FileShare
|
||||||
|
from govoplan_files.backend.storage.access import user_group_ids
|
||||||
|
from govoplan_files.backend.storage.files import (
|
||||||
|
list_assets_for_user,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.storage.folders import list_folders_for_user
|
||||||
|
from govoplan_files.backend.storage.share_state import effective_file_share_clause
|
||||||
|
|
||||||
|
|
||||||
|
from govoplan_files.backend.route_support import (
|
||||||
|
_asset_list_response,
|
||||||
|
_folder_response,
|
||||||
|
_is_admin,
|
||||||
|
)
|
||||||
|
|
||||||
|
_FILES_DELTA_COLLECTIONS = (FILES_ASSETS_COLLECTION, FILES_FOLDERS_COLLECTION)
|
||||||
|
FILES_LIST_CURSOR_SCOPE = "files.list.v1"
|
||||||
|
FOLDERS_LIST_CURSOR_SCOPE = "files.folders.list.v1"
|
||||||
|
DEFAULT_FILE_LIST_PAGE_SIZE = 500
|
||||||
|
|
||||||
|
|
||||||
|
def _cursor_http_error(exc: Exception) -> HTTPException:
|
||||||
|
return HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc))
|
||||||
|
|
||||||
|
|
||||||
|
def _cursor_page_size(
|
||||||
|
scope: str, cursor: str | None, explicit_page_size: int | None
|
||||||
|
) -> int | None:
|
||||||
|
if explicit_page_size is not None:
|
||||||
|
return explicit_page_size
|
||||||
|
if not cursor:
|
||||||
|
return DEFAULT_FILE_LIST_PAGE_SIZE
|
||||||
|
try:
|
||||||
|
values = decode_keyset_cursor(scope, cursor)
|
||||||
|
except KeysetCursorError as exc:
|
||||||
|
raise _cursor_http_error(exc) from exc
|
||||||
|
raw_page_size = values.get("page_size")
|
||||||
|
if not isinstance(raw_page_size, int) or raw_page_size < 1 or raw_page_size > 1000:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid pagination cursor"
|
||||||
|
)
|
||||||
|
return raw_page_size
|
||||||
|
|
||||||
|
|
||||||
|
def _files_list_fingerprint(
|
||||||
|
principal: ApiPrincipal,
|
||||||
|
*,
|
||||||
|
owner_type: Literal["user", "group"] | None,
|
||||||
|
owner_id: str | None,
|
||||||
|
campaign_id: str | None,
|
||||||
|
path_prefix: str | None,
|
||||||
|
campaign_usage: Literal["linked", "unlinked"] | None,
|
||||||
|
audit_relevant: bool | None,
|
||||||
|
page_size: int,
|
||||||
|
) -> str:
|
||||||
|
return keyset_query_fingerprint(
|
||||||
|
FILES_LIST_CURSOR_SCOPE,
|
||||||
|
{
|
||||||
|
"tenant_id": principal.tenant_id,
|
||||||
|
"actor": "admin" if _is_admin(principal) else principal.user.id,
|
||||||
|
"owner_type": owner_type,
|
||||||
|
"owner_id": owner_id,
|
||||||
|
"campaign_id": campaign_id,
|
||||||
|
"path_prefix": path_prefix or "",
|
||||||
|
"campaign_usage": campaign_usage or "",
|
||||||
|
"audit_relevant": audit_relevant,
|
||||||
|
"sort": "display_path.asc,updated_at.desc,id.asc",
|
||||||
|
"page_size": page_size,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _folders_list_fingerprint(
|
||||||
|
principal: ApiPrincipal,
|
||||||
|
*,
|
||||||
|
owner_type: Literal["user", "group"],
|
||||||
|
owner_id: str,
|
||||||
|
page_size: int,
|
||||||
|
) -> str:
|
||||||
|
return keyset_query_fingerprint(
|
||||||
|
FOLDERS_LIST_CURSOR_SCOPE,
|
||||||
|
{
|
||||||
|
"tenant_id": principal.tenant_id,
|
||||||
|
"actor": "admin" if _is_admin(principal) else principal.user.id,
|
||||||
|
"owner_type": owner_type,
|
||||||
|
"owner_id": owner_id,
|
||||||
|
"sort": "path.asc,id.asc",
|
||||||
|
"page_size": page_size,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _file_cursor_values(
|
||||||
|
cursor: str | None, *, fingerprint: str
|
||||||
|
) -> tuple[str | None, datetime | None, str | None]:
|
||||||
|
try:
|
||||||
|
values = decode_keyset_cursor(
|
||||||
|
FILES_LIST_CURSOR_SCOPE, cursor, fingerprint=fingerprint
|
||||||
|
)
|
||||||
|
except KeysetCursorError as exc:
|
||||||
|
raise _cursor_http_error(exc) from exc
|
||||||
|
if values is None:
|
||||||
|
return None, None, None
|
||||||
|
display_path = values.get("display_path")
|
||||||
|
updated_at = values.get("updated_at")
|
||||||
|
asset_id = values.get("id")
|
||||||
|
if (
|
||||||
|
not isinstance(display_path, str)
|
||||||
|
or not isinstance(updated_at, str)
|
||||||
|
or not isinstance(asset_id, str)
|
||||||
|
):
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid pagination cursor"
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
parsed_updated_at = datetime.fromisoformat(updated_at)
|
||||||
|
except ValueError as exc:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid pagination cursor"
|
||||||
|
) from exc
|
||||||
|
return display_path, parsed_updated_at, asset_id
|
||||||
|
|
||||||
|
|
||||||
|
def _folder_cursor_values(
|
||||||
|
cursor: str | None, *, fingerprint: str
|
||||||
|
) -> tuple[str | None, str | None]:
|
||||||
|
try:
|
||||||
|
values = decode_keyset_cursor(
|
||||||
|
FOLDERS_LIST_CURSOR_SCOPE, cursor, fingerprint=fingerprint
|
||||||
|
)
|
||||||
|
except KeysetCursorError as exc:
|
||||||
|
raise _cursor_http_error(exc) from exc
|
||||||
|
if values is None:
|
||||||
|
return None, None
|
||||||
|
folder_path = values.get("path")
|
||||||
|
folder_id = values.get("id")
|
||||||
|
if not isinstance(folder_path, str) or not isinstance(folder_id, str):
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid pagination cursor"
|
||||||
|
)
|
||||||
|
return folder_path, folder_id
|
||||||
|
|
||||||
|
|
||||||
|
def _next_file_list_cursor(
|
||||||
|
principal: ApiPrincipal,
|
||||||
|
assets: list[FileAsset],
|
||||||
|
*,
|
||||||
|
owner_type: Literal["user", "group"] | None,
|
||||||
|
owner_id: str | None,
|
||||||
|
campaign_id: str | None,
|
||||||
|
path_prefix: str | None,
|
||||||
|
campaign_usage: Literal["linked", "unlinked"] | None,
|
||||||
|
audit_relevant: bool | None,
|
||||||
|
page_size: int,
|
||||||
|
has_more: bool,
|
||||||
|
) -> str | None:
|
||||||
|
if not has_more or not assets:
|
||||||
|
return None
|
||||||
|
last = assets[-1]
|
||||||
|
return encode_keyset_cursor(
|
||||||
|
FILES_LIST_CURSOR_SCOPE,
|
||||||
|
fingerprint=_files_list_fingerprint(
|
||||||
|
principal,
|
||||||
|
owner_type=owner_type,
|
||||||
|
owner_id=owner_id,
|
||||||
|
campaign_id=campaign_id,
|
||||||
|
path_prefix=path_prefix,
|
||||||
|
campaign_usage=campaign_usage,
|
||||||
|
audit_relevant=audit_relevant,
|
||||||
|
page_size=page_size,
|
||||||
|
),
|
||||||
|
values={
|
||||||
|
"display_path": last.display_path,
|
||||||
|
"updated_at": last.updated_at,
|
||||||
|
"id": last.id,
|
||||||
|
"page_size": page_size,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _next_folder_list_cursor(
|
||||||
|
principal: ApiPrincipal,
|
||||||
|
folders: list[FileFolder],
|
||||||
|
*,
|
||||||
|
owner_type: Literal["user", "group"],
|
||||||
|
owner_id: str,
|
||||||
|
page_size: int,
|
||||||
|
has_more: bool,
|
||||||
|
) -> str | None:
|
||||||
|
if not has_more or not folders:
|
||||||
|
return None
|
||||||
|
last = folders[-1]
|
||||||
|
return encode_keyset_cursor(
|
||||||
|
FOLDERS_LIST_CURSOR_SCOPE,
|
||||||
|
fingerprint=_folders_list_fingerprint(
|
||||||
|
principal, owner_type=owner_type, owner_id=owner_id, page_size=page_size
|
||||||
|
),
|
||||||
|
values={"path": last.path, "id": last.id, "page_size": page_size},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _files_delta_watermark(session: Session, tenant_id: str) -> str:
|
||||||
|
return encode_sequence_watermark(
|
||||||
|
max_sequence_id(
|
||||||
|
session,
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
module_id=FILES_MODULE_ID,
|
||||||
|
collections=_FILES_DELTA_COLLECTIONS,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _full_file_delta_response(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
principal: ApiPrincipal,
|
||||||
|
owner_type: Literal["user", "group"] | None,
|
||||||
|
owner_id: str | None,
|
||||||
|
campaign_id: str | None,
|
||||||
|
path_prefix: str | None,
|
||||||
|
) -> FileDeltaResponse:
|
||||||
|
assets = list_assets_for_user(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
owner_type=owner_type,
|
||||||
|
owner_id=owner_id,
|
||||||
|
campaign_id=campaign_id,
|
||||||
|
path_prefix=path_prefix,
|
||||||
|
is_admin=_is_admin(principal),
|
||||||
|
)
|
||||||
|
folders = _visible_folders_for_delta(
|
||||||
|
session,
|
||||||
|
principal=principal,
|
||||||
|
owner_type=owner_type,
|
||||||
|
owner_id=owner_id,
|
||||||
|
path_prefix=path_prefix,
|
||||||
|
)
|
||||||
|
return FileDeltaResponse(
|
||||||
|
files=_asset_list_response(session, assets, include_shares=True),
|
||||||
|
folders=[_folder_response(folder) for folder in folders],
|
||||||
|
deleted=[],
|
||||||
|
watermark=_files_delta_watermark(session, principal.tenant_id),
|
||||||
|
has_more=False,
|
||||||
|
full=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _visible_folders_for_delta(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
principal: ApiPrincipal,
|
||||||
|
owner_type: Literal["user", "group"] | None,
|
||||||
|
owner_id: str | None,
|
||||||
|
path_prefix: str | None,
|
||||||
|
) -> list[FileFolder]:
|
||||||
|
if not owner_type or not owner_id:
|
||||||
|
return []
|
||||||
|
folders = list_folders_for_user(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
owner_type=owner_type,
|
||||||
|
owner_id=owner_id,
|
||||||
|
is_admin=_is_admin(principal),
|
||||||
|
)
|
||||||
|
if not path_prefix:
|
||||||
|
return folders
|
||||||
|
normalized = path_prefix.strip().strip("/")
|
||||||
|
if not normalized:
|
||||||
|
return folders
|
||||||
|
return [
|
||||||
|
folder
|
||||||
|
for folder in folders
|
||||||
|
if folder.path == normalized or folder.path.startswith(f"{normalized}/")
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _entry_path_matches(
|
||||||
|
entry_payload: dict[str, object], path_prefix: str | None
|
||||||
|
) -> bool:
|
||||||
|
if not path_prefix:
|
||||||
|
return True
|
||||||
|
normalized = path_prefix.strip().strip("/")
|
||||||
|
if not normalized:
|
||||||
|
return True
|
||||||
|
for key in ("path", "previous_path"):
|
||||||
|
value = entry_payload.get(key)
|
||||||
|
if isinstance(value, str) and (
|
||||||
|
value == normalized or value.startswith(f"{normalized}/")
|
||||||
|
):
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _entry_owner_matches(
|
||||||
|
entry_payload: dict[str, object],
|
||||||
|
owner_type: Literal["user", "group"] | None,
|
||||||
|
owner_id: str | None,
|
||||||
|
) -> bool:
|
||||||
|
if not owner_type or not owner_id:
|
||||||
|
return True
|
||||||
|
return (
|
||||||
|
entry_payload.get("owner_type") == owner_type
|
||||||
|
and entry_payload.get("owner_id") == owner_id
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _entry_campaign_matches(session: Session, entry, campaign_id: str | None) -> bool:
|
||||||
|
if not campaign_id:
|
||||||
|
return True
|
||||||
|
payload = entry.payload or {}
|
||||||
|
if (
|
||||||
|
payload.get("share_target_type") == "campaign"
|
||||||
|
and payload.get("share_target_id") == campaign_id
|
||||||
|
):
|
||||||
|
return True
|
||||||
|
if entry.resource_type != "file":
|
||||||
|
return False
|
||||||
|
return (
|
||||||
|
session.query(FileShare)
|
||||||
|
.filter(
|
||||||
|
FileShare.tenant_id == entry.tenant_id,
|
||||||
|
FileShare.file_asset_id == entry.resource_id,
|
||||||
|
FileShare.target_type == "campaign",
|
||||||
|
FileShare.target_id == campaign_id,
|
||||||
|
effective_file_share_clause(),
|
||||||
|
)
|
||||||
|
.first()
|
||||||
|
is not None
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _principal_group_ids_for_delta(
|
||||||
|
session: Session, principal: ApiPrincipal
|
||||||
|
) -> set[str]:
|
||||||
|
cache = session.info.setdefault("files_delta_group_ids", {})
|
||||||
|
key = (principal.tenant_id, principal.user.id)
|
||||||
|
if key not in cache:
|
||||||
|
cache[key] = set(
|
||||||
|
user_group_ids(
|
||||||
|
session, tenant_id=principal.tenant_id, user_id=principal.user.id
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return cache[key]
|
||||||
|
|
||||||
|
|
||||||
|
def _entry_subject_matches_principal(
|
||||||
|
session: Session, principal: ApiPrincipal, entry_payload: dict[str, object]
|
||||||
|
) -> bool:
|
||||||
|
if _is_admin(principal):
|
||||||
|
return True
|
||||||
|
owner_type = entry_payload.get("owner_type")
|
||||||
|
owner_id = entry_payload.get("owner_id")
|
||||||
|
if owner_type == "user" and owner_id == principal.user.id:
|
||||||
|
return True
|
||||||
|
if owner_type == "group" and isinstance(owner_id, str):
|
||||||
|
if owner_id in _principal_group_ids_for_delta(session, principal):
|
||||||
|
return True
|
||||||
|
share_target_type = entry_payload.get("share_target_type")
|
||||||
|
share_target_id = entry_payload.get("share_target_id")
|
||||||
|
if share_target_type == "user" and share_target_id == principal.user.id:
|
||||||
|
return True
|
||||||
|
if share_target_type == "tenant" and share_target_id == principal.tenant_id:
|
||||||
|
return True
|
||||||
|
if share_target_type == "group" and isinstance(share_target_id, str):
|
||||||
|
if share_target_id in _principal_group_ids_for_delta(session, principal):
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _entry_matches_delta_scope(
|
||||||
|
session: Session,
|
||||||
|
principal: ApiPrincipal,
|
||||||
|
entry,
|
||||||
|
*,
|
||||||
|
owner_type: Literal["user", "group"] | None,
|
||||||
|
owner_id: str | None,
|
||||||
|
campaign_id: str | None,
|
||||||
|
path_prefix: str | None,
|
||||||
|
) -> bool:
|
||||||
|
payload = entry.payload or {}
|
||||||
|
if (
|
||||||
|
not owner_type
|
||||||
|
and not campaign_id
|
||||||
|
and not _entry_subject_matches_principal(session, principal, payload)
|
||||||
|
):
|
||||||
|
return False
|
||||||
|
return (
|
||||||
|
_entry_owner_matches(payload, owner_type, owner_id)
|
||||||
|
and _entry_path_matches(payload, path_prefix)
|
||||||
|
and _entry_campaign_matches(session, entry, campaign_id)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _decode_files_delta_watermark(since: str) -> int:
|
||||||
|
try:
|
||||||
|
return decode_sequence_watermark(since)
|
||||||
|
except ValueError as exc:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
|
||||||
|
def _changed_delta_resource_ids(
|
||||||
|
entries: list[ChangeSequenceEntry],
|
||||||
|
) -> tuple[list[str], list[str]]:
|
||||||
|
file_ids = list(
|
||||||
|
dict.fromkeys(
|
||||||
|
entry.resource_id for entry in entries if entry.resource_type == "file"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
folder_ids = list(
|
||||||
|
dict.fromkeys(
|
||||||
|
entry.resource_id for entry in entries if entry.resource_type == "folder"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return file_ids, folder_ids
|
||||||
|
|
||||||
|
|
||||||
|
def _visible_assets_for_delta(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
principal: ApiPrincipal,
|
||||||
|
owner_type: Literal["user", "group"] | None,
|
||||||
|
owner_id: str | None,
|
||||||
|
campaign_id: str | None,
|
||||||
|
path_prefix: str | None,
|
||||||
|
changed_file_ids: list[str],
|
||||||
|
) -> dict[str, FileAsset]:
|
||||||
|
return {
|
||||||
|
asset.id: asset
|
||||||
|
for asset in list_assets_for_user(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
user_id=principal.user.id,
|
||||||
|
owner_type=owner_type,
|
||||||
|
owner_id=owner_id,
|
||||||
|
campaign_id=campaign_id,
|
||||||
|
path_prefix=path_prefix,
|
||||||
|
is_admin=_is_admin(principal),
|
||||||
|
)
|
||||||
|
if asset.id in changed_file_ids
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _changed_visible_folders_for_delta(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
principal: ApiPrincipal,
|
||||||
|
owner_type: Literal["user", "group"] | None,
|
||||||
|
owner_id: str | None,
|
||||||
|
path_prefix: str | None,
|
||||||
|
changed_folder_ids: list[str],
|
||||||
|
) -> dict[str, FileFolder]:
|
||||||
|
return {
|
||||||
|
folder.id: folder
|
||||||
|
for folder in _visible_folders_for_delta(
|
||||||
|
session,
|
||||||
|
principal=principal,
|
||||||
|
owner_type=owner_type,
|
||||||
|
owner_id=owner_id,
|
||||||
|
path_prefix=path_prefix,
|
||||||
|
)
|
||||||
|
if folder.id in changed_folder_ids
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _deleted_delta_items(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
principal: ApiPrincipal,
|
||||||
|
entries: list[ChangeSequenceEntry],
|
||||||
|
visible_assets: dict[str, FileAsset],
|
||||||
|
visible_folders: dict[str, FileFolder],
|
||||||
|
owner_type: Literal["user", "group"] | None,
|
||||||
|
owner_id: str | None,
|
||||||
|
campaign_id: str | None,
|
||||||
|
path_prefix: str | None,
|
||||||
|
) -> list[DeltaDeletedItem]:
|
||||||
|
deleted: dict[tuple[str, str], DeltaDeletedItem] = {}
|
||||||
|
for entry in entries:
|
||||||
|
if entry.resource_type == "file" and entry.resource_id in visible_assets:
|
||||||
|
continue
|
||||||
|
if entry.resource_type == "folder" and entry.resource_id in visible_folders:
|
||||||
|
continue
|
||||||
|
if not _entry_matches_delta_scope(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
entry,
|
||||||
|
owner_type=owner_type,
|
||||||
|
owner_id=owner_id,
|
||||||
|
campaign_id=campaign_id,
|
||||||
|
path_prefix=path_prefix,
|
||||||
|
):
|
||||||
|
continue
|
||||||
|
deleted[(entry.resource_type, entry.resource_id)] = DeltaDeletedItem(
|
||||||
|
id=entry.resource_id,
|
||||||
|
resource_type=entry.resource_type,
|
||||||
|
revision=encode_sequence_watermark(entry.id),
|
||||||
|
deleted_at=entry.created_at if entry.operation == "deleted" else None,
|
||||||
|
)
|
||||||
|
return list(deleted.values())
|
||||||
|
|
||||||
|
|
||||||
|
def _files_delta_response(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
principal: ApiPrincipal,
|
||||||
|
owner_type: Literal["user", "group"] | None,
|
||||||
|
owner_id: str | None,
|
||||||
|
campaign_id: str | None,
|
||||||
|
path_prefix: str | None,
|
||||||
|
since: str,
|
||||||
|
limit: int,
|
||||||
|
) -> FileDeltaResponse:
|
||||||
|
since_sequence = _decode_files_delta_watermark(since)
|
||||||
|
if sequence_watermark_is_expired(
|
||||||
|
session,
|
||||||
|
since=since_sequence,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
module_id=FILES_MODULE_ID,
|
||||||
|
collections=_FILES_DELTA_COLLECTIONS,
|
||||||
|
):
|
||||||
|
return _full_file_delta_response(
|
||||||
|
session,
|
||||||
|
principal=principal,
|
||||||
|
owner_type=owner_type,
|
||||||
|
owner_id=owner_id,
|
||||||
|
campaign_id=campaign_id,
|
||||||
|
path_prefix=path_prefix,
|
||||||
|
)
|
||||||
|
|
||||||
|
entries_plus_one = sequence_entries_since(
|
||||||
|
session,
|
||||||
|
since=since_sequence,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
module_id=FILES_MODULE_ID,
|
||||||
|
collections=_FILES_DELTA_COLLECTIONS,
|
||||||
|
limit=limit + 1,
|
||||||
|
)
|
||||||
|
has_more = len(entries_plus_one) > limit
|
||||||
|
entries = entries_plus_one[:limit]
|
||||||
|
|
||||||
|
changed_file_ids, changed_folder_ids = _changed_delta_resource_ids(entries)
|
||||||
|
visible_assets = _visible_assets_for_delta(
|
||||||
|
session,
|
||||||
|
principal=principal,
|
||||||
|
owner_type=owner_type,
|
||||||
|
owner_id=owner_id,
|
||||||
|
campaign_id=campaign_id,
|
||||||
|
path_prefix=path_prefix,
|
||||||
|
changed_file_ids=changed_file_ids,
|
||||||
|
)
|
||||||
|
visible_folders = _changed_visible_folders_for_delta(
|
||||||
|
session,
|
||||||
|
principal=principal,
|
||||||
|
owner_type=owner_type,
|
||||||
|
owner_id=owner_id,
|
||||||
|
path_prefix=path_prefix,
|
||||||
|
changed_folder_ids=changed_folder_ids,
|
||||||
|
)
|
||||||
|
deleted = _deleted_delta_items(
|
||||||
|
session,
|
||||||
|
principal=principal,
|
||||||
|
entries=entries,
|
||||||
|
visible_assets=visible_assets,
|
||||||
|
visible_folders=visible_folders,
|
||||||
|
owner_type=owner_type,
|
||||||
|
owner_id=owner_id,
|
||||||
|
campaign_id=campaign_id,
|
||||||
|
path_prefix=path_prefix,
|
||||||
|
)
|
||||||
|
|
||||||
|
watermark = (
|
||||||
|
encode_sequence_watermark(entries[-1].id)
|
||||||
|
if has_more and entries
|
||||||
|
else _files_delta_watermark(session, principal.tenant_id)
|
||||||
|
)
|
||||||
|
return FileDeltaResponse(
|
||||||
|
files=_asset_list_response(
|
||||||
|
session, list(visible_assets.values()), include_shares=True
|
||||||
|
),
|
||||||
|
folders=[_folder_response(folder) for folder in visible_folders.values()],
|
||||||
|
deleted=deleted,
|
||||||
|
watermark=watermark,
|
||||||
|
has_more=has_more,
|
||||||
|
full=False,
|
||||||
|
)
|
||||||
@@ -1,55 +1,91 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import mimetypes
|
import mimetypes
|
||||||
|
import re
|
||||||
|
import stat
|
||||||
|
import tarfile
|
||||||
import zipfile
|
import zipfile
|
||||||
|
from dataclasses import dataclass
|
||||||
from io import BytesIO
|
from io import BytesIO
|
||||||
from os import PathLike
|
from os import PathLike
|
||||||
from pathlib import Path
|
from pathlib import Path, PurePosixPath
|
||||||
from typing import Any, Iterable
|
from typing import Any, BinaryIO, Iterable, Literal
|
||||||
|
|
||||||
|
import pyzipper
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from govoplan_files.backend.db.models import FileAsset
|
from govoplan_files.backend.db.models import FileAsset
|
||||||
from govoplan_files.backend.storage.common import FileConflictResolution, FileStorageError, UploadedStoredFile
|
from govoplan_files.backend.storage.backends import (
|
||||||
from govoplan_files.backend.storage.backends import StorageBackendError, get_storage_backend
|
StorageBackendError,
|
||||||
from govoplan_files.backend.storage.files import create_file_asset, current_versions_and_blobs
|
get_storage_backend,
|
||||||
from govoplan_files.backend.storage.paths import filename_from_path, normalize_folder, normalize_logical_path
|
)
|
||||||
|
from govoplan_files.backend.storage.common import (
|
||||||
|
FileConflictResolution,
|
||||||
|
FileStorageError,
|
||||||
|
UploadedStoredFile,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.storage.files import (
|
||||||
|
create_file_asset,
|
||||||
|
current_versions_and_blobs,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.storage.paths import (
|
||||||
|
filename_from_path,
|
||||||
|
normalize_folder,
|
||||||
|
normalize_logical_path,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
_ZIP_READ_CHUNK_SIZE = 1024 * 1024
|
_ARCHIVE_READ_CHUNK_SIZE = 1024 * 1024
|
||||||
ZIP_UPLOAD_MAX_FILES = 1000
|
_WINDOWS_DRIVE_RE = re.compile(r"^[A-Za-z]:")
|
||||||
|
ARCHIVE_UPLOAD_MAX_ENTRIES = 10_000
|
||||||
|
# Kept for callers and documentation using the former ZIP-specific name.
|
||||||
|
ZIP_UPLOAD_MAX_FILES = ARCHIVE_UPLOAD_MAX_ENTRIES
|
||||||
|
SUPPORTED_ARCHIVE_SUFFIXES = (
|
||||||
|
".tar.bz2",
|
||||||
|
".tar.gz",
|
||||||
|
".tar.xz",
|
||||||
|
".tbz2",
|
||||||
|
".tgz",
|
||||||
|
".txz",
|
||||||
|
".tar",
|
||||||
|
".zip",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _read_zip_member(
|
class ArchivePasswordError(FileStorageError):
|
||||||
archive: zipfile.ZipFile,
|
pass
|
||||||
info: zipfile.ZipInfo,
|
|
||||||
*,
|
|
||||||
max_file_bytes: int,
|
|
||||||
max_total_bytes: int,
|
|
||||||
current_total: int,
|
|
||||||
) -> tuple[bytes, int]:
|
|
||||||
parts: list[bytes] = []
|
|
||||||
actual_size = 0
|
|
||||||
with archive.open(info) as source:
|
|
||||||
while True:
|
|
||||||
read_size = min(_ZIP_READ_CHUNK_SIZE, max_file_bytes + 1 - actual_size)
|
|
||||||
chunk = source.read(read_size)
|
|
||||||
if not chunk:
|
|
||||||
break
|
|
||||||
actual_size += len(chunk)
|
|
||||||
if actual_size > max_file_bytes:
|
|
||||||
raise FileStorageError(f"ZIP member {info.filename!r} exceeds per-file limit")
|
|
||||||
if current_total + actual_size > max_total_bytes:
|
|
||||||
raise FileStorageError("ZIP is too large after extraction")
|
|
||||||
parts.append(chunk)
|
|
||||||
return b"".join(parts), current_total + actual_size
|
|
||||||
|
|
||||||
|
|
||||||
def create_zip_file(session: Session, assets: Iterable[FileAsset], output_path: str | Path) -> None:
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class ArchiveEntry:
|
||||||
|
path: str
|
||||||
|
kind: Literal["file", "directory"]
|
||||||
|
size_bytes: int
|
||||||
|
compressed_size_bytes: int | None = None
|
||||||
|
encrypted: bool = False
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class ArchiveInspection:
|
||||||
|
archive_format: str
|
||||||
|
entries: tuple[ArchiveEntry, ...]
|
||||||
|
file_count: int
|
||||||
|
directory_count: int
|
||||||
|
expanded_size_bytes: int
|
||||||
|
compressed_size_bytes: int
|
||||||
|
requires_password: bool
|
||||||
|
password_verified: bool
|
||||||
|
|
||||||
|
|
||||||
|
def create_zip_file(
|
||||||
|
session: Session, assets: Iterable[FileAsset], output_path: str | Path
|
||||||
|
) -> None:
|
||||||
backend = get_storage_backend()
|
backend = get_storage_backend()
|
||||||
asset_list = list(assets)
|
asset_list = list(assets)
|
||||||
version_blobs = current_versions_and_blobs(session, asset_list)
|
version_blobs = current_versions_and_blobs(session, asset_list)
|
||||||
with zipfile.ZipFile(output_path, mode="w", compression=zipfile.ZIP_DEFLATED) as archive:
|
with zipfile.ZipFile(
|
||||||
|
output_path, mode="w", compression=zipfile.ZIP_DEFLATED
|
||||||
|
) as archive:
|
||||||
for asset in asset_list:
|
for asset in asset_list:
|
||||||
_version, blob = version_blobs[asset.id]
|
_version, blob = version_blobs[asset.id]
|
||||||
info = zipfile.ZipInfo(asset.display_path)
|
info = zipfile.ZipInfo(asset.display_path)
|
||||||
@@ -63,6 +99,147 @@ def create_zip_file(session: Session, assets: Iterable[FileAsset], output_path:
|
|||||||
raise FileStorageError(str(exc)) from exc
|
raise FileStorageError(str(exc)) from exc
|
||||||
|
|
||||||
|
|
||||||
|
def archive_format_for_filename(filename: str) -> str:
|
||||||
|
lowered = filename.strip().casefold()
|
||||||
|
if lowered.endswith(".zip"):
|
||||||
|
return "zip"
|
||||||
|
if lowered.endswith((".tar.gz", ".tgz")):
|
||||||
|
return "tar.gz"
|
||||||
|
if lowered.endswith((".tar.bz2", ".tbz2")):
|
||||||
|
return "tar.bz2"
|
||||||
|
if lowered.endswith((".tar.xz", ".txz")):
|
||||||
|
return "tar.xz"
|
||||||
|
if lowered.endswith(".tar"):
|
||||||
|
return "tar"
|
||||||
|
raise FileStorageError(
|
||||||
|
"Unsupported archive format. Use ZIP, TAR, TAR.GZ, TAR.BZ2, or TAR.XZ."
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def is_supported_archive_filename(filename: str) -> bool:
|
||||||
|
try:
|
||||||
|
archive_format_for_filename(filename)
|
||||||
|
except FileStorageError:
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def inspect_archive(
|
||||||
|
archive_data: bytes | str | PathLike[str],
|
||||||
|
*,
|
||||||
|
filename: str,
|
||||||
|
password: str | None = None,
|
||||||
|
max_entries: int = ARCHIVE_UPLOAD_MAX_ENTRIES,
|
||||||
|
max_expanded_bytes: int = 2 * 1024 * 1024 * 1024,
|
||||||
|
max_expansion_ratio: int = 100,
|
||||||
|
) -> ArchiveInspection:
|
||||||
|
archive_format = archive_format_for_filename(filename)
|
||||||
|
compressed_size = _archive_size(archive_data)
|
||||||
|
if archive_format == "zip":
|
||||||
|
entries, requires_password, password_verified = _inspect_zip(
|
||||||
|
archive_data,
|
||||||
|
password=password,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
entries = _inspect_tar(archive_data)
|
||||||
|
requires_password = False
|
||||||
|
password_verified = True
|
||||||
|
_validate_archive_limits(
|
||||||
|
entries,
|
||||||
|
compressed_size=compressed_size,
|
||||||
|
max_entries=max_entries,
|
||||||
|
max_expanded_bytes=max_expanded_bytes,
|
||||||
|
max_expansion_ratio=max_expansion_ratio,
|
||||||
|
)
|
||||||
|
complete_entries = _with_derived_directories(entries)
|
||||||
|
return ArchiveInspection(
|
||||||
|
archive_format=archive_format,
|
||||||
|
entries=tuple(complete_entries),
|
||||||
|
file_count=sum(entry.kind == "file" for entry in complete_entries),
|
||||||
|
directory_count=sum(
|
||||||
|
entry.kind == "directory" for entry in complete_entries
|
||||||
|
),
|
||||||
|
expanded_size_bytes=sum(
|
||||||
|
entry.size_bytes for entry in entries if entry.kind == "file"
|
||||||
|
),
|
||||||
|
compressed_size_bytes=compressed_size,
|
||||||
|
requires_password=requires_password,
|
||||||
|
password_verified=password_verified,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def extract_archive_upload(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
owner_type: str,
|
||||||
|
owner_id: str,
|
||||||
|
user_id: str,
|
||||||
|
archive_data: bytes | str | PathLike[str],
|
||||||
|
filename: str,
|
||||||
|
folder: str | None,
|
||||||
|
campaign_id: str | None,
|
||||||
|
selected_paths: Iterable[str] | None = None,
|
||||||
|
password: str | None = None,
|
||||||
|
conflict_strategy: str = "reject",
|
||||||
|
conflict_resolutions: Iterable[FileConflictResolution] | None = None,
|
||||||
|
metadata: dict[str, Any] | None = None,
|
||||||
|
is_admin: bool = False,
|
||||||
|
encryption_vault_id: str | None = None,
|
||||||
|
max_entries: int = ARCHIVE_UPLOAD_MAX_ENTRIES,
|
||||||
|
max_file_bytes: int = 50 * 1024 * 1024,
|
||||||
|
max_expanded_bytes: int = 2 * 1024 * 1024 * 1024,
|
||||||
|
max_expansion_ratio: int = 100,
|
||||||
|
) -> list[UploadedStoredFile]:
|
||||||
|
inspection = inspect_archive(
|
||||||
|
archive_data,
|
||||||
|
filename=filename,
|
||||||
|
password=password,
|
||||||
|
max_entries=max_entries,
|
||||||
|
max_expanded_bytes=max_expanded_bytes,
|
||||||
|
max_expansion_ratio=max_expansion_ratio,
|
||||||
|
)
|
||||||
|
if inspection.requires_password and not inspection.password_verified:
|
||||||
|
raise ArchivePasswordError("Archive password is required")
|
||||||
|
selected_files = _selected_file_paths(inspection.entries, selected_paths)
|
||||||
|
if not selected_files:
|
||||||
|
raise FileStorageError("Select at least one archive file to import")
|
||||||
|
actual_total_limit = min(
|
||||||
|
max_expanded_bytes,
|
||||||
|
inspection.compressed_size_bytes * max_expansion_ratio,
|
||||||
|
)
|
||||||
|
if inspection.archive_format == "zip":
|
||||||
|
members = _read_selected_zip_members(
|
||||||
|
archive_data,
|
||||||
|
selected_files=selected_files,
|
||||||
|
password=password,
|
||||||
|
max_file_bytes=max_file_bytes,
|
||||||
|
max_total_bytes=actual_total_limit,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
members = _read_selected_tar_members(
|
||||||
|
archive_data,
|
||||||
|
selected_files=selected_files,
|
||||||
|
max_file_bytes=max_file_bytes,
|
||||||
|
max_total_bytes=actual_total_limit,
|
||||||
|
)
|
||||||
|
return _store_archive_members(
|
||||||
|
session,
|
||||||
|
members=members,
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
owner_type=owner_type,
|
||||||
|
owner_id=owner_id,
|
||||||
|
user_id=user_id,
|
||||||
|
folder=folder,
|
||||||
|
campaign_id=campaign_id,
|
||||||
|
conflict_strategy=conflict_strategy,
|
||||||
|
conflict_resolutions=conflict_resolutions,
|
||||||
|
metadata=metadata,
|
||||||
|
is_admin=is_admin,
|
||||||
|
encryption_vault_id=encryption_vault_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def extract_zip_upload(
|
def extract_zip_upload(
|
||||||
session: Session,
|
session: Session,
|
||||||
*,
|
*,
|
||||||
@@ -77,55 +254,409 @@ def extract_zip_upload(
|
|||||||
conflict_resolutions: Iterable[FileConflictResolution] | None = None,
|
conflict_resolutions: Iterable[FileConflictResolution] | None = None,
|
||||||
metadata: dict[str, Any] | None = None,
|
metadata: dict[str, Any] | None = None,
|
||||||
is_admin: bool = False,
|
is_admin: bool = False,
|
||||||
|
encryption_vault_id: str | None = None,
|
||||||
max_files: int = ZIP_UPLOAD_MAX_FILES,
|
max_files: int = ZIP_UPLOAD_MAX_FILES,
|
||||||
max_file_bytes: int = 50 * 1024 * 1024,
|
max_file_bytes: int = 50 * 1024 * 1024,
|
||||||
max_total_bytes: int = 250 * 1024 * 1024,
|
max_total_bytes: int = 250 * 1024 * 1024,
|
||||||
) -> list[UploadedStoredFile]:
|
) -> list[UploadedStoredFile]:
|
||||||
uploaded: list[UploadedStoredFile] = []
|
"""Backward-compatible wrapper for the original immediate ZIP endpoint."""
|
||||||
total = 0
|
|
||||||
base_folder = normalize_folder(folder)
|
return extract_archive_upload(
|
||||||
|
session,
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
owner_type=owner_type,
|
||||||
|
owner_id=owner_id,
|
||||||
|
user_id=user_id,
|
||||||
|
archive_data=zip_data,
|
||||||
|
filename="archive.zip",
|
||||||
|
folder=folder,
|
||||||
|
campaign_id=campaign_id,
|
||||||
|
conflict_strategy=conflict_strategy,
|
||||||
|
conflict_resolutions=conflict_resolutions,
|
||||||
|
metadata=metadata,
|
||||||
|
is_admin=is_admin,
|
||||||
|
encryption_vault_id=encryption_vault_id,
|
||||||
|
max_entries=max_files,
|
||||||
|
max_file_bytes=max_file_bytes,
|
||||||
|
max_expanded_bytes=max_total_bytes,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _inspect_zip(
|
||||||
|
archive_data: bytes | str | PathLike[str],
|
||||||
|
*,
|
||||||
|
password: str | None,
|
||||||
|
) -> tuple[list[ArchiveEntry], bool, bool]:
|
||||||
try:
|
try:
|
||||||
source = BytesIO(zip_data) if isinstance(zip_data, bytes) else zip_data
|
with pyzipper.AESZipFile(_archive_source(archive_data)) as archive:
|
||||||
with zipfile.ZipFile(source) as archive:
|
infos = archive.infolist()
|
||||||
infos = [info for info in archive.infolist() if not info.is_dir()]
|
entries = [_zip_entry(info) for info in infos]
|
||||||
if len(infos) > max_files:
|
encrypted_info = next(
|
||||||
raise FileStorageError(f"ZIP contains too many files (limit {max_files})")
|
(
|
||||||
for info in infos:
|
info
|
||||||
if info.flag_bits & 0x1:
|
for info in infos
|
||||||
raise FileStorageError("Encrypted ZIP uploads are not supported")
|
if not info.is_dir() and bool(info.flag_bits & 0x1)
|
||||||
if info.file_size < 0:
|
),
|
||||||
raise FileStorageError("Invalid ZIP member")
|
None,
|
||||||
if info.file_size > max_file_bytes:
|
)
|
||||||
raise FileStorageError(f"ZIP member {info.filename!r} exceeds per-file limit")
|
if encrypted_info is None:
|
||||||
if total + info.file_size > max_total_bytes:
|
return entries, False, True
|
||||||
raise FileStorageError("ZIP is too large after extraction")
|
if not password:
|
||||||
inner_path = normalize_logical_path(info.filename)
|
return entries, True, False
|
||||||
target_path = f"{base_folder}/{inner_path}" if base_folder else inner_path
|
try:
|
||||||
data, total = _read_zip_member(
|
with archive.open(
|
||||||
archive,
|
encrypted_info, pwd=password.encode("utf-8")
|
||||||
info,
|
) as member:
|
||||||
max_file_bytes=max_file_bytes,
|
member.read(1)
|
||||||
max_total_bytes=max_total_bytes,
|
except (RuntimeError, ValueError, zipfile.BadZipFile) as exc:
|
||||||
current_total=total,
|
raise ArchivePasswordError("Archive password is incorrect") from exc
|
||||||
)
|
return entries, True, True
|
||||||
uploaded.append(
|
except ArchivePasswordError:
|
||||||
create_file_asset(
|
raise
|
||||||
session,
|
except (OSError, ValueError, zipfile.BadZipFile) as exc:
|
||||||
tenant_id=tenant_id,
|
raise FileStorageError("Invalid ZIP upload") from exc
|
||||||
owner_type=owner_type,
|
|
||||||
owner_id=owner_id,
|
|
||||||
user_id=user_id,
|
def _inspect_tar(
|
||||||
filename=filename_from_path(inner_path),
|
archive_data: bytes | str | PathLike[str],
|
||||||
data=data,
|
) -> list[ArchiveEntry]:
|
||||||
display_path=target_path,
|
try:
|
||||||
content_type=mimetypes.guess_type(inner_path)[0] or "application/octet-stream",
|
with _open_tar(archive_data) as archive:
|
||||||
metadata=metadata,
|
entries: list[ArchiveEntry] = []
|
||||||
campaign_id=campaign_id,
|
for member in archive.getmembers():
|
||||||
conflict_strategy=conflict_strategy,
|
path = _safe_member_path(member.name)
|
||||||
conflict_resolutions=conflict_resolutions,
|
if member.isdir():
|
||||||
is_admin=is_admin,
|
entries.append(
|
||||||
|
ArchiveEntry(path=path, kind="directory", size_bytes=0)
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
if not member.isfile():
|
||||||
|
raise FileStorageError(
|
||||||
|
f"Archive member {member.name!r} is not a regular file or directory"
|
||||||
|
)
|
||||||
|
entries.append(
|
||||||
|
ArchiveEntry(
|
||||||
|
path=path,
|
||||||
|
kind="file",
|
||||||
|
size_bytes=max(0, int(member.size)),
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
except zipfile.BadZipFile as exc:
|
return entries
|
||||||
raise FileStorageError("Invalid ZIP upload") from exc
|
except FileStorageError:
|
||||||
|
raise
|
||||||
|
except (OSError, tarfile.TarError) as exc:
|
||||||
|
raise FileStorageError("Invalid TAR upload") from exc
|
||||||
|
|
||||||
|
|
||||||
|
def _zip_entry(info: zipfile.ZipInfo) -> ArchiveEntry:
|
||||||
|
path = _safe_member_path(info.filename)
|
||||||
|
unix_mode = (info.external_attr >> 16) & 0xFFFF
|
||||||
|
file_type = stat.S_IFMT(unix_mode)
|
||||||
|
if file_type and not (
|
||||||
|
stat.S_ISREG(unix_mode) or stat.S_ISDIR(unix_mode)
|
||||||
|
):
|
||||||
|
raise FileStorageError(
|
||||||
|
f"Archive member {info.filename!r} is not a regular file or directory"
|
||||||
|
)
|
||||||
|
if info.file_size < 0 or info.compress_size < 0:
|
||||||
|
raise FileStorageError(f"Archive member {info.filename!r} has invalid size metadata")
|
||||||
|
return ArchiveEntry(
|
||||||
|
path=path,
|
||||||
|
kind="directory" if info.is_dir() else "file",
|
||||||
|
size_bytes=0 if info.is_dir() else int(info.file_size),
|
||||||
|
compressed_size_bytes=(
|
||||||
|
None if info.is_dir() else int(info.compress_size)
|
||||||
|
),
|
||||||
|
encrypted=bool(info.flag_bits & 0x1),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_archive_limits(
|
||||||
|
entries: list[ArchiveEntry],
|
||||||
|
*,
|
||||||
|
compressed_size: int,
|
||||||
|
max_entries: int,
|
||||||
|
max_expanded_bytes: int,
|
||||||
|
max_expansion_ratio: int,
|
||||||
|
) -> None:
|
||||||
|
if len(entries) > max_entries:
|
||||||
|
raise FileStorageError(
|
||||||
|
f"Archive contains too many entries (limit {max_entries})"
|
||||||
|
)
|
||||||
|
seen: dict[str, str] = {}
|
||||||
|
expanded_size = 0
|
||||||
|
for entry in entries:
|
||||||
|
previous_kind = seen.get(entry.path)
|
||||||
|
if previous_kind is not None:
|
||||||
|
raise FileStorageError(
|
||||||
|
f"Archive contains duplicate path {entry.path!r}"
|
||||||
|
)
|
||||||
|
seen[entry.path] = entry.kind
|
||||||
|
if entry.kind == "file":
|
||||||
|
expanded_size += entry.size_bytes
|
||||||
|
if expanded_size > max_expanded_bytes:
|
||||||
|
raise FileStorageError(
|
||||||
|
"Archive is too large after extraction "
|
||||||
|
f"(limit {max_expanded_bytes} bytes)"
|
||||||
|
)
|
||||||
|
if expanded_size and (
|
||||||
|
compressed_size <= 0
|
||||||
|
or expanded_size > compressed_size * max_expansion_ratio
|
||||||
|
):
|
||||||
|
raise FileStorageError(
|
||||||
|
"Archive expansion ratio exceeds "
|
||||||
|
f"{max_expansion_ratio}:1"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _with_derived_directories(
|
||||||
|
entries: list[ArchiveEntry],
|
||||||
|
) -> list[ArchiveEntry]:
|
||||||
|
by_path = {entry.path: entry for entry in entries}
|
||||||
|
for entry in entries:
|
||||||
|
parent = PurePosixPath(entry.path).parent
|
||||||
|
while str(parent) not in {"", "."}:
|
||||||
|
path = str(parent)
|
||||||
|
existing = by_path.get(path)
|
||||||
|
if existing and existing.kind != "directory":
|
||||||
|
raise FileStorageError(
|
||||||
|
f"Archive path {path!r} is both a file and a directory"
|
||||||
|
)
|
||||||
|
by_path.setdefault(
|
||||||
|
path,
|
||||||
|
ArchiveEntry(path=path, kind="directory", size_bytes=0),
|
||||||
|
)
|
||||||
|
parent = parent.parent
|
||||||
|
return sorted(
|
||||||
|
by_path.values(),
|
||||||
|
key=lambda entry: (
|
||||||
|
tuple(entry.path.casefold().split("/")),
|
||||||
|
entry.kind != "directory",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _selected_file_paths(
|
||||||
|
entries: tuple[ArchiveEntry, ...],
|
||||||
|
selected_paths: Iterable[str] | None,
|
||||||
|
) -> set[str]:
|
||||||
|
file_paths = {entry.path for entry in entries if entry.kind == "file"}
|
||||||
|
if selected_paths is None:
|
||||||
|
return file_paths
|
||||||
|
known = {entry.path: entry for entry in entries}
|
||||||
|
normalized = {_safe_member_path(path) for path in selected_paths}
|
||||||
|
unknown = normalized - known.keys()
|
||||||
|
if unknown:
|
||||||
|
raise FileStorageError(
|
||||||
|
f"Archive selection contains unknown path {sorted(unknown)[0]!r}"
|
||||||
|
)
|
||||||
|
selected_files: set[str] = set()
|
||||||
|
for path in normalized:
|
||||||
|
entry = known[path]
|
||||||
|
if entry.kind == "file":
|
||||||
|
selected_files.add(path)
|
||||||
|
continue
|
||||||
|
prefix = f"{path}/"
|
||||||
|
selected_files.update(
|
||||||
|
file_path
|
||||||
|
for file_path in file_paths
|
||||||
|
if file_path.startswith(prefix)
|
||||||
|
)
|
||||||
|
return selected_files
|
||||||
|
|
||||||
|
|
||||||
|
def _read_selected_zip_members(
|
||||||
|
archive_data: bytes | str | PathLike[str],
|
||||||
|
*,
|
||||||
|
selected_files: set[str],
|
||||||
|
password: str | None,
|
||||||
|
max_file_bytes: int,
|
||||||
|
max_total_bytes: int,
|
||||||
|
) -> list[tuple[str, bytes]]:
|
||||||
|
result: list[tuple[str, bytes]] = []
|
||||||
|
total = 0
|
||||||
|
try:
|
||||||
|
with pyzipper.AESZipFile(_archive_source(archive_data)) as archive:
|
||||||
|
for info in archive.infolist():
|
||||||
|
if info.is_dir():
|
||||||
|
continue
|
||||||
|
path = _safe_member_path(info.filename)
|
||||||
|
if path not in selected_files:
|
||||||
|
continue
|
||||||
|
pwd = password.encode("utf-8") if password else None
|
||||||
|
try:
|
||||||
|
with archive.open(info, pwd=pwd) as source:
|
||||||
|
data, total = _read_member(
|
||||||
|
source,
|
||||||
|
path=path,
|
||||||
|
max_file_bytes=max_file_bytes,
|
||||||
|
max_total_bytes=max_total_bytes,
|
||||||
|
current_total=total,
|
||||||
|
)
|
||||||
|
except (RuntimeError, ValueError, zipfile.BadZipFile) as exc:
|
||||||
|
if info.flag_bits & 0x1:
|
||||||
|
raise ArchivePasswordError(
|
||||||
|
"Archive password is incorrect"
|
||||||
|
) from exc
|
||||||
|
raise
|
||||||
|
result.append((path, data))
|
||||||
|
except (FileStorageError, ArchivePasswordError):
|
||||||
|
raise
|
||||||
|
except (OSError, ValueError, zipfile.BadZipFile) as exc:
|
||||||
|
raise FileStorageError("ZIP extraction failed") from exc
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def _read_selected_tar_members(
|
||||||
|
archive_data: bytes | str | PathLike[str],
|
||||||
|
*,
|
||||||
|
selected_files: set[str],
|
||||||
|
max_file_bytes: int,
|
||||||
|
max_total_bytes: int,
|
||||||
|
) -> list[tuple[str, bytes]]:
|
||||||
|
result: list[tuple[str, bytes]] = []
|
||||||
|
total = 0
|
||||||
|
try:
|
||||||
|
with _open_tar(archive_data) as archive:
|
||||||
|
for member in archive.getmembers():
|
||||||
|
if not member.isfile():
|
||||||
|
continue
|
||||||
|
path = _safe_member_path(member.name)
|
||||||
|
if path not in selected_files:
|
||||||
|
continue
|
||||||
|
source = archive.extractfile(member)
|
||||||
|
if source is None:
|
||||||
|
raise FileStorageError(
|
||||||
|
f"Archive member {path!r} could not be read"
|
||||||
|
)
|
||||||
|
with source:
|
||||||
|
data, total = _read_member(
|
||||||
|
source,
|
||||||
|
path=path,
|
||||||
|
max_file_bytes=max_file_bytes,
|
||||||
|
max_total_bytes=max_total_bytes,
|
||||||
|
current_total=total,
|
||||||
|
)
|
||||||
|
result.append((path, data))
|
||||||
|
except FileStorageError:
|
||||||
|
raise
|
||||||
|
except (OSError, tarfile.TarError) as exc:
|
||||||
|
raise FileStorageError("TAR extraction failed") from exc
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def _read_member(
|
||||||
|
source: BinaryIO,
|
||||||
|
*,
|
||||||
|
path: str,
|
||||||
|
max_file_bytes: int,
|
||||||
|
max_total_bytes: int,
|
||||||
|
current_total: int,
|
||||||
|
) -> tuple[bytes, int]:
|
||||||
|
parts: list[bytes] = []
|
||||||
|
actual_size = 0
|
||||||
|
while True:
|
||||||
|
read_size = min(
|
||||||
|
_ARCHIVE_READ_CHUNK_SIZE,
|
||||||
|
max_file_bytes + 1 - actual_size,
|
||||||
|
)
|
||||||
|
chunk = source.read(read_size)
|
||||||
|
if not chunk:
|
||||||
|
break
|
||||||
|
actual_size += len(chunk)
|
||||||
|
if actual_size > max_file_bytes:
|
||||||
|
raise FileStorageError(
|
||||||
|
f"Archive member {path!r} exceeds per-file limit"
|
||||||
|
)
|
||||||
|
if current_total + actual_size > max_total_bytes:
|
||||||
|
raise FileStorageError("Archive is too large after extraction")
|
||||||
|
parts.append(chunk)
|
||||||
|
return b"".join(parts), current_total + actual_size
|
||||||
|
|
||||||
|
|
||||||
|
def _store_archive_members(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
members: Iterable[tuple[str, bytes]],
|
||||||
|
tenant_id: str,
|
||||||
|
owner_type: str,
|
||||||
|
owner_id: str,
|
||||||
|
user_id: str,
|
||||||
|
folder: str | None,
|
||||||
|
campaign_id: str | None,
|
||||||
|
conflict_strategy: str,
|
||||||
|
conflict_resolutions: Iterable[FileConflictResolution] | None,
|
||||||
|
metadata: dict[str, Any] | None,
|
||||||
|
is_admin: bool,
|
||||||
|
encryption_vault_id: str | None,
|
||||||
|
) -> list[UploadedStoredFile]:
|
||||||
|
uploaded: list[UploadedStoredFile] = []
|
||||||
|
base_folder = normalize_folder(folder)
|
||||||
|
for inner_path, data in members:
|
||||||
|
target_path = (
|
||||||
|
f"{base_folder}/{inner_path}" if base_folder else inner_path
|
||||||
|
)
|
||||||
|
uploaded.append(
|
||||||
|
create_file_asset(
|
||||||
|
session,
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
owner_type=owner_type,
|
||||||
|
owner_id=owner_id,
|
||||||
|
user_id=user_id,
|
||||||
|
filename=filename_from_path(inner_path),
|
||||||
|
data=data,
|
||||||
|
display_path=target_path,
|
||||||
|
content_type=mimetypes.guess_type(inner_path)[0]
|
||||||
|
or "application/octet-stream",
|
||||||
|
metadata=metadata,
|
||||||
|
campaign_id=campaign_id,
|
||||||
|
conflict_strategy=conflict_strategy,
|
||||||
|
conflict_resolutions=conflict_resolutions,
|
||||||
|
is_admin=is_admin,
|
||||||
|
encryption_vault_id=encryption_vault_id,
|
||||||
|
)
|
||||||
|
)
|
||||||
return uploaded
|
return uploaded
|
||||||
|
|
||||||
|
|
||||||
|
def _safe_member_path(value: str) -> str:
|
||||||
|
raw = str(value or "").replace("\\", "/").strip()
|
||||||
|
if (
|
||||||
|
not raw
|
||||||
|
or "\x00" in raw
|
||||||
|
or raw.startswith("/")
|
||||||
|
or _WINDOWS_DRIVE_RE.match(raw)
|
||||||
|
):
|
||||||
|
raise FileStorageError(f"Unsafe archive member path {value!r}")
|
||||||
|
if any(part == ".." for part in raw.split("/")):
|
||||||
|
raise FileStorageError(f"Unsafe archive member path {value!r}")
|
||||||
|
try:
|
||||||
|
return normalize_logical_path(raw.rstrip("/"))
|
||||||
|
except ValueError as exc:
|
||||||
|
raise FileStorageError(f"Unsafe archive member path {value!r}") from exc
|
||||||
|
|
||||||
|
|
||||||
|
def _archive_size(archive_data: bytes | str | PathLike[str]) -> int:
|
||||||
|
if isinstance(archive_data, bytes):
|
||||||
|
return len(archive_data)
|
||||||
|
try:
|
||||||
|
return Path(archive_data).stat().st_size
|
||||||
|
except OSError as exc:
|
||||||
|
raise FileStorageError("Archive upload could not be read") from exc
|
||||||
|
|
||||||
|
|
||||||
|
def _archive_source(
|
||||||
|
archive_data: bytes | str | PathLike[str],
|
||||||
|
) -> BytesIO | str | PathLike[str]:
|
||||||
|
return BytesIO(archive_data) if isinstance(archive_data, bytes) else archive_data
|
||||||
|
|
||||||
|
|
||||||
|
def _open_tar(
|
||||||
|
archive_data: bytes | str | PathLike[str],
|
||||||
|
) -> tarfile.TarFile:
|
||||||
|
if isinstance(archive_data, bytes):
|
||||||
|
return tarfile.open(fileobj=BytesIO(archive_data), mode="r:*")
|
||||||
|
try:
|
||||||
|
return tarfile.open(name=archive_data, mode="r:*")
|
||||||
|
except OSError as exc:
|
||||||
|
raise FileStorageError("Archive upload could not be read") from exc
|
||||||
|
|||||||
@@ -1,207 +1,32 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from dataclasses import dataclass, field
|
from govoplan_core.core.object_storage import (
|
||||||
from pathlib import Path
|
LocalFilesystemStorageBackend,
|
||||||
from typing import Iterable, Protocol
|
S3StorageBackend,
|
||||||
|
StorageBackend,
|
||||||
from govoplan_core.security.outbound_http import (
|
StorageBackendError,
|
||||||
OutboundHttpError,
|
StorageObjectInfo,
|
||||||
response_limit,
|
StorageObjectMissing,
|
||||||
validate_unpinned_sdk_http_url,
|
StorageObjectPage,
|
||||||
|
configured_storage_backend,
|
||||||
)
|
)
|
||||||
|
|
||||||
from govoplan_files.backend.runtime import settings
|
from govoplan_files.backend.runtime import settings
|
||||||
|
|
||||||
|
|
||||||
class StorageBackendError(RuntimeError):
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
class StorageBackend(Protocol):
|
|
||||||
name: str
|
|
||||||
|
|
||||||
def put_bytes(self, key: str, data: bytes, *, content_type: str | None = None) -> None: ...
|
|
||||||
def get_bytes(self, key: str) -> bytes: ...
|
|
||||||
def iter_bytes(self, key: str, *, chunk_size: int = 1024 * 1024) -> Iterable[bytes]: ...
|
|
||||||
def delete(self, key: str) -> None: ...
|
|
||||||
def exists(self, key: str) -> bool: ...
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(slots=True)
|
|
||||||
class LocalFilesystemStorageBackend:
|
|
||||||
root: Path
|
|
||||||
fallback_roots: tuple[Path, ...] = field(default_factory=tuple)
|
|
||||||
name: str = "local"
|
|
||||||
|
|
||||||
def __post_init__(self) -> None:
|
|
||||||
self.root = self.root.expanduser().resolve()
|
|
||||||
self.fallback_roots = tuple(root.expanduser().resolve() for root in self.fallback_roots if root)
|
|
||||||
self.root.mkdir(parents=True, exist_ok=True)
|
|
||||||
|
|
||||||
def _path_for_root(self, root: Path, key: str) -> Path:
|
|
||||||
path = (root / key).resolve()
|
|
||||||
if not path.is_relative_to(root):
|
|
||||||
raise StorageBackendError("Storage key escapes local storage root")
|
|
||||||
return path
|
|
||||||
|
|
||||||
def _path(self, key: str) -> Path:
|
|
||||||
return self._path_for_root(self.root, key)
|
|
||||||
|
|
||||||
def _readable_path(self, key: str) -> Path:
|
|
||||||
primary = self._path(key)
|
|
||||||
if primary.exists() and primary.is_file():
|
|
||||||
return primary
|
|
||||||
for root in self.fallback_roots:
|
|
||||||
candidate = self._path_for_root(root, key)
|
|
||||||
if candidate.exists() and candidate.is_file():
|
|
||||||
return candidate
|
|
||||||
raise StorageBackendError("Stored object does not exist")
|
|
||||||
|
|
||||||
def put_bytes(self, key: str, data: bytes, *, content_type: str | None = None) -> None:
|
|
||||||
path = self._path(key)
|
|
||||||
path.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
path.write_bytes(data)
|
|
||||||
|
|
||||||
def get_bytes(self, key: str) -> bytes:
|
|
||||||
return self._readable_path(key).read_bytes()
|
|
||||||
|
|
||||||
def iter_bytes(self, key: str, *, chunk_size: int = 1024 * 1024) -> Iterable[bytes]:
|
|
||||||
path = self._readable_path(key)
|
|
||||||
with path.open("rb") as handle:
|
|
||||||
while True:
|
|
||||||
chunk = handle.read(chunk_size)
|
|
||||||
if not chunk:
|
|
||||||
break
|
|
||||||
yield chunk
|
|
||||||
|
|
||||||
def delete(self, key: str) -> None:
|
|
||||||
path = self._path(key)
|
|
||||||
if path.exists() and path.is_file():
|
|
||||||
path.unlink()
|
|
||||||
|
|
||||||
def exists(self, key: str) -> bool:
|
|
||||||
try:
|
|
||||||
self._readable_path(key)
|
|
||||||
except StorageBackendError:
|
|
||||||
return False
|
|
||||||
return True
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(slots=True)
|
|
||||||
class S3StorageBackend:
|
|
||||||
bucket: str
|
|
||||||
endpoint_url: str
|
|
||||||
region_name: str
|
|
||||||
access_key_id: str
|
|
||||||
secret_access_key: str
|
|
||||||
name: str = "s3"
|
|
||||||
|
|
||||||
@property
|
|
||||||
def client(self):
|
|
||||||
try:
|
|
||||||
endpoint_url = validate_unpinned_sdk_http_url(
|
|
||||||
self.endpoint_url,
|
|
||||||
label="File storage S3 endpoint",
|
|
||||||
)
|
|
||||||
except OutboundHttpError as exc:
|
|
||||||
raise StorageBackendError(str(exc)) from exc
|
|
||||||
try:
|
|
||||||
import boto3
|
|
||||||
except ModuleNotFoundError as exc:
|
|
||||||
raise StorageBackendError("boto3 is required for the S3 storage backend") from exc
|
|
||||||
return boto3.client(
|
|
||||||
"s3",
|
|
||||||
endpoint_url=endpoint_url,
|
|
||||||
region_name=self.region_name,
|
|
||||||
aws_access_key_id=self.access_key_id,
|
|
||||||
aws_secret_access_key=self.secret_access_key,
|
|
||||||
)
|
|
||||||
|
|
||||||
def put_bytes(self, key: str, data: bytes, *, content_type: str | None = None) -> None:
|
|
||||||
max_bytes = response_limit("file")
|
|
||||||
if len(data) > max_bytes:
|
|
||||||
raise StorageBackendError(f"Stored object exceeds the deployment limit of {max_bytes} bytes")
|
|
||||||
kwargs = {"Bucket": self.bucket, "Key": key, "Body": data}
|
|
||||||
if content_type:
|
|
||||||
kwargs["ContentType"] = content_type
|
|
||||||
self.client.put_object(**kwargs)
|
|
||||||
|
|
||||||
def get_bytes(self, key: str) -> bytes:
|
|
||||||
try:
|
|
||||||
obj = self.client.get_object(Bucket=self.bucket, Key=key)
|
|
||||||
max_bytes = response_limit("file")
|
|
||||||
body = obj["Body"]
|
|
||||||
try:
|
|
||||||
_reject_declared_object_size(obj, max_bytes=max_bytes)
|
|
||||||
data = body.read(max_bytes + 1)
|
|
||||||
if len(data) > max_bytes:
|
|
||||||
raise StorageBackendError(f"Stored object exceeds the deployment limit of {max_bytes} bytes")
|
|
||||||
return data
|
|
||||||
finally:
|
|
||||||
if hasattr(body, "close"):
|
|
||||||
body.close()
|
|
||||||
except Exception as exc: # pragma: no cover - depends on S3 backend
|
|
||||||
raise StorageBackendError(str(exc)) from exc
|
|
||||||
|
|
||||||
def iter_bytes(self, key: str, *, chunk_size: int = 1024 * 1024) -> Iterable[bytes]:
|
|
||||||
try:
|
|
||||||
obj = self.client.get_object(Bucket=self.bucket, Key=key)
|
|
||||||
max_bytes = response_limit("file")
|
|
||||||
body = obj["Body"]
|
|
||||||
try:
|
|
||||||
_reject_declared_object_size(obj, max_bytes=max_bytes)
|
|
||||||
total = 0
|
|
||||||
while True:
|
|
||||||
chunk = body.read(chunk_size)
|
|
||||||
if not chunk:
|
|
||||||
break
|
|
||||||
total += len(chunk)
|
|
||||||
if total > max_bytes:
|
|
||||||
raise StorageBackendError(f"Stored object exceeds the deployment limit of {max_bytes} bytes")
|
|
||||||
yield chunk
|
|
||||||
finally:
|
|
||||||
if hasattr(body, "close"):
|
|
||||||
body.close()
|
|
||||||
except Exception as exc: # pragma: no cover - depends on S3 backend
|
|
||||||
raise StorageBackendError(str(exc)) from exc
|
|
||||||
|
|
||||||
def delete(self, key: str) -> None:
|
|
||||||
self.client.delete_object(Bucket=self.bucket, Key=key)
|
|
||||||
|
|
||||||
def exists(self, key: str) -> bool:
|
|
||||||
try:
|
|
||||||
self.client.head_object(Bucket=self.bucket, Key=key)
|
|
||||||
return True
|
|
||||||
except Exception:
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
def _reject_declared_object_size(obj: object, *, max_bytes: int) -> None:
|
|
||||||
if not isinstance(obj, dict):
|
|
||||||
return
|
|
||||||
try:
|
|
||||||
declared_size = int(obj.get("ContentLength"))
|
|
||||||
except (TypeError, ValueError):
|
|
||||||
return
|
|
||||||
if declared_size > max_bytes:
|
|
||||||
raise StorageBackendError(f"Stored object exceeds the deployment limit of {max_bytes} bytes")
|
|
||||||
|
|
||||||
|
|
||||||
def _fallback_roots() -> tuple[Path, ...]:
|
|
||||||
raw = getattr(settings, "file_storage_local_fallback_roots", "") or ""
|
|
||||||
return tuple(Path(item.strip()) for item in str(raw).split(",") if item.strip())
|
|
||||||
|
|
||||||
|
|
||||||
def get_storage_backend() -> StorageBackend:
|
def get_storage_backend() -> StorageBackend:
|
||||||
backend = settings.file_storage_backend.lower().strip()
|
"""Return the deployment-owned backend for Files-managed objects."""
|
||||||
if backend in {"local", "filesystem", "fs"}:
|
|
||||||
return LocalFilesystemStorageBackend(Path(settings.file_storage_local_root), fallback_roots=_fallback_roots())
|
return configured_storage_backend(settings)
|
||||||
if backend in {"s3", "garage"}:
|
|
||||||
return S3StorageBackend(
|
|
||||||
bucket=settings.file_storage_s3_bucket or settings.s3_bucket,
|
__all__ = [
|
||||||
endpoint_url=settings.file_storage_s3_endpoint_url or settings.s3_endpoint_url,
|
"LocalFilesystemStorageBackend",
|
||||||
region_name=settings.file_storage_s3_region or settings.s3_region,
|
"S3StorageBackend",
|
||||||
access_key_id=settings.file_storage_s3_access_key_id or settings.s3_access_key_id,
|
"StorageBackend",
|
||||||
secret_access_key=settings.file_storage_s3_secret_access_key or settings.s3_secret_access_key,
|
"StorageBackendError",
|
||||||
)
|
"StorageObjectInfo",
|
||||||
raise StorageBackendError(f"Unsupported file storage backend: {settings.file_storage_backend}")
|
"StorageObjectMissing",
|
||||||
|
"StorageObjectPage",
|
||||||
|
"get_storage_backend",
|
||||||
|
]
|
||||||
|
|||||||
@@ -13,12 +13,16 @@ from typing import Any, Iterator
|
|||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from govoplan_files.backend.db.models import FileAsset, FileBlob, FileShare, FileVersion
|
from govoplan_files.backend.db.models import FileAsset, FileBlob, FileShare, FileVersion
|
||||||
from govoplan_files.backend.storage.backends import StorageBackend, StorageBackendError, get_storage_backend
|
from govoplan_files.backend.storage.backends import StorageBackend, get_storage_backend
|
||||||
from govoplan_files.backend.storage.common import FileStorageError
|
|
||||||
from govoplan_files.backend.storage.files import current_versions_and_blobs, get_asset_for_user, list_assets_for_user, share_files
|
from govoplan_files.backend.storage.files import current_versions_and_blobs, get_asset_for_user, list_assets_for_user, share_files
|
||||||
from govoplan_files.backend.storage.access import ensure_owner_access
|
from govoplan_files.backend.storage.access import ensure_owner_access
|
||||||
from govoplan_files.backend.storage.paths import normalize_folder, normalize_logical_path, safe_storage_component
|
from govoplan_files.backend.storage.paths import normalize_folder, normalize_logical_path, safe_storage_component
|
||||||
from govoplan_files.backend.storage.provenance import source_provenance_from_metadata, source_revision_from_metadata
|
from govoplan_files.backend.storage.provenance import source_provenance_from_metadata, source_revision_from_metadata
|
||||||
|
from govoplan_files.backend.storage.share_state import effective_file_share_clause
|
||||||
|
from govoplan_files.backend.storage.integrity import (
|
||||||
|
ensure_blob_is_readable,
|
||||||
|
read_verified_blob_bytes,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
MANAGED_SOURCE_PREFIX = "managed:"
|
MANAGED_SOURCE_PREFIX = "managed:"
|
||||||
@@ -165,7 +169,7 @@ def _campaign_linked_asset_ids(
|
|||||||
FileShare.file_asset_id.in_(chunk),
|
FileShare.file_asset_id.in_(chunk),
|
||||||
FileShare.target_type == "campaign",
|
FileShare.target_type == "campaign",
|
||||||
FileShare.target_id == campaign_id,
|
FileShare.target_id == campaign_id,
|
||||||
FileShare.revoked_at.is_(None),
|
effective_file_share_clause(),
|
||||||
)
|
)
|
||||||
.all()
|
.all()
|
||||||
)
|
)
|
||||||
@@ -301,11 +305,9 @@ def _materialize_managed_asset(
|
|||||||
target = _safe_local_target(local_root, relative_path)
|
target = _safe_local_target(local_root, relative_path)
|
||||||
target.parent.mkdir(parents=True, exist_ok=True)
|
target.parent.mkdir(parents=True, exist_ok=True)
|
||||||
version, blob = version_blobs[asset.id]
|
version, blob = version_blobs[asset.id]
|
||||||
|
ensure_blob_is_readable(blob)
|
||||||
if include_bytes:
|
if include_bytes:
|
||||||
try:
|
data = read_verified_blob_bytes(blob, backend=backend) if backend else b""
|
||||||
data = backend.get_bytes(blob.storage_key) if backend else b""
|
|
||||||
except StorageBackendError as exc:
|
|
||||||
raise FileStorageError(str(exc)) from exc
|
|
||||||
target.write_bytes(data)
|
target.write_bytes(data)
|
||||||
else:
|
else:
|
||||||
target.touch()
|
target.touch()
|
||||||
|
|||||||
@@ -15,8 +15,8 @@ from defusedxml import ElementTree as SafeElementTree
|
|||||||
|
|
||||||
from govoplan_core.security.outbound_http import (
|
from govoplan_core.security.outbound_http import (
|
||||||
OutboundHttpError,
|
OutboundHttpError,
|
||||||
validate_unpinned_sdk_host,
|
validate_outbound_host,
|
||||||
validate_unpinned_sdk_http_url,
|
validate_outbound_http_url,
|
||||||
)
|
)
|
||||||
|
|
||||||
from govoplan_files.backend.storage.http_client import ConnectorHttpError, request_connector_bytes
|
from govoplan_files.backend.storage.http_client import ConnectorHttpError, request_connector_bytes
|
||||||
@@ -27,6 +27,12 @@ from govoplan_files.backend.storage.connector_deployment import (
|
|||||||
validate_connector_tls_metadata,
|
validate_connector_tls_metadata,
|
||||||
)
|
)
|
||||||
from govoplan_files.backend.storage.connector_profiles import ConnectorProfile
|
from govoplan_files.backend.storage.connector_profiles import ConnectorProfile
|
||||||
|
from govoplan_files.backend.storage.sdk_peer_pinning import (
|
||||||
|
SdkPeerPinningError,
|
||||||
|
create_pinned_s3_client,
|
||||||
|
install_pinned_smb_transport,
|
||||||
|
pinned_smb_connection_cache,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class ConnectorBrowseError(RuntimeError):
|
class ConnectorBrowseError(RuntimeError):
|
||||||
@@ -246,6 +252,28 @@ def _browse_smb(profile: ConnectorProfile, *, path: str) -> list[ConnectorBrowse
|
|||||||
|
|
||||||
def _browse_s3(profile: ConnectorProfile, *, path: str, library_id: str | None, continuation_token: str | None) -> list[ConnectorBrowseItem]:
|
def _browse_s3(profile: ConnectorProfile, *, path: str, library_id: str | None, continuation_token: str | None) -> list[ConnectorBrowseItem]:
|
||||||
client = _s3_client(profile)
|
client = _s3_client(profile)
|
||||||
|
try:
|
||||||
|
return _browse_s3_with_client(
|
||||||
|
client,
|
||||||
|
profile=profile,
|
||||||
|
path=path,
|
||||||
|
library_id=library_id,
|
||||||
|
continuation_token=continuation_token,
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
close = getattr(client, "close", None)
|
||||||
|
if callable(close):
|
||||||
|
close()
|
||||||
|
|
||||||
|
|
||||||
|
def _browse_s3_with_client(
|
||||||
|
client: Any,
|
||||||
|
*,
|
||||||
|
profile: ConnectorProfile,
|
||||||
|
path: str,
|
||||||
|
library_id: str | None,
|
||||||
|
continuation_token: str | None,
|
||||||
|
) -> list[ConnectorBrowseItem]:
|
||||||
bucket = _s3_bucket(profile, library_id)
|
bucket = _s3_bucket(profile, library_id)
|
||||||
if not bucket:
|
if not bucket:
|
||||||
try:
|
try:
|
||||||
@@ -313,23 +341,22 @@ def _s3_client(profile: ConnectorProfile) -> Any:
|
|||||||
raise ConnectorBrowseError("Secret-ref S3 credentials need a runtime secret resolver before live browsing")
|
raise ConnectorBrowseError("Secret-ref S3 credentials need a runtime secret resolver before live browsing")
|
||||||
if profile.endpoint_url:
|
if profile.endpoint_url:
|
||||||
try:
|
try:
|
||||||
endpoint_url = validate_unpinned_sdk_http_url(
|
endpoint_url = validate_outbound_http_url(
|
||||||
profile.endpoint_url,
|
profile.endpoint_url,
|
||||||
label="S3 connector endpoint",
|
label="S3 connector endpoint",
|
||||||
)
|
)
|
||||||
except OutboundHttpError as exc:
|
except OutboundHttpError as exc:
|
||||||
raise ConnectorBrowseError(str(exc)) from exc
|
raise ConnectorBrowseError(str(exc)) from exc
|
||||||
else:
|
else:
|
||||||
raise ConnectorBrowseError(
|
endpoint_url = None
|
||||||
"S3 connector endpoint discovery uses an SDK transport that cannot guarantee connection-time DNS/IP "
|
|
||||||
"pinning; live S3 access is disabled until that transport supports pinning"
|
|
||||||
)
|
|
||||||
try:
|
try:
|
||||||
boto3 = import_module("boto3")
|
|
||||||
config_module = import_module("botocore.config")
|
config_module = import_module("botocore.config")
|
||||||
|
unsigned = import_module("botocore").UNSIGNED
|
||||||
except ImportError as exc:
|
except ImportError as exc:
|
||||||
raise ConnectorBrowseUnsupported("S3 connector browsing requires the optional boto3 dependency") from exc
|
raise ConnectorBrowseUnsupported("S3 connector browsing requires the optional boto3 dependency") from exc
|
||||||
kwargs: dict[str, object] = {"endpoint_url": endpoint_url}
|
kwargs: dict[str, object] = {}
|
||||||
|
if endpoint_url:
|
||||||
|
kwargs["endpoint_url"] = endpoint_url
|
||||||
region = _metadata_string(profile, "region") or _metadata_string(profile, "aws_region")
|
region = _metadata_string(profile, "region") or _metadata_string(profile, "aws_region")
|
||||||
if region:
|
if region:
|
||||||
kwargs["region_name"] = region
|
kwargs["region_name"] = region
|
||||||
@@ -346,10 +373,21 @@ def _s3_client(profile: ConnectorProfile) -> Any:
|
|||||||
if verify is not None:
|
if verify is not None:
|
||||||
kwargs["verify"] = verify
|
kwargs["verify"] = verify
|
||||||
addressing_style = _s3_addressing_style(profile)
|
addressing_style = _s3_addressing_style(profile)
|
||||||
|
config_values: dict[str, object] = {
|
||||||
|
"proxies": {},
|
||||||
|
"retries": {"mode": "standard", "max_attempts": 4},
|
||||||
|
}
|
||||||
if addressing_style:
|
if addressing_style:
|
||||||
kwargs["config"] = config_module.Config(s3={"addressing_style": addressing_style})
|
config_values["s3"] = {"addressing_style": addressing_style}
|
||||||
|
if bool(access_key) != bool(secret_key):
|
||||||
|
raise ConnectorBrowseError("S3 connectors require both an access key and a secret key")
|
||||||
|
if not access_key:
|
||||||
|
config_values["signature_version"] = unsigned
|
||||||
|
kwargs["config"] = config_module.Config(**config_values)
|
||||||
try:
|
try:
|
||||||
return boto3.client("s3", **kwargs)
|
return create_pinned_s3_client(**kwargs)
|
||||||
|
except SdkPeerPinningError as exc:
|
||||||
|
raise ConnectorBrowseError(str(exc)) from exc
|
||||||
except Exception as exc: # pragma: no cover - concrete exception types are dependency-version specific
|
except Exception as exc: # pragma: no cover - concrete exception types are dependency-version specific
|
||||||
raise ConnectorBrowseError(f"S3 connector could not be initialized: {exc}") from exc
|
raise ConnectorBrowseError(f"S3 connector could not be initialized: {exc}") from exc
|
||||||
|
|
||||||
@@ -783,7 +821,7 @@ def _smb_location(profile: ConnectorProfile) -> _SmbLocation:
|
|||||||
raise ConnectorBrowseError("SMB connector endpoint_url must include a server")
|
raise ConnectorBrowseError("SMB connector endpoint_url must include a server")
|
||||||
port = parsed.port or _int(profile.metadata.get("port")) or 445
|
port = parsed.port or _int(profile.metadata.get("port")) or 445
|
||||||
try:
|
try:
|
||||||
validate_unpinned_sdk_host(server, port=port, label="SMB connector endpoint")
|
validate_outbound_host(server, port=port, label="SMB connector endpoint")
|
||||||
except OutboundHttpError as exc:
|
except OutboundHttpError as exc:
|
||||||
raise ConnectorBrowseError(str(exc)) from exc
|
raise ConnectorBrowseError(str(exc)) from exc
|
||||||
path_parts = [part for part in unquote(parsed.path or "").strip("/").split("/") if part]
|
path_parts = [part for part in unquote(parsed.path or "").strip("/").split("/") if part]
|
||||||
@@ -810,6 +848,7 @@ def _smb_unc_path(location: _SmbLocation, path: str) -> str:
|
|||||||
def _smb_client_kwargs(profile: ConnectorProfile, location: _SmbLocation) -> dict[str, object]:
|
def _smb_client_kwargs(profile: ConnectorProfile, location: _SmbLocation) -> dict[str, object]:
|
||||||
kwargs: dict[str, object] = {
|
kwargs: dict[str, object] = {
|
||||||
"port": location.port,
|
"port": location.port,
|
||||||
|
"connection_cache": pinned_smb_connection_cache(),
|
||||||
"require_signing": _metadata_bool(profile, "require_signing", default=True),
|
"require_signing": _metadata_bool(profile, "require_signing", default=True),
|
||||||
"auth_protocol": _metadata_string(profile, "auth_protocol") or "ntlm",
|
"auth_protocol": _metadata_string(profile, "auth_protocol") or "ntlm",
|
||||||
}
|
}
|
||||||
@@ -845,9 +884,11 @@ def _profile_token(profile: ConnectorProfile) -> str | None:
|
|||||||
|
|
||||||
def _smbclient_module() -> Any:
|
def _smbclient_module() -> Any:
|
||||||
try:
|
try:
|
||||||
return import_module("smbclient")
|
return install_pinned_smb_transport(import_module("smbclient"))
|
||||||
except ImportError as exc:
|
except ImportError as exc:
|
||||||
raise ConnectorBrowseUnsupported("SMB connector browsing requires the optional smbprotocol dependency") from exc
|
raise ConnectorBrowseUnsupported("SMB connector browsing requires the optional smbprotocol dependency") from exc
|
||||||
|
except SdkPeerPinningError as exc:
|
||||||
|
raise ConnectorBrowseUnsupported(str(exc)) from exc
|
||||||
|
|
||||||
|
|
||||||
def _smb_entry_stat(entry: object) -> object | None:
|
def _smb_entry_stat(entry: object) -> object | None:
|
||||||
|
|||||||
@@ -5,8 +5,17 @@ from dataclasses import dataclass
|
|||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
from sqlalchemy import inspect
|
||||||
|
|
||||||
from govoplan_core.core.policy import normalize_policy_scope_type, policy_source_path
|
from govoplan_core.core.policy import normalize_policy_scope_type, policy_source_path
|
||||||
|
from govoplan_core.security.credential_envelopes import (
|
||||||
|
CredentialAccessContext,
|
||||||
|
CredentialEnvelope,
|
||||||
|
CredentialEnvelopeError,
|
||||||
|
ResolvedCredentialEnvelope,
|
||||||
|
list_credential_envelopes,
|
||||||
|
resolve_credential_envelope,
|
||||||
|
)
|
||||||
from govoplan_core.security.secrets import decrypt_secret, encrypt_secret
|
from govoplan_core.security.secrets import decrypt_secret, encrypt_secret
|
||||||
from govoplan_files.backend.db.models import FileConnectorCredential
|
from govoplan_files.backend.db.models import FileConnectorCredential
|
||||||
from govoplan_files.backend.storage.common import FileStorageError
|
from govoplan_files.backend.storage.common import FileStorageError
|
||||||
@@ -14,6 +23,8 @@ from govoplan_files.backend.storage.connector_deployment import reject_api_contr
|
|||||||
from govoplan_files.backend.storage.connector_policy import ConnectorPolicySource, connector_policy_sources_from_payload
|
from govoplan_files.backend.storage.connector_policy import ConnectorPolicySource, connector_policy_sources_from_payload
|
||||||
from govoplan_files.backend.storage.connector_profiles import supported_connector_providers
|
from govoplan_files.backend.storage.connector_profiles import supported_connector_providers
|
||||||
|
|
||||||
|
CORE_CREDENTIAL_ENVELOPE_PREFIX = "credential-envelope:"
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
@dataclass(frozen=True, slots=True)
|
||||||
class ConnectorCredential:
|
class ConnectorCredential:
|
||||||
@@ -33,6 +44,7 @@ class ConnectorCredential:
|
|||||||
policy_sources: tuple[ConnectorPolicySource, ...] = ()
|
policy_sources: tuple[ConnectorPolicySource, ...] = ()
|
||||||
metadata: Mapping[str, Any] | None = None
|
metadata: Mapping[str, Any] | None = None
|
||||||
source_kind: str = "database"
|
source_kind: str = "database"
|
||||||
|
has_secret: bool = False
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def source_path(self) -> str:
|
def source_path(self) -> str:
|
||||||
@@ -58,7 +70,7 @@ class ConnectorCredential:
|
|||||||
return True
|
return True
|
||||||
# Environment references are deliberately unavailable to API-managed
|
# Environment references are deliberately unavailable to API-managed
|
||||||
# credential rows. Legacy rows remain visible but fail closed.
|
# credential rows. Legacy rows remain visible but fail closed.
|
||||||
return bool(self.secret_ref or self.password_value or self.token_value)
|
return bool(self.has_secret or self.secret_ref or self.password_value or self.token_value)
|
||||||
|
|
||||||
def to_response(self) -> dict[str, Any]:
|
def to_response(self) -> dict[str, Any]:
|
||||||
return {
|
return {
|
||||||
@@ -85,7 +97,159 @@ def list_database_connector_credentials(
|
|||||||
tenant_id: str,
|
tenant_id: str,
|
||||||
include_disabled: bool = False,
|
include_disabled: bool = False,
|
||||||
) -> list[ConnectorCredential]:
|
) -> list[ConnectorCredential]:
|
||||||
return [connector_credential_from_row(row) for row in list_connector_credential_rows(session, tenant_id=tenant_id, include_disabled=include_disabled)]
|
local = [
|
||||||
|
connector_credential_from_row(row)
|
||||||
|
for row in list_connector_credential_rows(
|
||||||
|
session,
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
include_disabled=include_disabled,
|
||||||
|
)
|
||||||
|
]
|
||||||
|
return [
|
||||||
|
*local,
|
||||||
|
*list_reusable_connector_credentials(
|
||||||
|
session,
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
include_disabled=include_disabled,
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def reusable_credential_reference(credential_id: str) -> str:
|
||||||
|
return f"{CORE_CREDENTIAL_ENVELOPE_PREFIX}{credential_id}"
|
||||||
|
|
||||||
|
|
||||||
|
def reusable_credential_id(credential_ref: str | None) -> str | None:
|
||||||
|
if not credential_ref or not credential_ref.startswith(CORE_CREDENTIAL_ENVELOPE_PREFIX):
|
||||||
|
return None
|
||||||
|
value = credential_ref.removeprefix(CORE_CREDENTIAL_ENVELOPE_PREFIX).strip()
|
||||||
|
return value or None
|
||||||
|
|
||||||
|
|
||||||
|
def file_credential_context(
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
profile_id: str | None = None,
|
||||||
|
scope_type: str = "tenant",
|
||||||
|
scope_id: str | None = None,
|
||||||
|
administrative: bool = False,
|
||||||
|
) -> CredentialAccessContext:
|
||||||
|
return CredentialAccessContext(
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
user_id=scope_id if scope_type == "user" else None,
|
||||||
|
group_ids=frozenset({scope_id}) if scope_type == "group" and scope_id else frozenset(),
|
||||||
|
target_scope_type=scope_type,
|
||||||
|
target_scope_id=scope_id or tenant_id,
|
||||||
|
module_id="files",
|
||||||
|
server_ref=f"files:{profile_id}" if profile_id else None,
|
||||||
|
administrative=administrative,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def list_reusable_connector_credentials(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
include_disabled: bool = False,
|
||||||
|
) -> list[ConnectorCredential]:
|
||||||
|
if not inspect(session.get_bind()).has_table(CredentialEnvelope.__tablename__):
|
||||||
|
return []
|
||||||
|
context = file_credential_context(tenant_id=tenant_id, administrative=True)
|
||||||
|
return [
|
||||||
|
connector_credential_from_envelope(row)
|
||||||
|
for row in list_credential_envelopes(
|
||||||
|
session,
|
||||||
|
context=context,
|
||||||
|
include_inactive=include_disabled,
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_reusable_connector_credential(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
credential_ref: str,
|
||||||
|
profile_id: str,
|
||||||
|
scope_type: str,
|
||||||
|
scope_id: str | None,
|
||||||
|
) -> ConnectorCredential:
|
||||||
|
credential_id = reusable_credential_id(credential_ref)
|
||||||
|
if credential_id is None:
|
||||||
|
raise FileStorageError("Reusable credential reference is invalid")
|
||||||
|
try:
|
||||||
|
resolved = resolve_credential_envelope(
|
||||||
|
session,
|
||||||
|
credential_id=credential_id,
|
||||||
|
context=file_credential_context(
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
profile_id=profile_id,
|
||||||
|
scope_type=scope_type,
|
||||||
|
scope_id=scope_id,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
except CredentialEnvelopeError as exc:
|
||||||
|
raise FileStorageError("Reusable credential is unavailable to this file connection") from exc
|
||||||
|
return connector_credential_from_resolved_envelope(
|
||||||
|
resolved,
|
||||||
|
scope_type=scope_type,
|
||||||
|
scope_id=scope_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def connector_credential_from_envelope(row: CredentialEnvelope) -> ConnectorCredential:
|
||||||
|
return ConnectorCredential(
|
||||||
|
id=reusable_credential_reference(row.id),
|
||||||
|
label=row.name,
|
||||||
|
scope_type=row.scope_type,
|
||||||
|
scope_id=row.scope_id,
|
||||||
|
enabled=row.is_active,
|
||||||
|
credential_mode=_envelope_credential_mode(row.credential_kind, row.secret_keys),
|
||||||
|
username=_clean_public_value(row.public_data, "username"),
|
||||||
|
source_kind="credential_envelope",
|
||||||
|
has_secret=bool(row.secret_data_encrypted),
|
||||||
|
metadata={
|
||||||
|
"credential_envelope_id": row.id,
|
||||||
|
"credential_kind": row.credential_kind,
|
||||||
|
"allowed_modules": list(row.allowed_modules or []),
|
||||||
|
"allowed_server_refs": list(row.allowed_server_refs or []),
|
||||||
|
"inherit_to_lower_scopes": bool(row.inherit_to_lower_scopes),
|
||||||
|
"revision": row.revision,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def connector_credential_from_resolved_envelope(
|
||||||
|
row: ResolvedCredentialEnvelope,
|
||||||
|
*,
|
||||||
|
scope_type: str,
|
||||||
|
scope_id: str | None,
|
||||||
|
) -> ConnectorCredential:
|
||||||
|
return ConnectorCredential(
|
||||||
|
id=reusable_credential_reference(row.id),
|
||||||
|
label=row.name,
|
||||||
|
scope_type=scope_type,
|
||||||
|
scope_id=scope_id,
|
||||||
|
enabled=True,
|
||||||
|
credential_mode=_envelope_credential_mode(row.credential_kind, row.secret_data),
|
||||||
|
username=_clean_public_value(row.public_data, "username"),
|
||||||
|
password_value=_first_secret(row.secret_data, "password", "secret"),
|
||||||
|
token_value=_first_secret(
|
||||||
|
row.secret_data,
|
||||||
|
"access_token",
|
||||||
|
"bearer_token",
|
||||||
|
"token",
|
||||||
|
"api_key",
|
||||||
|
),
|
||||||
|
source_kind="credential_envelope",
|
||||||
|
has_secret=bool(row.secret_data),
|
||||||
|
metadata={
|
||||||
|
"credential_envelope_id": row.id,
|
||||||
|
"credential_kind": row.credential_kind,
|
||||||
|
"inherit_to_lower_scopes": True,
|
||||||
|
"revision": row.revision,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def list_connector_credential_rows(
|
def list_connector_credential_rows(
|
||||||
@@ -348,6 +512,27 @@ def _policy_source_response(source: ConnectorPolicySource) -> dict[str, Any]:
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _envelope_credential_mode(kind: str, secret_values: Mapping[str, Any] | list[str]) -> str:
|
||||||
|
keys = {str(key) for key in secret_values}
|
||||||
|
if kind in {"token", "oauth2", "api_key"} or keys.intersection(
|
||||||
|
{"access_token", "bearer_token", "token", "api_key"}
|
||||||
|
):
|
||||||
|
return "token"
|
||||||
|
return "basic"
|
||||||
|
|
||||||
|
|
||||||
|
def _clean_public_value(values: Mapping[str, Any] | None, key: str) -> str | None:
|
||||||
|
return _clean((values or {}).get(key))
|
||||||
|
|
||||||
|
|
||||||
|
def _first_secret(values: Mapping[str, Any], *keys: str) -> str | None:
|
||||||
|
for key in keys:
|
||||||
|
value = _clean(values.get(key))
|
||||||
|
if value is not None:
|
||||||
|
return value
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
def _clean(value: object) -> str | None:
|
def _clean(value: object) -> str | None:
|
||||||
if value is None:
|
if value is None:
|
||||||
return None
|
return None
|
||||||
|
|||||||
@@ -0,0 +1,139 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from collections import deque
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
|
||||||
|
from govoplan_files.backend.storage.connector_browse import (
|
||||||
|
ConnectorBrowseItem,
|
||||||
|
browse_connector_profile,
|
||||||
|
normalize_connector_browse_path,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.storage.connector_profiles import ConnectorProfile
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class ConnectorFolderSkip:
|
||||||
|
path: str
|
||||||
|
reason: str
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(slots=True)
|
||||||
|
class ConnectorFolderDiscovery:
|
||||||
|
files: list[ConnectorBrowseItem] = field(default_factory=list)
|
||||||
|
skipped: list[ConnectorFolderSkip] = field(default_factory=list)
|
||||||
|
truncated: bool = False
|
||||||
|
|
||||||
|
|
||||||
|
def discover_connector_folder(
|
||||||
|
profile: ConnectorProfile,
|
||||||
|
*,
|
||||||
|
path: str,
|
||||||
|
library_id: str | None,
|
||||||
|
recursive: bool,
|
||||||
|
max_files: int,
|
||||||
|
max_depth: int,
|
||||||
|
) -> ConnectorFolderDiscovery:
|
||||||
|
"""Discover a bounded, deterministic set of files below one connector folder."""
|
||||||
|
|
||||||
|
root_path = normalize_connector_browse_path(path)
|
||||||
|
queue: deque[tuple[str, int]] = deque([(root_path, 0)])
|
||||||
|
visited_folders: set[str] = set()
|
||||||
|
seen_files: set[tuple[str, str]] = set()
|
||||||
|
result = ConnectorFolderDiscovery()
|
||||||
|
|
||||||
|
while queue:
|
||||||
|
folder_path, depth = queue.popleft()
|
||||||
|
if folder_path in visited_folders:
|
||||||
|
continue
|
||||||
|
visited_folders.add(folder_path)
|
||||||
|
|
||||||
|
continuation_token: str | None = None
|
||||||
|
seen_tokens: set[str] = set()
|
||||||
|
while True:
|
||||||
|
items = browse_connector_profile(
|
||||||
|
profile,
|
||||||
|
path=folder_path,
|
||||||
|
library_id=library_id,
|
||||||
|
continuation_token=continuation_token,
|
||||||
|
)
|
||||||
|
for item in items:
|
||||||
|
item_path = normalize_connector_browse_path(item.path)
|
||||||
|
if item.kind == "file":
|
||||||
|
identity = (str(item.external_id or ""), item_path)
|
||||||
|
if identity in seen_files:
|
||||||
|
continue
|
||||||
|
if len(result.files) >= max_files:
|
||||||
|
result.truncated = True
|
||||||
|
result.skipped.append(
|
||||||
|
ConnectorFolderSkip(
|
||||||
|
path=item_path or folder_path,
|
||||||
|
reason=f"The manual sync limit of {max_files} files was reached.",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return result
|
||||||
|
seen_files.add(identity)
|
||||||
|
result.files.append(item)
|
||||||
|
continue
|
||||||
|
|
||||||
|
if item.kind == "folder":
|
||||||
|
if not recursive:
|
||||||
|
result.skipped.append(
|
||||||
|
ConnectorFolderSkip(
|
||||||
|
path=item_path,
|
||||||
|
reason="Subfolder skipped because recursive sync is disabled.",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
elif depth >= max_depth:
|
||||||
|
result.skipped.append(
|
||||||
|
ConnectorFolderSkip(
|
||||||
|
path=item_path,
|
||||||
|
reason=f"Subfolder skipped at the configured depth limit of {max_depth}.",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
queue.append((item_path, depth + 1))
|
||||||
|
continue
|
||||||
|
|
||||||
|
result.skipped.append(
|
||||||
|
ConnectorFolderSkip(
|
||||||
|
path=item_path,
|
||||||
|
reason=f"Connector item kind {item.kind!r} cannot be synchronized as a file.",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
next_token = _next_continuation_token(items)
|
||||||
|
if not next_token or next_token in seen_tokens:
|
||||||
|
break
|
||||||
|
seen_tokens.add(next_token)
|
||||||
|
continuation_token = next_token
|
||||||
|
|
||||||
|
result.files.sort(key=lambda item: normalize_connector_browse_path(item.path).casefold())
|
||||||
|
result.skipped.sort(key=lambda item: (item.path.casefold(), item.reason))
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def connector_relative_path(*, space_root: str, item_path: str) -> str:
|
||||||
|
root = normalize_connector_browse_path(space_root)
|
||||||
|
item = normalize_connector_browse_path(item_path)
|
||||||
|
if not root:
|
||||||
|
return item
|
||||||
|
if item == root:
|
||||||
|
return item.rsplit("/", 1)[-1]
|
||||||
|
prefix = f"{root}/"
|
||||||
|
if not item.startswith(prefix):
|
||||||
|
raise ValueError("Connector returned a file outside the linked connector-space root")
|
||||||
|
return item[len(prefix) :]
|
||||||
|
|
||||||
|
|
||||||
|
def join_connector_path(*parts: str | None) -> str:
|
||||||
|
return normalize_connector_browse_path(
|
||||||
|
"/".join(str(part or "").strip("/\\") for part in parts if str(part or "").strip("/\\"))
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _next_continuation_token(items: list[ConnectorBrowseItem]) -> str | None:
|
||||||
|
for item in reversed(items):
|
||||||
|
value = item.metadata.get("next_continuation_token")
|
||||||
|
if value is not None and str(value).strip():
|
||||||
|
return str(value).strip()
|
||||||
|
return None
|
||||||
@@ -314,15 +314,20 @@ def _read_s3_file(profile: ConnectorProfile, *, library_id: str, path: str, max_
|
|||||||
if not key:
|
if not key:
|
||||||
raise ConnectorImportError("S3 import requires an object key")
|
raise ConnectorImportError("S3 import requires an object key")
|
||||||
client = _s3_import_client(profile)
|
client = _s3_import_client(profile)
|
||||||
detail = _s3_object_detail(client, bucket=bucket, key=key, max_bytes=max_bytes)
|
try:
|
||||||
version_id = _clean(detail.get("VersionId"))
|
detail = _s3_object_detail(client, bucket=bucket, key=key, max_bytes=max_bytes)
|
||||||
response, data = _download_s3_object(
|
version_id = _clean(detail.get("VersionId"))
|
||||||
client,
|
response, data = _download_s3_object(
|
||||||
bucket=bucket,
|
client,
|
||||||
key=key,
|
bucket=bucket,
|
||||||
version_id=version_id,
|
key=key,
|
||||||
max_bytes=max_bytes,
|
version_id=version_id,
|
||||||
)
|
max_bytes=max_bytes,
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
close = getattr(client, "close", None)
|
||||||
|
if callable(close):
|
||||||
|
close()
|
||||||
content_type = _clean(response.get("ContentType") if isinstance(response, dict) else None) or _clean(detail.get("ContentType")) or mimetypes.guess_type(key)[0]
|
content_type = _clean(response.get("ContentType") if isinstance(response, dict) else None) or _clean(detail.get("ContentType")) or mimetypes.guess_type(key)[0]
|
||||||
etag = _clean(response.get("ETag") if isinstance(response, dict) else None) or _clean(detail.get("ETag"))
|
etag = _clean(response.get("ETag") if isinstance(response, dict) else None) or _clean(detail.get("ETag"))
|
||||||
filename = filename_from_path(key)
|
filename = filename_from_path(key)
|
||||||
|
|||||||
@@ -246,11 +246,11 @@ def _applied_fields(policy: Mapping[str, Any]) -> tuple[str, ...]:
|
|||||||
|
|
||||||
def _matches_field(request: ConnectorAccessRequest, field: str, patterns: list[str]) -> bool:
|
def _matches_field(request: ConnectorAccessRequest, field: str, patterns: list[str]) -> bool:
|
||||||
if field == "connectors":
|
if field == "connectors":
|
||||||
return _matches_exact(request.connector_id, patterns)
|
return _matches_reference(request.connector_id, patterns)
|
||||||
if field == "credentials":
|
if field == "credentials":
|
||||||
return _matches_exact(request.credential_id, patterns)
|
return _matches_reference(request.credential_id, patterns)
|
||||||
if field == "providers":
|
if field == "providers":
|
||||||
return _matches_exact(request.provider, patterns)
|
return _matches_reference(request.provider, patterns)
|
||||||
if field == "external_ids":
|
if field == "external_ids":
|
||||||
return _matches_glob(request.external_id, patterns)
|
return _matches_glob(request.external_id, patterns)
|
||||||
if field == "external_paths":
|
if field == "external_paths":
|
||||||
@@ -260,11 +260,11 @@ def _matches_field(request: ConnectorAccessRequest, field: str, patterns: list[s
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
def _matches_exact(value: str | None, patterns: list[str]) -> bool:
|
def _matches_reference(value: str | None, patterns: list[str]) -> bool:
|
||||||
if value is None:
|
if value is None:
|
||||||
return False
|
return False
|
||||||
clean = value.casefold()
|
clean = value.casefold()
|
||||||
return any(pattern == "*" or clean == pattern.casefold() for pattern in patterns)
|
return any(fnmatchcase(clean, pattern.casefold()) for pattern in patterns)
|
||||||
|
|
||||||
|
|
||||||
def _matches_glob(value: str | None, patterns: list[str]) -> bool:
|
def _matches_glob(value: str | None, patterns: list[str]) -> bool:
|
||||||
|
|||||||
@@ -10,7 +10,13 @@ from govoplan_core.security.secrets import decrypt_secret, encrypt_secret
|
|||||||
from govoplan_files.backend.db.models import FileConnectorCredential, FileConnectorProfile
|
from govoplan_files.backend.db.models import FileConnectorCredential, FileConnectorProfile
|
||||||
from govoplan_files.backend.storage.common import FileStorageError
|
from govoplan_files.backend.storage.common import FileStorageError
|
||||||
from govoplan_files.backend.storage.connector_deployment import reject_api_controlled_deployment_references
|
from govoplan_files.backend.storage.connector_deployment import reject_api_controlled_deployment_references
|
||||||
from govoplan_files.backend.storage.connector_credential_store import connector_credential_from_row, credential_rows_by_id
|
from govoplan_files.backend.storage.connector_credential_store import (
|
||||||
|
ConnectorCredential,
|
||||||
|
connector_credential_from_row,
|
||||||
|
credential_rows_by_id,
|
||||||
|
resolve_reusable_connector_credential,
|
||||||
|
reusable_credential_id,
|
||||||
|
)
|
||||||
from govoplan_files.backend.storage.connector_policy import connector_policy_sources_from_payload
|
from govoplan_files.backend.storage.connector_policy import connector_policy_sources_from_payload
|
||||||
from govoplan_files.backend.storage.connector_profiles import ConnectorProfile, supported_connector_providers
|
from govoplan_files.backend.storage.connector_profiles import ConnectorProfile, supported_connector_providers
|
||||||
|
|
||||||
@@ -50,10 +56,28 @@ def select_database_connector_profiles(
|
|||||||
profile_ids = {row.id for row in rows}
|
profile_ids = {row.id for row in rows}
|
||||||
if row_visible is not None:
|
if row_visible is not None:
|
||||||
rows = [row for row in rows if row_visible(row)]
|
rows = [row for row in rows if row_visible(row)]
|
||||||
credential_ids = {_clean(row.credential_profile_id) for row in rows if _clean(row.credential_profile_id)}
|
credential_ids = {
|
||||||
|
_clean(row.credential_profile_id)
|
||||||
|
for row in rows
|
||||||
|
if _clean(row.credential_profile_id) and not reusable_credential_id(row.credential_profile_id)
|
||||||
|
}
|
||||||
credentials = credential_rows_by_id(session, tenant_id=tenant_id, credential_ids={item for item in credential_ids if item}, include_disabled=include_disabled)
|
credentials = credential_rows_by_id(session, tenant_id=tenant_id, credential_ids={item for item in credential_ids if item}, include_disabled=include_disabled)
|
||||||
return (
|
return (
|
||||||
[connector_profile_from_row(row, credential_row=credentials.get(row.credential_profile_id or "")) for row in rows],
|
[
|
||||||
|
connector_profile_from_row(
|
||||||
|
row,
|
||||||
|
credential_row=(
|
||||||
|
_resolve_profile_reusable_credential(
|
||||||
|
session,
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
row=row,
|
||||||
|
)
|
||||||
|
if reusable_credential_id(row.credential_profile_id)
|
||||||
|
else credentials.get(row.credential_profile_id or "")
|
||||||
|
),
|
||||||
|
)
|
||||||
|
for row in rows
|
||||||
|
],
|
||||||
profile_ids,
|
profile_ids,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -73,7 +97,10 @@ def list_connector_profile_rows(
|
|||||||
return query.order_by(FileConnectorProfile.scope_type.asc(), FileConnectorProfile.label.asc()).all()
|
return query.order_by(FileConnectorProfile.scope_type.asc(), FileConnectorProfile.label.asc()).all()
|
||||||
|
|
||||||
|
|
||||||
def connector_profile_from_row(row: FileConnectorProfile, credential_row: FileConnectorCredential | None = None) -> ConnectorProfile:
|
def connector_profile_from_row(
|
||||||
|
row: FileConnectorProfile,
|
||||||
|
credential_row: FileConnectorCredential | ConnectorCredential | None = None,
|
||||||
|
) -> ConnectorProfile:
|
||||||
policy_sources = []
|
policy_sources = []
|
||||||
if row.policy:
|
if row.policy:
|
||||||
policy_sources = connector_policy_sources_from_payload({
|
policy_sources = connector_policy_sources_from_payload({
|
||||||
@@ -82,9 +109,18 @@ def connector_profile_from_row(row: FileConnectorProfile, credential_row: FileCo
|
|||||||
"label": row.label,
|
"label": row.label,
|
||||||
"policy": row.policy,
|
"policy": row.policy,
|
||||||
})
|
})
|
||||||
credential = connector_credential_from_row(credential_row) if credential_row is not None else None
|
credential = (
|
||||||
|
credential_row
|
||||||
|
if isinstance(credential_row, ConnectorCredential)
|
||||||
|
else connector_credential_from_row(credential_row)
|
||||||
|
if credential_row is not None
|
||||||
|
else None
|
||||||
|
)
|
||||||
if credential:
|
if credential:
|
||||||
policy_sources.extend(credential.policy_sources)
|
policy_sources.extend(credential.policy_sources)
|
||||||
|
metadata = dict(row.metadata_ or {})
|
||||||
|
if reusable_credential_id(row.credential_profile_id) and credential is None:
|
||||||
|
metadata["credential_unavailable"] = True
|
||||||
return ConnectorProfile(
|
return ConnectorProfile(
|
||||||
id=row.id,
|
id=row.id,
|
||||||
label=row.label,
|
label=row.label,
|
||||||
@@ -105,11 +141,30 @@ def connector_profile_from_row(row: FileConnectorProfile, credential_row: FileCo
|
|||||||
token_value=credential.token_value if credential else decrypt_secret(row.token_encrypted),
|
token_value=credential.token_value if credential else decrypt_secret(row.token_encrypted),
|
||||||
capabilities=tuple(_string_list(row.capabilities)),
|
capabilities=tuple(_string_list(row.capabilities)),
|
||||||
policy_sources=tuple(policy_sources),
|
policy_sources=tuple(policy_sources),
|
||||||
metadata=dict(row.metadata_ or {}),
|
metadata=metadata,
|
||||||
source_kind="database",
|
source_kind="database",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _resolve_profile_reusable_credential(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
row: FileConnectorProfile,
|
||||||
|
) -> ConnectorCredential | None:
|
||||||
|
try:
|
||||||
|
return resolve_reusable_connector_credential(
|
||||||
|
session,
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
credential_ref=row.credential_profile_id or "",
|
||||||
|
profile_id=row.id,
|
||||||
|
scope_type=row.scope_type,
|
||||||
|
scope_id=row.scope_id,
|
||||||
|
)
|
||||||
|
except FileStorageError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
def get_connector_profile_row(
|
def get_connector_profile_row(
|
||||||
session: Session,
|
session: Session,
|
||||||
*,
|
*,
|
||||||
|
|||||||
@@ -32,6 +32,7 @@ class ConnectorProviderDescriptor:
|
|||||||
"installed": self.installed,
|
"installed": self.installed,
|
||||||
"browse_supported": self.browse_supported,
|
"browse_supported": self.browse_supported,
|
||||||
"import_supported": self.import_supported,
|
"import_supported": self.import_supported,
|
||||||
|
"write_supported": self.provider == "s3",
|
||||||
"optional_dependency": self.optional_dependency,
|
"optional_dependency": self.optional_dependency,
|
||||||
"permission_model": self.permission_model,
|
"permission_model": self.permission_model,
|
||||||
"sync_strategy": self.sync_strategy,
|
"sync_strategy": self.sync_strategy,
|
||||||
@@ -110,7 +111,7 @@ def connector_provider_descriptors() -> tuple[ConnectorProviderDescriptor, ...]:
|
|||||||
conflict_strategy="Managed import/sync uses existing files conflict_strategy handling after download.",
|
conflict_strategy="Managed import/sync uses existing files conflict_strategy handling after download.",
|
||||||
preview_strategy="Previews are generated from the frozen managed file after import or sync, not directly from the share.",
|
preview_strategy="Previews are generated from the frozen managed file after import or sync, not directly from the share.",
|
||||||
audit_events=("files.connector.imported", "files.connector.synced", "files.connector.accessed"),
|
audit_events=("files.connector.imported", "files.connector.synced", "files.connector.accessed"),
|
||||||
notes="Browse/import logic is implemented, but live smbprotocol access fails closed until the SDK supports connection-time DNS/IP pinning for initial connections and DFS referrals.",
|
notes="Initial sessions, reconnects, aliases, and DFS referral targets use the Files-owned pinned smbprotocol transport and the deployment-wide private-network policy.",
|
||||||
),
|
),
|
||||||
ConnectorProviderDescriptor(
|
ConnectorProviderDescriptor(
|
||||||
provider="s3",
|
provider="s3",
|
||||||
@@ -125,7 +126,7 @@ def connector_provider_descriptors() -> tuple[ConnectorProviderDescriptor, ...]:
|
|||||||
conflict_strategy="Managed import/sync uses existing files conflict_strategy handling after object download.",
|
conflict_strategy="Managed import/sync uses existing files conflict_strategy handling after object download.",
|
||||||
preview_strategy="Previews are generated from the frozen managed file after import or sync, not directly from the bucket.",
|
preview_strategy="Previews are generated from the frozen managed file after import or sync, not directly from the bucket.",
|
||||||
audit_events=("files.connector.imported", "files.connector.synced", "files.connector.accessed"),
|
audit_events=("files.connector.imported", "files.connector.synced", "files.connector.accessed"),
|
||||||
notes="Browse/import logic is implemented, but live boto3 access fails closed until its HTTP transport supports connection-time DNS/IP pinning and redirect revalidation.",
|
notes="Every botocore connection pool uses the Files-owned pinned transport, including retries, redirects, endpoint discovery, and virtual-host aliases; outbound proxies and ambient credential discovery are disabled.",
|
||||||
),
|
),
|
||||||
ConnectorProviderDescriptor(
|
ConnectorProviderDescriptor(
|
||||||
provider="sharepoint",
|
provider="sharepoint",
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ from govoplan_files.backend.storage.connector_profiles import ConnectorProfile
|
|||||||
|
|
||||||
|
|
||||||
SYNC_MODES = {"manual"}
|
SYNC_MODES = {"manual"}
|
||||||
|
WRITABLE_PROVIDERS = {"s3"}
|
||||||
|
|
||||||
|
|
||||||
def connector_space_owner_id(space: FileConnectorSpace) -> str:
|
def connector_space_owner_id(space: FileConnectorSpace) -> str:
|
||||||
@@ -40,6 +41,7 @@ def create_connector_space(
|
|||||||
ensure_owner_access(session, tenant_id=tenant_id, owner_type=owner_type, owner_id=owner_id, user_id=user_id, is_admin=is_admin)
|
ensure_owner_access(session, tenant_id=tenant_id, owner_type=owner_type, owner_id=owner_id, user_id=user_id, is_admin=is_admin)
|
||||||
label = _normalize_label(label)
|
label = _normalize_label(label)
|
||||||
sync_mode = _normalize_sync_mode(sync_mode)
|
sync_mode = _normalize_sync_mode(sync_mode)
|
||||||
|
validate_connector_space_write_mode(profile, read_only=read_only)
|
||||||
remote_path = normalize_connector_browse_path(remote_path)
|
remote_path = normalize_connector_browse_path(remote_path)
|
||||||
library_id = _clean_optional(library_id)
|
library_id = _clean_optional(library_id)
|
||||||
|
|
||||||
@@ -145,6 +147,7 @@ def update_connector_space(
|
|||||||
library_id: str | None = None,
|
library_id: str | None = None,
|
||||||
remote_path: str | None = None,
|
remote_path: str | None = None,
|
||||||
sync_mode: str | None = None,
|
sync_mode: str | None = None,
|
||||||
|
read_only: bool | None = None,
|
||||||
is_active: bool | None = None,
|
is_active: bool | None = None,
|
||||||
metadata: Mapping[str, Any] | None = None,
|
metadata: Mapping[str, Any] | None = None,
|
||||||
is_admin: bool = False,
|
is_admin: bool = False,
|
||||||
@@ -178,6 +181,8 @@ def update_connector_space(
|
|||||||
space.remote_path = normalize_connector_browse_path(remote_path)
|
space.remote_path = normalize_connector_browse_path(remote_path)
|
||||||
if sync_mode is not None:
|
if sync_mode is not None:
|
||||||
space.sync_mode = _normalize_sync_mode(sync_mode)
|
space.sync_mode = _normalize_sync_mode(sync_mode)
|
||||||
|
if read_only is not None:
|
||||||
|
space.read_only = bool(read_only)
|
||||||
if is_active is not None:
|
if is_active is not None:
|
||||||
space.is_active = bool(is_active)
|
space.is_active = bool(is_active)
|
||||||
if metadata is not None:
|
if metadata is not None:
|
||||||
@@ -187,6 +192,17 @@ def update_connector_space(
|
|||||||
return space
|
return space
|
||||||
|
|
||||||
|
|
||||||
|
def validate_connector_space_write_mode(
|
||||||
|
profile: ConnectorProfile, *, read_only: bool
|
||||||
|
) -> None:
|
||||||
|
if read_only:
|
||||||
|
return
|
||||||
|
if profile.provider not in WRITABLE_PROVIDERS or "write" not in profile.capabilities:
|
||||||
|
raise FileStorageError(
|
||||||
|
"Two-way mode requires an S3 connection with the write capability enabled"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def soft_delete_connector_space(
|
def soft_delete_connector_space(
|
||||||
session: Session,
|
session: Session,
|
||||||
space: FileConnectorSpace,
|
space: FileConnectorSpace,
|
||||||
|
|||||||
@@ -136,11 +136,6 @@ def connector_profile_usable_for_import(profile: ConnectorProfile) -> bool:
|
|||||||
and descriptor.import_supported
|
and descriptor.import_supported
|
||||||
):
|
):
|
||||||
return False
|
return False
|
||||||
# These SDK paths intentionally fail closed until every connection peer and
|
|
||||||
# SDK-managed redirect/referral can be pinned and revalidated.
|
|
||||||
if profile.provider in {"s3", "smb"}:
|
|
||||||
return False
|
|
||||||
|
|
||||||
# Prove that the initial root browse performed by the current Files UI is
|
# Prove that the initial root browse performed by the current Files UI is
|
||||||
# policy-allowed. A later selected remote path/item is checked again.
|
# policy-allowed. A later selected remote path/item is checked again.
|
||||||
return connector_policy_decision(
|
return connector_policy_decision(
|
||||||
|
|||||||
@@ -0,0 +1,438 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
import hashlib
|
||||||
|
from typing import Any, Mapping
|
||||||
|
|
||||||
|
from govoplan_core.core.recovery import (
|
||||||
|
RecoveryGuaranteeError,
|
||||||
|
RecoveryMode,
|
||||||
|
RecoveryPlan,
|
||||||
|
RecoveryStatus,
|
||||||
|
)
|
||||||
|
from govoplan_core.core.recovery_runtime import (
|
||||||
|
RecoveryOperationBusy,
|
||||||
|
RecoveryOperationStateConflict,
|
||||||
|
begin_durable_recovery_operation,
|
||||||
|
)
|
||||||
|
from govoplan_core.core.runtime_coordination import process_runtime_identity
|
||||||
|
from govoplan_core.db.session import get_database
|
||||||
|
from govoplan_files.backend.storage.common import FileStorageError
|
||||||
|
from govoplan_files.backend.storage.connector_browse import (
|
||||||
|
ConnectorBrowseError,
|
||||||
|
ConnectorBrowseUnsupported,
|
||||||
|
_clean,
|
||||||
|
_s3_bucket,
|
||||||
|
_s3_client,
|
||||||
|
_s3_object_key,
|
||||||
|
normalize_connector_browse_path,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.storage.connector_profiles import ConnectorProfile
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class ConnectorWriteResult:
|
||||||
|
recovery_operation_id: str
|
||||||
|
status: str
|
||||||
|
replayed: bool
|
||||||
|
provider: str
|
||||||
|
remote_path: str
|
||||||
|
revision: str | None
|
||||||
|
checksum_sha256: str
|
||||||
|
size_bytes: int
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class _HeadProbe:
|
||||||
|
observed: Mapping[str, Any] | None
|
||||||
|
verified: bool
|
||||||
|
exception_type: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def write_connector_file(
|
||||||
|
profile: ConnectorProfile,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
library_id: str | None,
|
||||||
|
remote_path: str,
|
||||||
|
data: bytes,
|
||||||
|
content_type: str | None,
|
||||||
|
expected_revision: str | None,
|
||||||
|
idempotency_key: str,
|
||||||
|
) -> ConnectorWriteResult:
|
||||||
|
if profile.provider != "s3" or "write" not in profile.capabilities:
|
||||||
|
raise FileStorageError(
|
||||||
|
"Remote writes require an S3 connection with the write capability enabled"
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
normalized_path = normalize_connector_browse_path(remote_path)
|
||||||
|
except ConnectorBrowseError as exc:
|
||||||
|
raise FileStorageError(str(exc)) from exc
|
||||||
|
if not normalized_path:
|
||||||
|
raise FileStorageError("Remote write requires an object path")
|
||||||
|
try:
|
||||||
|
bucket = _s3_bucket(profile, library_id)
|
||||||
|
key = _s3_object_key(profile, normalized_path)
|
||||||
|
except ConnectorBrowseError as exc:
|
||||||
|
raise FileStorageError(str(exc)) from exc
|
||||||
|
if not bucket or not key:
|
||||||
|
raise FileStorageError("S3 remote write requires a bucket and object path")
|
||||||
|
checksum = hashlib.sha256(data).hexdigest()
|
||||||
|
target_digest = hashlib.sha256(f"{bucket}:{key}".encode("utf-8")).hexdigest()
|
||||||
|
request = {
|
||||||
|
"tenant_id": tenant_id,
|
||||||
|
"connector_profile_id": profile.id,
|
||||||
|
"provider": profile.provider,
|
||||||
|
"remote_target_sha256": target_digest,
|
||||||
|
"content_sha256": checksum,
|
||||||
|
"size_bytes": len(data),
|
||||||
|
"expected_revision": expected_revision,
|
||||||
|
}
|
||||||
|
try:
|
||||||
|
started = begin_durable_recovery_operation(
|
||||||
|
get_database().SessionLocal,
|
||||||
|
identity=process_runtime_identity(),
|
||||||
|
module_id="files",
|
||||||
|
operation_type="connector-s3-write",
|
||||||
|
idempotency_key=f"files-connector-write:{idempotency_key}",
|
||||||
|
request=request,
|
||||||
|
recovery_plan=RecoveryPlan(
|
||||||
|
mode=RecoveryMode.FORWARD_RECOVERY,
|
||||||
|
preconditions=(
|
||||||
|
"the connector space is explicitly configured for two-way writes",
|
||||||
|
"the connection and inherited policy allow the exact remote path",
|
||||||
|
"a conditional create or expected remote revision prevents blind overwrite",
|
||||||
|
),
|
||||||
|
forward_recovery_steps=(
|
||||||
|
"inspect provider metadata for the request and content digests",
|
||||||
|
"resolve success only for an exact digest match or retry after verified absence",
|
||||||
|
),
|
||||||
|
verification_steps=(
|
||||||
|
"read S3 object metadata after the write",
|
||||||
|
"match the recorded request and content digests without downloading content",
|
||||||
|
),
|
||||||
|
),
|
||||||
|
precondition_evidence={
|
||||||
|
"remote_target_sha256": target_digest,
|
||||||
|
"content_sha256": checksum,
|
||||||
|
"size_bytes": len(data),
|
||||||
|
"conditional_write": True,
|
||||||
|
},
|
||||||
|
lease_resource_key=f"files:connector:{tenant_id}:{profile.id}:{target_digest[:40]}",
|
||||||
|
lease_ttl_seconds=15 * 60,
|
||||||
|
resource_type="file_connector_object",
|
||||||
|
resource_id=target_digest,
|
||||||
|
metadata={
|
||||||
|
"resources": ["postgresql", "s3-connector"],
|
||||||
|
"provider": "s3",
|
||||||
|
"connector_profile_id": profile.id,
|
||||||
|
},
|
||||||
|
block_unresolved_resource=True,
|
||||||
|
)
|
||||||
|
except (RecoveryOperationBusy, RecoveryOperationStateConflict) as exc:
|
||||||
|
raise FileStorageError(
|
||||||
|
"This remote object is owned by another write or unresolved recovery operation"
|
||||||
|
) from exc
|
||||||
|
except (RecoveryGuaranteeError, RuntimeError, ValueError) as exc:
|
||||||
|
raise FileStorageError(
|
||||||
|
"The Files recovery ledger is unavailable; the remote object was not changed"
|
||||||
|
) from exc
|
||||||
|
if started.replayed or started.operation is None:
|
||||||
|
return ConnectorWriteResult(
|
||||||
|
recovery_operation_id=started.operation_id,
|
||||||
|
status=started.status,
|
||||||
|
replayed=True,
|
||||||
|
provider="s3",
|
||||||
|
remote_path=normalized_path,
|
||||||
|
revision=None,
|
||||||
|
checksum_sha256=checksum,
|
||||||
|
size_bytes=len(data),
|
||||||
|
)
|
||||||
|
operation = started.operation
|
||||||
|
try:
|
||||||
|
client = _s3_client(profile)
|
||||||
|
except (ConnectorBrowseError, ConnectorBrowseUnsupported) as exc:
|
||||||
|
operation.reject(
|
||||||
|
summary="The S3 client was unavailable before the remote effect",
|
||||||
|
evidence=_verified(
|
||||||
|
{"external_effect_started": False}, exception_type=type(exc).__name__
|
||||||
|
),
|
||||||
|
)
|
||||||
|
raise FileStorageError(str(exc)) from exc
|
||||||
|
try:
|
||||||
|
try:
|
||||||
|
before = _head_object(client, bucket=bucket, key=key)
|
||||||
|
except Exception as exc:
|
||||||
|
operation.reject(
|
||||||
|
summary="Remote preconditions could not be inspected before the effect",
|
||||||
|
evidence=_verified(
|
||||||
|
{"external_effect_started": False},
|
||||||
|
exception_type=type(exc).__name__,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
raise FileStorageError("S3 remote revision lookup failed") from exc
|
||||||
|
expected = _normalize_revision(expected_revision)
|
||||||
|
if before is not None and expected is None:
|
||||||
|
operation.reject(
|
||||||
|
summary="A blind remote overwrite was rejected",
|
||||||
|
evidence=_verified(
|
||||||
|
{"remote_object_exists": True, "expected_revision_supplied": False}
|
||||||
|
),
|
||||||
|
)
|
||||||
|
raise FileStorageError(
|
||||||
|
"The remote object already exists; reload it and supply its expected revision"
|
||||||
|
)
|
||||||
|
if expected is not None and (
|
||||||
|
before is None or expected not in _observed_revisions(before)
|
||||||
|
):
|
||||||
|
operation.reject(
|
||||||
|
summary="The expected remote revision did not match",
|
||||||
|
evidence=_verified(
|
||||||
|
{
|
||||||
|
"remote_object_exists": before is not None,
|
||||||
|
"expected_revision_matches": False,
|
||||||
|
}
|
||||||
|
),
|
||||||
|
)
|
||||||
|
raise FileStorageError("The remote object changed; reload before writing")
|
||||||
|
params: dict[str, object] = {
|
||||||
|
"Bucket": bucket,
|
||||||
|
"Key": key,
|
||||||
|
"Body": data,
|
||||||
|
"Metadata": {
|
||||||
|
"govoplan-sha256": checksum,
|
||||||
|
"govoplan-request-id": started.operation_id,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
if content_type:
|
||||||
|
params["ContentType"] = content_type
|
||||||
|
if before is None:
|
||||||
|
params["IfNoneMatch"] = "*"
|
||||||
|
else:
|
||||||
|
params["IfMatch"] = str(before.get("ETag") or expected_revision or "")
|
||||||
|
try:
|
||||||
|
client.put_object(**params)
|
||||||
|
except Exception as exc:
|
||||||
|
probe = _probe_head_object(client, bucket=bucket, key=key)
|
||||||
|
if probe.verified and _matches_effect(
|
||||||
|
probe.observed,
|
||||||
|
checksum=checksum,
|
||||||
|
operation_id=started.operation_id,
|
||||||
|
):
|
||||||
|
operation.succeed(
|
||||||
|
evidence=_success_evidence(
|
||||||
|
probe.observed,
|
||||||
|
checksum=checksum,
|
||||||
|
operation_id=started.operation_id,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return _result(
|
||||||
|
started.operation_id,
|
||||||
|
normalized_path,
|
||||||
|
checksum,
|
||||||
|
len(data),
|
||||||
|
probe.observed,
|
||||||
|
)
|
||||||
|
operation.unresolved(
|
||||||
|
status=RecoveryStatus.OUTCOME_UNKNOWN,
|
||||||
|
summary="The S3 write outcome requires reconciliation",
|
||||||
|
evidence={
|
||||||
|
"remote_target_sha256": target_digest,
|
||||||
|
"observed": _public_observation(probe),
|
||||||
|
"exception_type": type(exc).__name__,
|
||||||
|
},
|
||||||
|
failure_summary="Inspect the remote object metadata before retrying this path",
|
||||||
|
)
|
||||||
|
return ConnectorWriteResult(
|
||||||
|
recovery_operation_id=started.operation_id,
|
||||||
|
status=RecoveryStatus.OUTCOME_UNKNOWN.value,
|
||||||
|
replayed=False,
|
||||||
|
provider="s3",
|
||||||
|
remote_path=normalized_path,
|
||||||
|
revision=_revision(probe.observed),
|
||||||
|
checksum_sha256=checksum,
|
||||||
|
size_bytes=len(data),
|
||||||
|
)
|
||||||
|
probe = _probe_head_object(client, bucket=bucket, key=key)
|
||||||
|
if not probe.verified:
|
||||||
|
operation.unresolved(
|
||||||
|
status=RecoveryStatus.OUTCOME_UNKNOWN,
|
||||||
|
summary="The S3 write returned but provider evidence could not be queried",
|
||||||
|
evidence={
|
||||||
|
"remote_target_sha256": target_digest,
|
||||||
|
"observed": _public_observation(probe),
|
||||||
|
},
|
||||||
|
failure_summary="Inspect the remote object metadata before retrying this path",
|
||||||
|
)
|
||||||
|
return ConnectorWriteResult(
|
||||||
|
recovery_operation_id=started.operation_id,
|
||||||
|
status=RecoveryStatus.OUTCOME_UNKNOWN.value,
|
||||||
|
replayed=False,
|
||||||
|
provider="s3",
|
||||||
|
remote_path=normalized_path,
|
||||||
|
revision=None,
|
||||||
|
checksum_sha256=checksum,
|
||||||
|
size_bytes=len(data),
|
||||||
|
)
|
||||||
|
observed = probe.observed
|
||||||
|
if not _matches_effect(
|
||||||
|
observed, checksum=checksum, operation_id=started.operation_id
|
||||||
|
):
|
||||||
|
operation.unresolved(
|
||||||
|
status=RecoveryStatus.RECOVERY_REQUIRED,
|
||||||
|
summary="The S3 write returned but exact provider evidence did not match",
|
||||||
|
evidence={
|
||||||
|
"remote_target_sha256": target_digest,
|
||||||
|
"observed": _public_observation(probe),
|
||||||
|
},
|
||||||
|
failure_summary="Reconcile the remote object before another write",
|
||||||
|
)
|
||||||
|
return ConnectorWriteResult(
|
||||||
|
recovery_operation_id=started.operation_id,
|
||||||
|
status=RecoveryStatus.RECOVERY_REQUIRED.value,
|
||||||
|
replayed=False,
|
||||||
|
provider="s3",
|
||||||
|
remote_path=normalized_path,
|
||||||
|
revision=_revision(observed),
|
||||||
|
checksum_sha256=checksum,
|
||||||
|
size_bytes=len(data),
|
||||||
|
)
|
||||||
|
operation.succeed(
|
||||||
|
evidence=_success_evidence(
|
||||||
|
observed, checksum=checksum, operation_id=started.operation_id
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return _result(
|
||||||
|
started.operation_id,
|
||||||
|
normalized_path,
|
||||||
|
checksum,
|
||||||
|
len(data),
|
||||||
|
observed,
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
close = getattr(client, "close", None)
|
||||||
|
if callable(close):
|
||||||
|
close()
|
||||||
|
|
||||||
|
|
||||||
|
def _head_object(client: Any, *, bucket: str, key: str) -> Mapping[str, Any] | None:
|
||||||
|
try:
|
||||||
|
response = client.head_object(Bucket=bucket, Key=key)
|
||||||
|
except Exception as exc:
|
||||||
|
if _is_not_found(exc):
|
||||||
|
return None
|
||||||
|
raise
|
||||||
|
if not isinstance(response, Mapping):
|
||||||
|
raise FileStorageError("S3 connector returned invalid object metadata")
|
||||||
|
return response
|
||||||
|
|
||||||
|
|
||||||
|
def _probe_head_object(client: Any, *, bucket: str, key: str) -> _HeadProbe:
|
||||||
|
try:
|
||||||
|
return _HeadProbe(
|
||||||
|
observed=_head_object(client, bucket=bucket, key=key), verified=True
|
||||||
|
)
|
||||||
|
except Exception as exc:
|
||||||
|
return _HeadProbe(
|
||||||
|
observed=None,
|
||||||
|
verified=False,
|
||||||
|
exception_type=type(exc).__name__,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _is_not_found(exc: Exception) -> bool:
|
||||||
|
response = getattr(exc, "response", None)
|
||||||
|
if not isinstance(response, Mapping):
|
||||||
|
return False
|
||||||
|
error = response.get("Error")
|
||||||
|
code = error.get("Code") if isinstance(error, Mapping) else None
|
||||||
|
status = response.get("ResponseMetadata")
|
||||||
|
http_status = status.get("HTTPStatusCode") if isinstance(status, Mapping) else None
|
||||||
|
return str(code).casefold() in {"404", "nosuchkey", "notfound"} or http_status == 404
|
||||||
|
|
||||||
|
|
||||||
|
def _metadata(observed: Mapping[str, Any] | None) -> Mapping[str, Any]:
|
||||||
|
value = observed.get("Metadata") if observed else None
|
||||||
|
return value if isinstance(value, Mapping) else {}
|
||||||
|
|
||||||
|
|
||||||
|
def _matches_effect(
|
||||||
|
observed: Mapping[str, Any] | None, *, checksum: str, operation_id: str
|
||||||
|
) -> bool:
|
||||||
|
metadata = _metadata(observed)
|
||||||
|
return bool(
|
||||||
|
observed is not None
|
||||||
|
and _clean(metadata.get("govoplan-sha256")) == checksum
|
||||||
|
and _clean(metadata.get("govoplan-request-id")) == operation_id
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _revision(observed: Mapping[str, Any] | None) -> str | None:
|
||||||
|
if observed is None:
|
||||||
|
return None
|
||||||
|
return _clean(observed.get("VersionId") or observed.get("ETag"))
|
||||||
|
|
||||||
|
|
||||||
|
def _normalize_revision(value: str | None) -> str | None:
|
||||||
|
cleaned = _clean(value)
|
||||||
|
return cleaned.strip('"') if cleaned else None
|
||||||
|
|
||||||
|
|
||||||
|
def _observed_revisions(observed: Mapping[str, Any]) -> set[str]:
|
||||||
|
return {
|
||||||
|
normalized
|
||||||
|
for value in (observed.get("VersionId"), observed.get("ETag"))
|
||||||
|
if (normalized := _normalize_revision(_clean(value))) is not None
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _success_evidence(
|
||||||
|
observed: Mapping[str, Any] | None, *, checksum: str, operation_id: str
|
||||||
|
) -> dict[str, object]:
|
||||||
|
return _verified(
|
||||||
|
{
|
||||||
|
"remote_object_present": observed is not None,
|
||||||
|
"content_digest_matches": _clean(_metadata(observed).get("govoplan-sha256")) == checksum,
|
||||||
|
"request_marker_matches": _clean(_metadata(observed).get("govoplan-request-id")) == operation_id,
|
||||||
|
},
|
||||||
|
revision=_revision(observed),
|
||||||
|
content_sha256=checksum,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _public_observation(probe: _HeadProbe) -> dict[str, object]:
|
||||||
|
observed = probe.observed
|
||||||
|
return {
|
||||||
|
"probe_verified": probe.verified,
|
||||||
|
"probe_exception_type": probe.exception_type,
|
||||||
|
"present": observed is not None,
|
||||||
|
"revision": _revision(observed),
|
||||||
|
"has_content_digest": bool(_clean(_metadata(observed).get("govoplan-sha256"))),
|
||||||
|
"has_request_marker": bool(_clean(_metadata(observed).get("govoplan-request-id"))),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _verified(checks: dict[str, object], **details: object) -> dict[str, object]:
|
||||||
|
return {"verified": True, "checks": checks, **details}
|
||||||
|
|
||||||
|
|
||||||
|
def _result(
|
||||||
|
operation_id: str,
|
||||||
|
remote_path: str,
|
||||||
|
checksum: str,
|
||||||
|
size_bytes: int,
|
||||||
|
observed: Mapping[str, Any] | None,
|
||||||
|
) -> ConnectorWriteResult:
|
||||||
|
return ConnectorWriteResult(
|
||||||
|
recovery_operation_id=operation_id,
|
||||||
|
status=RecoveryStatus.SUCCEEDED.value,
|
||||||
|
replayed=False,
|
||||||
|
provider="s3",
|
||||||
|
remote_path=remote_path,
|
||||||
|
revision=_revision(observed),
|
||||||
|
checksum_sha256=checksum,
|
||||||
|
size_bytes=size_bytes,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["ConnectorWriteResult", "write_connector_file"]
|
||||||
@@ -0,0 +1,93 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from govoplan_core.core.encryption import (
|
||||||
|
ContentProtectionRequest,
|
||||||
|
ContentUnprotectionRequest,
|
||||||
|
ProtectedContent,
|
||||||
|
encryption_content_cipher,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.runtime import get_registry
|
||||||
|
from govoplan_files.backend.storage.common import FileStorageError
|
||||||
|
|
||||||
|
|
||||||
|
FILES_PROTECTION_PROFILE = "files-server-envelope-v1"
|
||||||
|
|
||||||
|
|
||||||
|
def protect_blob_content(
|
||||||
|
session: object,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
blob_id: str,
|
||||||
|
vault_id: str,
|
||||||
|
ciphertext_ref: str,
|
||||||
|
plaintext: bytes,
|
||||||
|
actor_id: str,
|
||||||
|
content_type: str | None,
|
||||||
|
) -> ProtectedContent:
|
||||||
|
capability = encryption_content_cipher(get_registry())
|
||||||
|
if capability is None:
|
||||||
|
raise FileStorageError(
|
||||||
|
"File encryption was requested, but the Encryption module is unavailable."
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
return capability.protect_content(
|
||||||
|
session,
|
||||||
|
request=ContentProtectionRequest(
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
owner_module="files",
|
||||||
|
resource_type="file_blob",
|
||||||
|
resource_id=blob_id,
|
||||||
|
profile_id=FILES_PROTECTION_PROFILE,
|
||||||
|
vault_id=vault_id,
|
||||||
|
ciphertext_ref=ciphertext_ref,
|
||||||
|
plaintext=plaintext,
|
||||||
|
policy_decision_ref="files:explicit-vault-selection:v1",
|
||||||
|
idempotency_key=f"file-blob:{blob_id}:content:v1",
|
||||||
|
actor_id=actor_id,
|
||||||
|
metadata={
|
||||||
|
"content_type": content_type or "application/octet-stream",
|
||||||
|
},
|
||||||
|
),
|
||||||
|
)
|
||||||
|
except Exception as exc:
|
||||||
|
raise FileStorageError(
|
||||||
|
"Managed file content could not be protected by the configured vault."
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
|
||||||
|
def unprotect_blob_content(
|
||||||
|
session: object,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
blob_id: str,
|
||||||
|
envelope_id: str,
|
||||||
|
ciphertext: bytes,
|
||||||
|
) -> bytes:
|
||||||
|
capability = encryption_content_cipher(get_registry())
|
||||||
|
if capability is None:
|
||||||
|
raise FileStorageError(
|
||||||
|
"This file is encrypted and cannot be read while Encryption is unavailable."
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
return capability.unprotect_content(
|
||||||
|
session,
|
||||||
|
request=ContentUnprotectionRequest(
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
owner_module="files",
|
||||||
|
resource_type="file_blob",
|
||||||
|
resource_id=blob_id,
|
||||||
|
envelope_id=envelope_id,
|
||||||
|
ciphertext=ciphertext,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
except Exception as exc:
|
||||||
|
raise FileStorageError(
|
||||||
|
"Managed file content could not be opened with its protection envelope."
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"FILES_PROTECTION_PROFILE",
|
||||||
|
"protect_blob_content",
|
||||||
|
"unprotect_blob_content",
|
||||||
|
]
|
||||||
@@ -2,21 +2,32 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import hashlib
|
import hashlib
|
||||||
import mimetypes
|
import mimetypes
|
||||||
|
from datetime import datetime
|
||||||
from pathlib import PurePosixPath
|
from pathlib import PurePosixPath
|
||||||
from typing import Any, Iterable
|
from typing import Any, Iterable
|
||||||
from uuid import uuid4
|
from uuid import uuid4
|
||||||
|
|
||||||
from sqlalchemy import and_, or_
|
from sqlalchemy import and_, exists, func, or_
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from govoplan_core.core.campaigns import CAPABILITY_CAMPAIGNS_ACCESS, CampaignAccessProvider
|
from govoplan_core.core.campaigns import CAPABILITY_CAMPAIGNS_ACCESS, CampaignAccessProvider
|
||||||
from govoplan_files.backend.db.models import CampaignAttachmentUse, FileAsset, FileBlob, FileShare, FileVersion
|
from govoplan_files.backend.db.models import CampaignAttachmentUse, FileAsset, FileBlob, FileShare, FileVersion
|
||||||
from govoplan_files.backend.runtime import get_registry, settings
|
from govoplan_files.backend.runtime import get_registry, settings
|
||||||
from govoplan_files.backend.storage.access import ensure_owner_access, ensure_share_target_exists, user_group_ids
|
from govoplan_files.backend.storage.access import ensure_owner_access, ensure_share_target_exists, user_group_ids
|
||||||
from govoplan_files.backend.storage.backends import StorageBackendError, get_storage_backend
|
from govoplan_files.backend.storage.backends import (
|
||||||
|
StorageBackendError,
|
||||||
|
StorageObjectMissing,
|
||||||
|
get_storage_backend,
|
||||||
|
)
|
||||||
from govoplan_files.backend.storage.common import FileConflictResolution, FileStorageError, UploadedStoredFile, utcnow
|
from govoplan_files.backend.storage.common import FileConflictResolution, FileStorageError, UploadedStoredFile, utcnow
|
||||||
from govoplan_files.backend.storage.paths import filename_from_path, join_folder_filename, normalize_folder, normalize_logical_path, safe_storage_component
|
from govoplan_files.backend.storage.paths import filename_from_path, join_folder_filename, normalize_folder, normalize_logical_path
|
||||||
from govoplan_files.backend.storage.provenance import source_provenance_from_metadata
|
from govoplan_files.backend.storage.provenance import source_provenance_from_metadata
|
||||||
|
from govoplan_files.backend.storage.recovery import begin_blob_write_recovery
|
||||||
|
from govoplan_files.backend.storage.integrity import (
|
||||||
|
QUARANTINED_BLOB_STATUSES,
|
||||||
|
read_verified_blob_bytes,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.storage.share_state import effective_file_share_clause
|
||||||
|
|
||||||
|
|
||||||
def _campaign_access_provider() -> CampaignAccessProvider:
|
def _campaign_access_provider() -> CampaignAccessProvider:
|
||||||
@@ -51,8 +62,9 @@ def _storage_backend_name() -> str:
|
|||||||
return settings.file_storage_backend.lower().strip()
|
return settings.file_storage_backend.lower().strip()
|
||||||
|
|
||||||
|
|
||||||
def _storage_key(*, tenant_id: str, checksum: str, filename: str) -> str:
|
def _storage_key(*, tenant_id: str, checksum: str) -> str:
|
||||||
return f"tenants/{tenant_id}/files/{checksum[:2]}/{uuid4().hex}-{safe_storage_component(filename)}"
|
# Object locators remain opaque so recovery evidence never persists names.
|
||||||
|
return f"tenants/{tenant_id}/files/{checksum[:2]}/{uuid4().hex}.blob"
|
||||||
|
|
||||||
|
|
||||||
def _get_or_create_blob(
|
def _get_or_create_blob(
|
||||||
@@ -62,40 +74,144 @@ def _get_or_create_blob(
|
|||||||
data: bytes,
|
data: bytes,
|
||||||
filename: str,
|
filename: str,
|
||||||
content_type: str | None,
|
content_type: str | None,
|
||||||
|
actor_id: str,
|
||||||
|
encryption_vault_id: str | None = None,
|
||||||
) -> FileBlob:
|
) -> FileBlob:
|
||||||
checksum = hashlib.sha256(data).hexdigest()
|
checksum = hashlib.sha256(data).hexdigest()
|
||||||
size = len(data)
|
size = len(data)
|
||||||
|
vault_id = str(encryption_vault_id or "").strip() or None
|
||||||
|
protection_discriminator = f"vault:{vault_id}" if vault_id else "plaintext"
|
||||||
blob = (
|
blob = (
|
||||||
session.query(FileBlob)
|
session.query(FileBlob)
|
||||||
.filter(FileBlob.tenant_id == tenant_id, FileBlob.checksum_sha256 == checksum, FileBlob.size_bytes == size)
|
.filter(FileBlob.tenant_id == tenant_id, FileBlob.checksum_sha256 == checksum, FileBlob.size_bytes == size, FileBlob.protection_discriminator == protection_discriminator)
|
||||||
|
.with_for_update()
|
||||||
.one_or_none()
|
.one_or_none()
|
||||||
)
|
)
|
||||||
if blob:
|
if blob:
|
||||||
backend = get_storage_backend()
|
backend = get_storage_backend()
|
||||||
if not backend.exists(blob.storage_key):
|
repair_required = (
|
||||||
|
blob.integrity_status in QUARANTINED_BLOB_STATUSES
|
||||||
|
or blob.quarantined_at is not None
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
backend.stat(blob.storage_key)
|
||||||
|
except StorageObjectMissing:
|
||||||
|
repair_required = True
|
||||||
|
except StorageBackendError as exc:
|
||||||
|
raise FileStorageError(str(exc)) from exc
|
||||||
|
if repair_required:
|
||||||
|
repair_token = hashlib.sha256(
|
||||||
|
repr(
|
||||||
|
(
|
||||||
|
blob.integrity_status,
|
||||||
|
blob.integrity_checked_at,
|
||||||
|
blob.quarantined_at,
|
||||||
|
blob.storage_checksum_sha256,
|
||||||
|
)
|
||||||
|
).encode("utf-8")
|
||||||
|
).hexdigest()
|
||||||
|
recovery = begin_blob_write_recovery(
|
||||||
|
session,
|
||||||
|
backend=backend,
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
blob_id=blob.id,
|
||||||
|
storage_key=blob.storage_key,
|
||||||
|
semantic_checksum_sha256=checksum,
|
||||||
|
semantic_size_bytes=size,
|
||||||
|
protection_discriminator=protection_discriminator,
|
||||||
|
created_new=False,
|
||||||
|
repair_token=repair_token,
|
||||||
|
)
|
||||||
|
stored_data = data
|
||||||
|
expected_envelope_id = blob.encryption_envelope_id
|
||||||
|
if vault_id:
|
||||||
|
from govoplan_files.backend.storage.content_protection import protect_blob_content
|
||||||
|
|
||||||
|
protected = protect_blob_content(
|
||||||
|
session,
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
blob_id=blob.id,
|
||||||
|
vault_id=vault_id,
|
||||||
|
ciphertext_ref=blob.storage_key,
|
||||||
|
plaintext=data,
|
||||||
|
actor_id=actor_id,
|
||||||
|
content_type=content_type,
|
||||||
|
)
|
||||||
|
if blob.encryption_envelope_id not in {None, protected.envelope.envelope_id}:
|
||||||
|
raise FileStorageError("The existing encrypted blob has another protection envelope.")
|
||||||
|
stored_data = protected.ciphertext
|
||||||
|
blob.encryption_envelope_id = protected.envelope.envelope_id
|
||||||
|
expected_envelope_id = protected.envelope.envelope_id
|
||||||
|
blob.storage_checksum_sha256 = hashlib.sha256(stored_data).hexdigest()
|
||||||
|
blob.storage_size_bytes = len(stored_data)
|
||||||
|
recovery.prepare_stored_bytes(
|
||||||
|
stored_data,
|
||||||
|
envelope_id=expected_envelope_id,
|
||||||
|
)
|
||||||
try:
|
try:
|
||||||
backend.put_bytes(blob.storage_key, data, content_type=content_type)
|
backend.put_bytes(blob.storage_key, stored_data, content_type="application/octet-stream" if vault_id else content_type)
|
||||||
except StorageBackendError as exc:
|
except StorageBackendError as exc:
|
||||||
raise FileStorageError(str(exc)) from exc
|
raise FileStorageError(str(exc)) from exc
|
||||||
|
blob.integrity_status = "verified"
|
||||||
|
blob.integrity_checked_at = utcnow()
|
||||||
|
blob.integrity_failure = None
|
||||||
|
blob.quarantined_at = None
|
||||||
blob.ref_count += 1
|
blob.ref_count += 1
|
||||||
session.add(blob)
|
session.add(blob)
|
||||||
return blob
|
return blob
|
||||||
|
|
||||||
storage_key = _storage_key(tenant_id=tenant_id, checksum=checksum, filename=filename)
|
blob_id = str(uuid4())
|
||||||
|
storage_key = _storage_key(tenant_id=tenant_id, checksum=checksum)
|
||||||
backend = get_storage_backend()
|
backend = get_storage_backend()
|
||||||
|
recovery = begin_blob_write_recovery(
|
||||||
|
session,
|
||||||
|
backend=backend,
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
blob_id=blob_id,
|
||||||
|
storage_key=storage_key,
|
||||||
|
semantic_checksum_sha256=checksum,
|
||||||
|
semantic_size_bytes=size,
|
||||||
|
protection_discriminator=protection_discriminator,
|
||||||
|
created_new=True,
|
||||||
|
)
|
||||||
|
stored_data = data
|
||||||
|
envelope_id = None
|
||||||
|
if vault_id:
|
||||||
|
from govoplan_files.backend.storage.content_protection import protect_blob_content
|
||||||
|
|
||||||
|
protected = protect_blob_content(
|
||||||
|
session,
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
blob_id=blob_id,
|
||||||
|
vault_id=vault_id,
|
||||||
|
ciphertext_ref=storage_key,
|
||||||
|
plaintext=data,
|
||||||
|
actor_id=actor_id,
|
||||||
|
content_type=content_type,
|
||||||
|
)
|
||||||
|
stored_data = protected.ciphertext
|
||||||
|
envelope_id = protected.envelope.envelope_id
|
||||||
|
recovery.prepare_stored_bytes(stored_data, envelope_id=envelope_id)
|
||||||
try:
|
try:
|
||||||
backend.put_bytes(storage_key, data, content_type=content_type)
|
backend.put_bytes(storage_key, stored_data, content_type="application/octet-stream" if vault_id else content_type)
|
||||||
except StorageBackendError as exc:
|
except StorageBackendError as exc:
|
||||||
raise FileStorageError(str(exc)) from exc
|
raise FileStorageError(str(exc)) from exc
|
||||||
blob = FileBlob(
|
blob = FileBlob(
|
||||||
|
id=blob_id,
|
||||||
tenant_id=tenant_id,
|
tenant_id=tenant_id,
|
||||||
storage_backend=_storage_backend_name(),
|
storage_backend=_storage_backend_name(),
|
||||||
storage_bucket=_storage_bucket_name(),
|
storage_bucket=_storage_bucket_name(),
|
||||||
storage_key=storage_key,
|
storage_key=storage_key,
|
||||||
checksum_sha256=checksum,
|
checksum_sha256=checksum,
|
||||||
size_bytes=size,
|
size_bytes=size,
|
||||||
|
protection_discriminator=protection_discriminator,
|
||||||
|
encryption_envelope_id=envelope_id,
|
||||||
|
storage_checksum_sha256=hashlib.sha256(stored_data).hexdigest() if vault_id else None,
|
||||||
|
storage_size_bytes=len(stored_data) if vault_id else None,
|
||||||
content_type=content_type,
|
content_type=content_type,
|
||||||
ref_count=1,
|
ref_count=1,
|
||||||
|
integrity_status="verified",
|
||||||
|
integrity_checked_at=utcnow(),
|
||||||
)
|
)
|
||||||
session.add(blob)
|
session.add(blob)
|
||||||
session.flush()
|
session.flush()
|
||||||
@@ -120,6 +236,7 @@ def create_file_asset(
|
|||||||
conflict_strategy: str = "reject",
|
conflict_strategy: str = "reject",
|
||||||
conflict_resolutions: Iterable[FileConflictResolution] | None = None,
|
conflict_resolutions: Iterable[FileConflictResolution] | None = None,
|
||||||
is_admin: bool = False,
|
is_admin: bool = False,
|
||||||
|
encryption_vault_id: str | None = None,
|
||||||
) -> UploadedStoredFile:
|
) -> UploadedStoredFile:
|
||||||
owner_type = owner_type.lower().strip()
|
owner_type = owner_type.lower().strip()
|
||||||
ensure_owner_access(session, tenant_id=tenant_id, owner_type=owner_type, owner_id=owner_id, user_id=user_id, is_admin=is_admin)
|
ensure_owner_access(session, tenant_id=tenant_id, owner_type=owner_type, owner_id=owner_id, user_id=user_id, is_admin=is_admin)
|
||||||
@@ -147,7 +264,7 @@ def create_file_asset(
|
|||||||
elif action == "rename":
|
elif action == "rename":
|
||||||
logical_path = _next_available_logical_path(session, tenant_id=tenant_id, owner_type=owner_type, owner_id=owner_id, desired_path=logical_path)
|
logical_path = _next_available_logical_path(session, tenant_id=tenant_id, owner_type=owner_type, owner_id=owner_id, desired_path=logical_path)
|
||||||
|
|
||||||
blob = _get_or_create_blob(session, tenant_id=tenant_id, data=data, filename=safe_filename, content_type=content_type)
|
blob = _get_or_create_blob(session, tenant_id=tenant_id, data=data, filename=safe_filename, content_type=content_type, actor_id=user_id, encryption_vault_id=encryption_vault_id)
|
||||||
asset = FileAsset(
|
asset = FileAsset(
|
||||||
tenant_id=tenant_id,
|
tenant_id=tenant_id,
|
||||||
owner_type=owner_type,
|
owner_type=owner_type,
|
||||||
@@ -279,6 +396,7 @@ def update_file_asset_content(
|
|||||||
data: bytes,
|
data: bytes,
|
||||||
content_type: str | None,
|
content_type: str | None,
|
||||||
metadata: dict[str, Any],
|
metadata: dict[str, Any],
|
||||||
|
encryption_vault_id: str | None = None,
|
||||||
) -> tuple[UploadedStoredFile, str]:
|
) -> tuple[UploadedStoredFile, str]:
|
||||||
if asset.tenant_id != tenant_id or asset.deleted_at is not None:
|
if asset.tenant_id != tenant_id or asset.deleted_at is not None:
|
||||||
raise FileStorageError("File not found")
|
raise FileStorageError("File not found")
|
||||||
@@ -289,10 +407,23 @@ def update_file_asset_content(
|
|||||||
checksum = hashlib.sha256(data).hexdigest()
|
checksum = hashlib.sha256(data).hexdigest()
|
||||||
asset.metadata_ = metadata
|
asset.metadata_ = metadata
|
||||||
session.add(asset)
|
session.add(asset)
|
||||||
if current_blob.checksum_sha256 == checksum and current_blob.size_bytes == len(data):
|
inherited_vault_id = encryption_vault_id
|
||||||
|
if inherited_vault_id is None and current_blob.encryption_envelope_id:
|
||||||
|
prefix = "vault:"
|
||||||
|
if current_blob.protection_discriminator.startswith(prefix):
|
||||||
|
inherited_vault_id = current_blob.protection_discriminator[len(prefix) :]
|
||||||
|
inherited_vault_id = str(inherited_vault_id or "").strip() or None
|
||||||
|
target_protection = (
|
||||||
|
f"vault:{inherited_vault_id}" if inherited_vault_id else "plaintext"
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
current_blob.checksum_sha256 == checksum
|
||||||
|
and current_blob.size_bytes == len(data)
|
||||||
|
and current_blob.protection_discriminator == target_protection
|
||||||
|
):
|
||||||
return UploadedStoredFile(asset=asset, version=current_version, blob=current_blob), "unchanged"
|
return UploadedStoredFile(asset=asset, version=current_version, blob=current_blob), "unchanged"
|
||||||
|
|
||||||
blob = _get_or_create_blob(session, tenant_id=tenant_id, data=data, filename=safe_filename, content_type=content_type)
|
blob = _get_or_create_blob(session, tenant_id=tenant_id, data=data, filename=safe_filename, content_type=content_type, actor_id=user_id, encryption_vault_id=inherited_vault_id)
|
||||||
version = FileVersion(
|
version = FileVersion(
|
||||||
tenant_id=tenant_id,
|
tenant_id=tenant_id,
|
||||||
file_asset_id=asset.id,
|
file_asset_id=asset.id,
|
||||||
@@ -328,7 +459,7 @@ def get_asset_for_user(session: Session, *, tenant_id: str, user_id: str, asset_
|
|||||||
.filter(
|
.filter(
|
||||||
FileShare.tenant_id == tenant_id,
|
FileShare.tenant_id == tenant_id,
|
||||||
FileShare.file_asset_id == asset.id,
|
FileShare.file_asset_id == asset.id,
|
||||||
FileShare.revoked_at.is_(None),
|
effective_file_share_clause(),
|
||||||
FileShare.permission.in_(permission_values),
|
FileShare.permission.in_(permission_values),
|
||||||
or_(
|
or_(
|
||||||
(FileShare.target_type == "user") & (FileShare.target_id == user_id),
|
(FileShare.target_type == "user") & (FileShare.target_id == user_id),
|
||||||
@@ -343,6 +474,32 @@ def get_asset_for_user(session: Session, *, tenant_id: str, user_id: str, asset_
|
|||||||
return asset
|
return asset
|
||||||
|
|
||||||
|
|
||||||
|
def get_asset_for_share_management(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
user_id: str,
|
||||||
|
asset_id: str,
|
||||||
|
is_admin: bool = False,
|
||||||
|
) -> FileAsset:
|
||||||
|
asset = session.get(FileAsset, asset_id)
|
||||||
|
if not asset or asset.tenant_id != tenant_id or asset.deleted_at is not None:
|
||||||
|
raise FileStorageError("File not found")
|
||||||
|
if is_admin:
|
||||||
|
return asset
|
||||||
|
owns_asset = (
|
||||||
|
asset.owner_type == "user"
|
||||||
|
and asset.owner_user_id == user_id
|
||||||
|
) or (
|
||||||
|
asset.owner_type == "group"
|
||||||
|
and asset.owner_group_id
|
||||||
|
in user_group_ids(session, tenant_id=tenant_id, user_id=user_id)
|
||||||
|
)
|
||||||
|
if not owns_asset:
|
||||||
|
raise FileStorageError("Only file owners and administrators can manage shares")
|
||||||
|
return asset
|
||||||
|
|
||||||
|
|
||||||
def list_assets_for_user(
|
def list_assets_for_user(
|
||||||
session: Session,
|
session: Session,
|
||||||
*,
|
*,
|
||||||
@@ -352,6 +509,8 @@ def list_assets_for_user(
|
|||||||
owner_id: str | None = None,
|
owner_id: str | None = None,
|
||||||
campaign_id: str | None = None,
|
campaign_id: str | None = None,
|
||||||
path_prefix: str | None = None,
|
path_prefix: str | None = None,
|
||||||
|
campaign_usage: str | None = None,
|
||||||
|
audit_relevant: bool | None = None,
|
||||||
include_deleted: bool = False,
|
include_deleted: bool = False,
|
||||||
is_admin: bool = False,
|
is_admin: bool = False,
|
||||||
) -> list[FileAsset]:
|
) -> list[FileAsset]:
|
||||||
@@ -363,12 +522,49 @@ def list_assets_for_user(
|
|||||||
owner_id=owner_id,
|
owner_id=owner_id,
|
||||||
campaign_id=campaign_id,
|
campaign_id=campaign_id,
|
||||||
path_prefix=path_prefix,
|
path_prefix=path_prefix,
|
||||||
|
campaign_usage=campaign_usage,
|
||||||
|
audit_relevant=audit_relevant,
|
||||||
include_deleted=include_deleted,
|
include_deleted=include_deleted,
|
||||||
is_admin=is_admin,
|
is_admin=is_admin,
|
||||||
)
|
)
|
||||||
return query.order_by(FileAsset.display_path.asc(), FileAsset.updated_at.desc(), FileAsset.id.asc()).all()
|
return query.order_by(FileAsset.display_path.asc(), FileAsset.updated_at.desc(), FileAsset.id.asc()).all()
|
||||||
|
|
||||||
|
|
||||||
|
def list_recent_assets_for_user(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
user_id: str,
|
||||||
|
limit: int,
|
||||||
|
owner_type: str | None = None,
|
||||||
|
owner_id: str | None = None,
|
||||||
|
campaign_id: str | None = None,
|
||||||
|
path_prefix: str | None = None,
|
||||||
|
campaign_usage: str | None = None,
|
||||||
|
audit_relevant: bool | None = None,
|
||||||
|
is_admin: bool = False,
|
||||||
|
) -> list[FileAsset]:
|
||||||
|
"""Return a bounded recent projection through the normal access query."""
|
||||||
|
|
||||||
|
query = _asset_visibility_query_for_user(
|
||||||
|
session,
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
user_id=user_id,
|
||||||
|
owner_type=owner_type,
|
||||||
|
owner_id=owner_id,
|
||||||
|
campaign_id=campaign_id,
|
||||||
|
path_prefix=path_prefix,
|
||||||
|
campaign_usage=campaign_usage,
|
||||||
|
audit_relevant=audit_relevant,
|
||||||
|
is_admin=is_admin,
|
||||||
|
)
|
||||||
|
return (
|
||||||
|
query.order_by(FileAsset.updated_at.desc(), FileAsset.id.asc())
|
||||||
|
.limit(max(1, limit))
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def list_assets_for_user_window(
|
def list_assets_for_user_window(
|
||||||
session: Session,
|
session: Session,
|
||||||
*,
|
*,
|
||||||
@@ -378,6 +574,8 @@ def list_assets_for_user_window(
|
|||||||
owner_id: str | None = None,
|
owner_id: str | None = None,
|
||||||
campaign_id: str | None = None,
|
campaign_id: str | None = None,
|
||||||
path_prefix: str | None = None,
|
path_prefix: str | None = None,
|
||||||
|
campaign_usage: str | None = None,
|
||||||
|
audit_relevant: bool | None = None,
|
||||||
include_deleted: bool = False,
|
include_deleted: bool = False,
|
||||||
is_admin: bool = False,
|
is_admin: bool = False,
|
||||||
page_size: int,
|
page_size: int,
|
||||||
@@ -393,6 +591,8 @@ def list_assets_for_user_window(
|
|||||||
owner_id=owner_id,
|
owner_id=owner_id,
|
||||||
campaign_id=campaign_id,
|
campaign_id=campaign_id,
|
||||||
path_prefix=path_prefix,
|
path_prefix=path_prefix,
|
||||||
|
campaign_usage=campaign_usage,
|
||||||
|
audit_relevant=audit_relevant,
|
||||||
include_deleted=include_deleted,
|
include_deleted=include_deleted,
|
||||||
is_admin=is_admin,
|
is_admin=is_admin,
|
||||||
)
|
)
|
||||||
@@ -417,6 +617,8 @@ def _asset_visibility_query_for_user(
|
|||||||
owner_id: str | None = None,
|
owner_id: str | None = None,
|
||||||
campaign_id: str | None = None,
|
campaign_id: str | None = None,
|
||||||
path_prefix: str | None = None,
|
path_prefix: str | None = None,
|
||||||
|
campaign_usage: str | None = None,
|
||||||
|
audit_relevant: bool | None = None,
|
||||||
include_deleted: bool = False,
|
include_deleted: bool = False,
|
||||||
is_admin: bool = False,
|
is_admin: bool = False,
|
||||||
):
|
):
|
||||||
@@ -430,30 +632,97 @@ def _asset_visibility_query_for_user(
|
|||||||
if owner_type == "group" and owner_id:
|
if owner_type == "group" and owner_id:
|
||||||
query = query.filter(FileAsset.owner_group_id == owner_id)
|
query = query.filter(FileAsset.owner_group_id == owner_id)
|
||||||
if campaign_id:
|
if campaign_id:
|
||||||
query = query.join(FileShare, FileShare.file_asset_id == FileAsset.id).filter(
|
campaign_share = exists().where(
|
||||||
FileShare.tenant_id == tenant_id,
|
FileShare.tenant_id == tenant_id,
|
||||||
|
FileShare.file_asset_id == FileAsset.id,
|
||||||
FileShare.target_type == "campaign",
|
FileShare.target_type == "campaign",
|
||||||
FileShare.target_id == campaign_id,
|
FileShare.target_id == campaign_id,
|
||||||
FileShare.revoked_at.is_(None),
|
effective_file_share_clause(),
|
||||||
)
|
)
|
||||||
|
query = query.filter(campaign_share)
|
||||||
elif not is_admin and not owner_type:
|
elif not is_admin and not owner_type:
|
||||||
group_ids = user_group_ids(session, tenant_id=tenant_id, user_id=user_id)
|
group_ids = user_group_ids(session, tenant_id=tenant_id, user_id=user_id)
|
||||||
query = query.outerjoin(FileShare, FileShare.file_asset_id == FileAsset.id).filter(
|
active_share = exists().where(
|
||||||
|
FileShare.tenant_id == tenant_id,
|
||||||
|
FileShare.file_asset_id == FileAsset.id,
|
||||||
|
effective_file_share_clause(),
|
||||||
|
or_(
|
||||||
|
(FileShare.target_type == "user") & (FileShare.target_id == user_id),
|
||||||
|
(FileShare.target_type == "group") & (FileShare.target_id.in_(group_ids)),
|
||||||
|
(FileShare.target_type == "tenant") & (FileShare.target_id == tenant_id),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
query = query.filter(
|
||||||
or_(
|
or_(
|
||||||
(FileAsset.owner_type == "user") & (FileAsset.owner_user_id == user_id),
|
(FileAsset.owner_type == "user") & (FileAsset.owner_user_id == user_id),
|
||||||
(FileAsset.owner_type == "group") & (FileAsset.owner_group_id.in_(group_ids)),
|
(FileAsset.owner_type == "group") & (FileAsset.owner_group_id.in_(group_ids)),
|
||||||
(FileShare.revoked_at.is_(None)) & (FileShare.target_type == "user") & (FileShare.target_id == user_id),
|
active_share,
|
||||||
(FileShare.revoked_at.is_(None)) & (FileShare.target_type == "group") & (FileShare.target_id.in_(group_ids)),
|
|
||||||
(FileShare.revoked_at.is_(None)) & (FileShare.target_type == "tenant") & (FileShare.target_id == tenant_id),
|
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
if path_prefix:
|
if path_prefix:
|
||||||
prefix = normalize_folder(path_prefix)
|
prefix = normalize_folder(path_prefix)
|
||||||
if prefix:
|
if prefix:
|
||||||
query = query.filter(FileAsset.display_path.like(f"{prefix}/%"))
|
query = query.filter(FileAsset.display_path.like(f"{prefix}/%"))
|
||||||
|
campaign_share_exists = exists().where(
|
||||||
|
FileShare.tenant_id == tenant_id,
|
||||||
|
FileShare.file_asset_id == FileAsset.id,
|
||||||
|
FileShare.target_type == "campaign",
|
||||||
|
effective_file_share_clause(),
|
||||||
|
)
|
||||||
|
campaign_use_exists = exists().where(
|
||||||
|
CampaignAttachmentUse.tenant_id == tenant_id,
|
||||||
|
CampaignAttachmentUse.file_asset_id == FileAsset.id,
|
||||||
|
)
|
||||||
|
if campaign_usage == "linked":
|
||||||
|
query = query.filter(or_(campaign_share_exists, campaign_use_exists))
|
||||||
|
elif campaign_usage == "unlinked":
|
||||||
|
query = query.filter(~or_(campaign_share_exists, campaign_use_exists))
|
||||||
|
|
||||||
|
if audit_relevant is not None:
|
||||||
|
sent_use_exists = exists().where(
|
||||||
|
CampaignAttachmentUse.tenant_id == tenant_id,
|
||||||
|
CampaignAttachmentUse.file_asset_id == FileAsset.id,
|
||||||
|
CampaignAttachmentUse.use_stage == "sent",
|
||||||
|
)
|
||||||
|
query = query.filter(sent_use_exists if audit_relevant else ~sent_use_exists)
|
||||||
return query
|
return query
|
||||||
|
|
||||||
|
|
||||||
|
def count_assets_for_user(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
user_id: str,
|
||||||
|
owner_type: str | None = None,
|
||||||
|
owner_id: str | None = None,
|
||||||
|
campaign_id: str | None = None,
|
||||||
|
path_prefix: str | None = None,
|
||||||
|
campaign_usage: str | None = None,
|
||||||
|
audit_relevant: bool | None = None,
|
||||||
|
include_deleted: bool = False,
|
||||||
|
is_admin: bool = False,
|
||||||
|
) -> int:
|
||||||
|
query = _asset_visibility_query_for_user(
|
||||||
|
session,
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
user_id=user_id,
|
||||||
|
owner_type=owner_type,
|
||||||
|
owner_id=owner_id,
|
||||||
|
campaign_id=campaign_id,
|
||||||
|
path_prefix=path_prefix,
|
||||||
|
campaign_usage=campaign_usage,
|
||||||
|
audit_relevant=audit_relevant,
|
||||||
|
include_deleted=include_deleted,
|
||||||
|
is_admin=is_admin,
|
||||||
|
)
|
||||||
|
return int(
|
||||||
|
query.order_by(None)
|
||||||
|
.with_entities(func.count(func.distinct(FileAsset.id)))
|
||||||
|
.scalar()
|
||||||
|
or 0
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def current_version_and_blob(session: Session, asset: FileAsset) -> tuple[FileVersion, FileBlob]:
|
def current_version_and_blob(session: Session, asset: FileAsset) -> tuple[FileVersion, FileBlob]:
|
||||||
if not asset.current_version_id:
|
if not asset.current_version_id:
|
||||||
raise FileStorageError("File has no current version")
|
raise FileStorageError("File has no current version")
|
||||||
@@ -495,10 +764,22 @@ def current_versions_and_blobs(session: Session, assets: Iterable[FileAsset]) ->
|
|||||||
def read_asset_bytes(session: Session, asset: FileAsset) -> tuple[bytes, FileVersion, FileBlob]:
|
def read_asset_bytes(session: Session, asset: FileAsset) -> tuple[bytes, FileVersion, FileBlob]:
|
||||||
version, blob = current_version_and_blob(session, asset)
|
version, blob = current_version_and_blob(session, asset)
|
||||||
backend = get_storage_backend()
|
backend = get_storage_backend()
|
||||||
try:
|
return read_verified_blob_bytes(blob, backend=backend), version, blob
|
||||||
return backend.get_bytes(blob.storage_key), version, blob
|
|
||||||
except StorageBackendError as exc:
|
|
||||||
raise FileStorageError(str(exc)) from exc
|
def read_asset_version_bytes(
|
||||||
|
session: Session,
|
||||||
|
asset: FileAsset,
|
||||||
|
version_id: str,
|
||||||
|
) -> tuple[bytes, FileVersion, FileBlob]:
|
||||||
|
version = session.get(FileVersion, version_id)
|
||||||
|
if version is None or version.file_asset_id != asset.id:
|
||||||
|
raise FileStorageError("File version not found")
|
||||||
|
blob = session.get(FileBlob, version.blob_id)
|
||||||
|
if blob is None or blob.tenant_id != asset.tenant_id:
|
||||||
|
raise FileStorageError("File blob not found")
|
||||||
|
backend = get_storage_backend()
|
||||||
|
return read_verified_blob_bytes(blob, backend=backend), version, blob
|
||||||
|
|
||||||
|
|
||||||
def share_file(
|
def share_file(
|
||||||
@@ -510,6 +791,7 @@ def share_file(
|
|||||||
target_id: str,
|
target_id: str,
|
||||||
permission: str,
|
permission: str,
|
||||||
user_id: str,
|
user_id: str,
|
||||||
|
expires_at: datetime | None = None,
|
||||||
) -> FileShare:
|
) -> FileShare:
|
||||||
target_type = target_type.lower().strip()
|
target_type = target_type.lower().strip()
|
||||||
permission = permission.lower().strip()
|
permission = permission.lower().strip()
|
||||||
@@ -519,6 +801,7 @@ def share_file(
|
|||||||
raise FileStorageError("Unsupported share target")
|
raise FileStorageError("Unsupported share target")
|
||||||
if permission not in {"read", "write", "manage"}:
|
if permission not in {"read", "write", "manage"}:
|
||||||
raise FileStorageError("Unsupported file permission")
|
raise FileStorageError("Unsupported file permission")
|
||||||
|
_validate_share_expiry(expires_at)
|
||||||
if target_type in {"user", "group", "tenant"}:
|
if target_type in {"user", "group", "tenant"}:
|
||||||
ensure_share_target_exists(tenant_id=tenant_id, target_type=target_type, target_id=target_id)
|
ensure_share_target_exists(tenant_id=tenant_id, target_type=target_type, target_id=target_id)
|
||||||
if target_type == "campaign":
|
if target_type == "campaign":
|
||||||
@@ -536,6 +819,7 @@ def share_file(
|
|||||||
)
|
)
|
||||||
if existing:
|
if existing:
|
||||||
existing.permission = permission
|
existing.permission = permission
|
||||||
|
existing.expires_at = expires_at
|
||||||
session.add(existing)
|
session.add(existing)
|
||||||
return existing
|
return existing
|
||||||
share = FileShare(
|
share = FileShare(
|
||||||
@@ -545,6 +829,7 @@ def share_file(
|
|||||||
target_id=target_id,
|
target_id=target_id,
|
||||||
permission=permission,
|
permission=permission,
|
||||||
created_by_user_id=user_id,
|
created_by_user_id=user_id,
|
||||||
|
expires_at=expires_at,
|
||||||
)
|
)
|
||||||
session.add(share)
|
session.add(share)
|
||||||
return share
|
return share
|
||||||
@@ -559,6 +844,7 @@ def share_files(
|
|||||||
target_id: str,
|
target_id: str,
|
||||||
permission: str,
|
permission: str,
|
||||||
user_id: str,
|
user_id: str,
|
||||||
|
expires_at: datetime | None = None,
|
||||||
) -> list[FileShare]:
|
) -> list[FileShare]:
|
||||||
target_type = target_type.lower().strip()
|
target_type = target_type.lower().strip()
|
||||||
permission = permission.lower().strip()
|
permission = permission.lower().strip()
|
||||||
@@ -566,6 +852,7 @@ def share_files(
|
|||||||
raise FileStorageError("Unsupported share target")
|
raise FileStorageError("Unsupported share target")
|
||||||
if permission not in {"read", "write", "manage"}:
|
if permission not in {"read", "write", "manage"}:
|
||||||
raise FileStorageError("Unsupported file permission")
|
raise FileStorageError("Unsupported file permission")
|
||||||
|
_validate_share_expiry(expires_at)
|
||||||
if target_type in {"user", "group", "tenant"}:
|
if target_type in {"user", "group", "tenant"}:
|
||||||
ensure_share_target_exists(tenant_id=tenant_id, target_type=target_type, target_id=target_id)
|
ensure_share_target_exists(tenant_id=tenant_id, target_type=target_type, target_id=target_id)
|
||||||
if target_type == "campaign":
|
if target_type == "campaign":
|
||||||
@@ -593,6 +880,7 @@ def share_files(
|
|||||||
existing = existing_by_asset.get(asset.id)
|
existing = existing_by_asset.get(asset.id)
|
||||||
if existing:
|
if existing:
|
||||||
existing.permission = permission
|
existing.permission = permission
|
||||||
|
existing.expires_at = expires_at
|
||||||
session.add(existing)
|
session.add(existing)
|
||||||
shares.append(existing)
|
shares.append(existing)
|
||||||
continue
|
continue
|
||||||
@@ -603,12 +891,88 @@ def share_files(
|
|||||||
target_id=target_id,
|
target_id=target_id,
|
||||||
permission=permission,
|
permission=permission,
|
||||||
created_by_user_id=user_id,
|
created_by_user_id=user_id,
|
||||||
|
expires_at=expires_at,
|
||||||
)
|
)
|
||||||
session.add(share)
|
session.add(share)
|
||||||
shares.append(share)
|
shares.append(share)
|
||||||
return shares
|
return shares
|
||||||
|
|
||||||
|
|
||||||
|
def list_file_shares(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
asset_id: str,
|
||||||
|
include_inactive: bool = False,
|
||||||
|
) -> list[FileShare]:
|
||||||
|
query = session.query(FileShare).filter(
|
||||||
|
FileShare.tenant_id == tenant_id,
|
||||||
|
FileShare.file_asset_id == asset_id,
|
||||||
|
)
|
||||||
|
if not include_inactive:
|
||||||
|
query = query.filter(effective_file_share_clause())
|
||||||
|
return query.order_by(FileShare.created_at.desc(), FileShare.id.desc()).all()
|
||||||
|
|
||||||
|
|
||||||
|
def revoke_file_share(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
asset_id: str,
|
||||||
|
share_id: str,
|
||||||
|
user_id: str,
|
||||||
|
) -> tuple[FileShare, bool]:
|
||||||
|
share = (
|
||||||
|
session.query(FileShare)
|
||||||
|
.filter(
|
||||||
|
FileShare.id == share_id,
|
||||||
|
FileShare.tenant_id == tenant_id,
|
||||||
|
FileShare.file_asset_id == asset_id,
|
||||||
|
)
|
||||||
|
.one_or_none()
|
||||||
|
)
|
||||||
|
if share is None:
|
||||||
|
raise FileStorageError("File share not found")
|
||||||
|
if share.revoked_at is not None:
|
||||||
|
return share, False
|
||||||
|
share.revoked_at = utcnow()
|
||||||
|
share.revoked_by_user_id = user_id
|
||||||
|
session.add(share)
|
||||||
|
return share, True
|
||||||
|
|
||||||
|
|
||||||
|
def current_file_share_for_target(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
asset_id: str,
|
||||||
|
target_type: str,
|
||||||
|
target_id: str,
|
||||||
|
) -> FileShare | None:
|
||||||
|
return (
|
||||||
|
session.query(FileShare)
|
||||||
|
.filter(
|
||||||
|
FileShare.tenant_id == tenant_id,
|
||||||
|
FileShare.file_asset_id == asset_id,
|
||||||
|
FileShare.target_type == target_type.lower().strip(),
|
||||||
|
FileShare.target_id == target_id,
|
||||||
|
FileShare.revoked_at.is_(None),
|
||||||
|
)
|
||||||
|
.order_by(FileShare.created_at.desc(), FileShare.id.desc())
|
||||||
|
.first()
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_share_expiry(expires_at: datetime | None) -> None:
|
||||||
|
if expires_at is None:
|
||||||
|
return
|
||||||
|
from govoplan_files.backend.storage.share_state import file_share_is_active
|
||||||
|
|
||||||
|
candidate = FileShare(expires_at=expires_at)
|
||||||
|
if not file_share_is_active(candidate):
|
||||||
|
raise FileStorageError("File share expiry must be in the future")
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def soft_delete_assets(session: Session, assets: Iterable[FileAsset]) -> int:
|
def soft_delete_assets(session: Session, assets: Iterable[FileAsset]) -> int:
|
||||||
count = 0
|
count = 0
|
||||||
@@ -744,7 +1108,7 @@ def _resolution_by_path(conflict_resolutions: Iterable[FileConflictResolution] |
|
|||||||
|
|
||||||
def _normalize_conflict_strategy(strategy: str | None) -> str:
|
def _normalize_conflict_strategy(strategy: str | None) -> str:
|
||||||
normalized = (strategy or "reject").lower().strip()
|
normalized = (strategy or "reject").lower().strip()
|
||||||
if normalized not in {"reject", "overwrite", "rename"}:
|
if normalized not in {"reject", "overwrite", "rename", "skip"}:
|
||||||
raise FileStorageError("Unsupported conflict strategy")
|
raise FileStorageError("Unsupported conflict strategy")
|
||||||
return normalized
|
return normalized
|
||||||
|
|
||||||
|
|||||||
@@ -185,15 +185,23 @@ class _OutboundPolicyHTTPTransport(httpx.BaseTransport):
|
|||||||
self._connection_pool.close()
|
self._connection_pool.close()
|
||||||
|
|
||||||
|
|
||||||
|
_CONNECTOR_HTTP_CLIENT = httpx.Client(
|
||||||
|
transport=_OutboundPolicyHTTPTransport(),
|
||||||
|
follow_redirects=False,
|
||||||
|
timeout=15.0,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@contextmanager
|
@contextmanager
|
||||||
def _stream_connector_request(method: str, url: str, **kwargs: Any) -> Iterator[httpx.Response]:
|
def _stream_connector_request(method: str, url: str, **kwargs: Any) -> Iterator[httpx.Response]:
|
||||||
with httpx.Client(
|
timeout = kwargs.pop("timeout", 15.0)
|
||||||
transport=_OutboundPolicyHTTPTransport(),
|
with _CONNECTOR_HTTP_CLIENT.stream(
|
||||||
follow_redirects=False,
|
method,
|
||||||
timeout=kwargs.pop("timeout", 15.0),
|
url,
|
||||||
) as client:
|
timeout=timeout,
|
||||||
with client.stream(method, url, **kwargs) as response:
|
**kwargs,
|
||||||
yield response
|
) as response:
|
||||||
|
yield response
|
||||||
|
|
||||||
|
|
||||||
def request_connector_bytes(
|
def request_connector_bytes(
|
||||||
|
|||||||
@@ -0,0 +1,550 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
from dataclasses import dataclass
|
||||||
|
|
||||||
|
from sqlalchemy.orm import Session, object_session
|
||||||
|
|
||||||
|
from govoplan_files.backend.db.models import (
|
||||||
|
FileBlob,
|
||||||
|
FileIntegrityFinding,
|
||||||
|
FileIntegrityScan,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.storage.backends import (
|
||||||
|
StorageBackend,
|
||||||
|
StorageBackendError,
|
||||||
|
StorageObjectMissing,
|
||||||
|
get_storage_backend,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.storage.recovery import (
|
||||||
|
begin_orphan_cleanup_recovery,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.storage.common import FileStorageError, utcnow
|
||||||
|
|
||||||
|
|
||||||
|
TERMINAL_SCAN_STATUSES = {"completed", "cancelled"}
|
||||||
|
QUARANTINED_BLOB_STATUSES = {
|
||||||
|
"missing",
|
||||||
|
"size_mismatch",
|
||||||
|
"checksum_mismatch",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class BlobInspection:
|
||||||
|
valid: bool
|
||||||
|
kind: str
|
||||||
|
observed_size_bytes: int | None = None
|
||||||
|
observed_checksum_sha256: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class IntegrityActionResult:
|
||||||
|
action: str
|
||||||
|
changed: bool
|
||||||
|
dry_run: bool
|
||||||
|
finding: FileIntegrityFinding
|
||||||
|
inspection: BlobInspection | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def storage_prefix_for_tenant(tenant_id: str) -> str:
|
||||||
|
return f"tenants/{tenant_id}/files/"
|
||||||
|
|
||||||
|
|
||||||
|
def create_integrity_scan(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
user_id: str,
|
||||||
|
verify_checksums: bool = True,
|
||||||
|
batch_size: int = 100,
|
||||||
|
backend: StorageBackend | None = None,
|
||||||
|
) -> FileIntegrityScan:
|
||||||
|
active_backend = backend or get_storage_backend()
|
||||||
|
scan = FileIntegrityScan(
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
storage_backend=active_backend.name,
|
||||||
|
storage_prefix=storage_prefix_for_tenant(tenant_id),
|
||||||
|
verify_checksums=verify_checksums,
|
||||||
|
batch_size=max(1, min(int(batch_size), 1000)),
|
||||||
|
created_by_user_id=user_id,
|
||||||
|
)
|
||||||
|
session.add(scan)
|
||||||
|
session.flush()
|
||||||
|
return scan
|
||||||
|
|
||||||
|
|
||||||
|
def run_integrity_scan_batch(
|
||||||
|
session: Session,
|
||||||
|
scan: FileIntegrityScan,
|
||||||
|
*,
|
||||||
|
backend: StorageBackend | None = None,
|
||||||
|
) -> FileIntegrityScan:
|
||||||
|
if scan.status in TERMINAL_SCAN_STATUSES:
|
||||||
|
return scan
|
||||||
|
active_backend = backend or get_storage_backend()
|
||||||
|
if active_backend.name != scan.storage_backend:
|
||||||
|
raise FileStorageError(
|
||||||
|
"The configured storage backend changed after this integrity scan started"
|
||||||
|
)
|
||||||
|
if scan.started_at is None:
|
||||||
|
scan.started_at = utcnow()
|
||||||
|
scan.status = "running"
|
||||||
|
scan.last_error = None
|
||||||
|
if scan.phase == "blobs":
|
||||||
|
_scan_blob_batch(session, scan, active_backend)
|
||||||
|
elif scan.phase == "objects":
|
||||||
|
_scan_object_batch(session, scan, active_backend)
|
||||||
|
else:
|
||||||
|
scan.phase = "completed"
|
||||||
|
scan.status = "completed"
|
||||||
|
scan.completed_at = utcnow()
|
||||||
|
scan.revision += 1
|
||||||
|
session.add(scan)
|
||||||
|
return scan
|
||||||
|
|
||||||
|
|
||||||
|
def mark_integrity_scan_failed(
|
||||||
|
scan: FileIntegrityScan,
|
||||||
|
*,
|
||||||
|
error: Exception,
|
||||||
|
) -> None:
|
||||||
|
scan.status = "failed"
|
||||||
|
scan.last_error = type(error).__name__[:255]
|
||||||
|
scan.revision += 1
|
||||||
|
|
||||||
|
|
||||||
|
def inspect_blob(
|
||||||
|
blob: FileBlob,
|
||||||
|
*,
|
||||||
|
backend: StorageBackend,
|
||||||
|
verify_checksum: bool = True,
|
||||||
|
) -> BlobInspection:
|
||||||
|
expected_size = blob.storage_size_bytes if blob.storage_size_bytes is not None else blob.size_bytes
|
||||||
|
expected_checksum = blob.storage_checksum_sha256 if blob.storage_checksum_sha256 is not None else blob.checksum_sha256
|
||||||
|
try:
|
||||||
|
info = backend.stat(blob.storage_key)
|
||||||
|
except StorageObjectMissing:
|
||||||
|
return BlobInspection(valid=False, kind="missing")
|
||||||
|
if info.size_bytes != expected_size:
|
||||||
|
return BlobInspection(
|
||||||
|
valid=False,
|
||||||
|
kind="size_mismatch",
|
||||||
|
observed_size_bytes=info.size_bytes,
|
||||||
|
)
|
||||||
|
if not verify_checksum:
|
||||||
|
return BlobInspection(
|
||||||
|
valid=True,
|
||||||
|
kind="verified",
|
||||||
|
observed_size_bytes=info.size_bytes,
|
||||||
|
)
|
||||||
|
digest = hashlib.sha256()
|
||||||
|
observed_size = 0
|
||||||
|
for chunk in backend.iter_bytes(blob.storage_key):
|
||||||
|
observed_size += len(chunk)
|
||||||
|
digest.update(chunk)
|
||||||
|
observed_checksum = digest.hexdigest()
|
||||||
|
if observed_size != expected_size:
|
||||||
|
return BlobInspection(
|
||||||
|
valid=False,
|
||||||
|
kind="size_mismatch",
|
||||||
|
observed_size_bytes=observed_size,
|
||||||
|
observed_checksum_sha256=observed_checksum,
|
||||||
|
)
|
||||||
|
if observed_checksum != expected_checksum:
|
||||||
|
return BlobInspection(
|
||||||
|
valid=False,
|
||||||
|
kind="checksum_mismatch",
|
||||||
|
observed_size_bytes=observed_size,
|
||||||
|
observed_checksum_sha256=observed_checksum,
|
||||||
|
)
|
||||||
|
return BlobInspection(
|
||||||
|
valid=True,
|
||||||
|
kind="verified",
|
||||||
|
observed_size_bytes=observed_size,
|
||||||
|
observed_checksum_sha256=observed_checksum,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def apply_blob_inspection(
|
||||||
|
session: Session,
|
||||||
|
blob: FileBlob,
|
||||||
|
inspection: BlobInspection,
|
||||||
|
*,
|
||||||
|
checked_at=None,
|
||||||
|
) -> None:
|
||||||
|
timestamp = checked_at or utcnow()
|
||||||
|
blob.integrity_checked_at = timestamp
|
||||||
|
if inspection.valid:
|
||||||
|
blob.integrity_status = "verified"
|
||||||
|
blob.integrity_failure = None
|
||||||
|
blob.quarantined_at = None
|
||||||
|
else:
|
||||||
|
blob.integrity_status = inspection.kind
|
||||||
|
blob.integrity_failure = inspection.kind
|
||||||
|
blob.quarantined_at = blob.quarantined_at or timestamp
|
||||||
|
session.add(blob)
|
||||||
|
|
||||||
|
|
||||||
|
def ensure_blob_is_readable(blob: FileBlob) -> None:
|
||||||
|
if blob.integrity_status in QUARANTINED_BLOB_STATUSES or blob.quarantined_at:
|
||||||
|
raise FileStorageError(
|
||||||
|
"Managed file content is quarantined because integrity verification failed"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def read_verified_blob_bytes(
|
||||||
|
blob: FileBlob,
|
||||||
|
*,
|
||||||
|
backend: StorageBackend,
|
||||||
|
) -> bytes:
|
||||||
|
ensure_blob_is_readable(blob)
|
||||||
|
try:
|
||||||
|
data = backend.get_bytes(blob.storage_key)
|
||||||
|
except StorageBackendError as exc:
|
||||||
|
raise FileStorageError("Managed file content is not available") from exc
|
||||||
|
expected_storage_size = blob.storage_size_bytes if blob.storage_size_bytes is not None else blob.size_bytes
|
||||||
|
expected_storage_checksum = blob.storage_checksum_sha256 if blob.storage_checksum_sha256 is not None else blob.checksum_sha256
|
||||||
|
if len(data) != expected_storage_size:
|
||||||
|
raise FileStorageError(
|
||||||
|
"Managed file content failed its recorded size verification"
|
||||||
|
)
|
||||||
|
if hashlib.sha256(data).hexdigest() != expected_storage_checksum:
|
||||||
|
raise FileStorageError(
|
||||||
|
"Managed file content failed its recorded checksum verification"
|
||||||
|
)
|
||||||
|
if blob.encryption_envelope_id:
|
||||||
|
session = object_session(blob)
|
||||||
|
if session is None:
|
||||||
|
raise FileStorageError("Encrypted managed content requires an attached database session")
|
||||||
|
from govoplan_files.backend.storage.content_protection import unprotect_blob_content
|
||||||
|
|
||||||
|
data = unprotect_blob_content(
|
||||||
|
session,
|
||||||
|
tenant_id=blob.tenant_id,
|
||||||
|
blob_id=blob.id,
|
||||||
|
envelope_id=blob.encryption_envelope_id,
|
||||||
|
ciphertext=data,
|
||||||
|
)
|
||||||
|
if len(data) != blob.size_bytes:
|
||||||
|
raise FileStorageError("Decrypted managed file content failed its recorded size verification")
|
||||||
|
if hashlib.sha256(data).hexdigest() != blob.checksum_sha256:
|
||||||
|
raise FileStorageError("Decrypted managed file content failed its recorded checksum verification")
|
||||||
|
return data
|
||||||
|
|
||||||
|
|
||||||
|
def recheck_integrity_finding(
|
||||||
|
session: Session,
|
||||||
|
finding: FileIntegrityFinding,
|
||||||
|
*,
|
||||||
|
user_id: str,
|
||||||
|
dry_run: bool,
|
||||||
|
backend: StorageBackend | None = None,
|
||||||
|
) -> IntegrityActionResult:
|
||||||
|
if finding.kind == "orphan_object" or not finding.blob_id:
|
||||||
|
raise FileStorageError("Only blob integrity findings can be rechecked")
|
||||||
|
blob = session.get(FileBlob, finding.blob_id)
|
||||||
|
if blob is None or blob.tenant_id != finding.tenant_id:
|
||||||
|
raise FileStorageError("Integrity finding blob no longer exists")
|
||||||
|
active_backend = backend or get_storage_backend()
|
||||||
|
scan = session.get(FileIntegrityScan, finding.scan_id)
|
||||||
|
if scan is None or scan.storage_backend != active_backend.name:
|
||||||
|
raise FileStorageError(
|
||||||
|
"The configured storage backend does not match the integrity finding"
|
||||||
|
)
|
||||||
|
inspection = inspect_blob(
|
||||||
|
blob,
|
||||||
|
backend=active_backend,
|
||||||
|
verify_checksum=True,
|
||||||
|
)
|
||||||
|
changed = False
|
||||||
|
if not dry_run:
|
||||||
|
previous = (
|
||||||
|
blob.integrity_status,
|
||||||
|
blob.quarantined_at,
|
||||||
|
finding.state,
|
||||||
|
)
|
||||||
|
apply_blob_inspection(session, blob, inspection)
|
||||||
|
_update_finding_from_inspection(finding, inspection)
|
||||||
|
if inspection.valid:
|
||||||
|
finding.state = "resolved"
|
||||||
|
finding.resolved_at = utcnow()
|
||||||
|
finding.resolved_by_user_id = user_id
|
||||||
|
session.add(finding)
|
||||||
|
finding.revision += 1
|
||||||
|
changed = previous != (
|
||||||
|
blob.integrity_status,
|
||||||
|
blob.quarantined_at,
|
||||||
|
finding.state,
|
||||||
|
)
|
||||||
|
return IntegrityActionResult(
|
||||||
|
action="recheck",
|
||||||
|
changed=changed,
|
||||||
|
dry_run=dry_run,
|
||||||
|
finding=finding,
|
||||||
|
inspection=inspection,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def cleanup_orphan_finding(
|
||||||
|
session: Session,
|
||||||
|
finding: FileIntegrityFinding,
|
||||||
|
*,
|
||||||
|
user_id: str,
|
||||||
|
dry_run: bool,
|
||||||
|
backend: StorageBackend | None = None,
|
||||||
|
) -> IntegrityActionResult:
|
||||||
|
if finding.kind != "orphan_object":
|
||||||
|
raise FileStorageError("Only orphan-object findings can be cleaned up")
|
||||||
|
scan = session.get(FileIntegrityScan, finding.scan_id)
|
||||||
|
if scan is None or scan.tenant_id != finding.tenant_id:
|
||||||
|
raise FileStorageError("Integrity scan not found")
|
||||||
|
if not finding.storage_key.startswith(scan.storage_prefix):
|
||||||
|
raise FileStorageError("Orphan cleanup is outside the scan storage scope")
|
||||||
|
referenced = (
|
||||||
|
session.query(FileBlob.id)
|
||||||
|
.filter(
|
||||||
|
FileBlob.tenant_id == finding.tenant_id,
|
||||||
|
FileBlob.storage_backend == scan.storage_backend,
|
||||||
|
FileBlob.storage_key == finding.storage_key,
|
||||||
|
)
|
||||||
|
.first()
|
||||||
|
)
|
||||||
|
if referenced:
|
||||||
|
if not dry_run and finding.state != "resolved":
|
||||||
|
finding.state = "resolved"
|
||||||
|
finding.resolved_at = utcnow()
|
||||||
|
finding.resolved_by_user_id = user_id
|
||||||
|
finding.revision += 1
|
||||||
|
session.add(finding)
|
||||||
|
return IntegrityActionResult(
|
||||||
|
action="retained_referenced",
|
||||||
|
changed=True,
|
||||||
|
dry_run=False,
|
||||||
|
finding=finding,
|
||||||
|
)
|
||||||
|
return IntegrityActionResult(
|
||||||
|
action="retained_referenced",
|
||||||
|
changed=False,
|
||||||
|
dry_run=dry_run,
|
||||||
|
finding=finding,
|
||||||
|
)
|
||||||
|
if finding.state == "deleted":
|
||||||
|
return IntegrityActionResult(
|
||||||
|
action="already_deleted",
|
||||||
|
changed=False,
|
||||||
|
dry_run=dry_run,
|
||||||
|
finding=finding,
|
||||||
|
)
|
||||||
|
if dry_run:
|
||||||
|
return IntegrityActionResult(
|
||||||
|
action="would_delete",
|
||||||
|
changed=False,
|
||||||
|
dry_run=True,
|
||||||
|
finding=finding,
|
||||||
|
)
|
||||||
|
active_backend = backend or get_storage_backend()
|
||||||
|
if active_backend.name != scan.storage_backend:
|
||||||
|
raise FileStorageError(
|
||||||
|
"The configured storage backend does not match the integrity finding"
|
||||||
|
)
|
||||||
|
begin_orphan_cleanup_recovery(
|
||||||
|
session,
|
||||||
|
finding,
|
||||||
|
backend=active_backend,
|
||||||
|
user_id=user_id,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
active_backend.stat(finding.storage_key)
|
||||||
|
except StorageObjectMissing:
|
||||||
|
action = "already_absent"
|
||||||
|
else:
|
||||||
|
active_backend.delete(finding.storage_key)
|
||||||
|
action = "deleted"
|
||||||
|
finding.state = "deleted"
|
||||||
|
finding.resolved_at = utcnow()
|
||||||
|
finding.resolved_by_user_id = user_id
|
||||||
|
finding.revision += 1
|
||||||
|
session.add(finding)
|
||||||
|
return IntegrityActionResult(
|
||||||
|
action=action,
|
||||||
|
changed=True,
|
||||||
|
dry_run=False,
|
||||||
|
finding=finding,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _scan_blob_batch(
|
||||||
|
session: Session,
|
||||||
|
scan: FileIntegrityScan,
|
||||||
|
backend: StorageBackend,
|
||||||
|
) -> None:
|
||||||
|
query = session.query(FileBlob).filter(
|
||||||
|
FileBlob.tenant_id == scan.tenant_id,
|
||||||
|
FileBlob.storage_backend == scan.storage_backend,
|
||||||
|
)
|
||||||
|
if scan.blob_cursor:
|
||||||
|
query = query.filter(FileBlob.id > scan.blob_cursor)
|
||||||
|
blobs = query.order_by(FileBlob.id.asc()).limit(scan.batch_size).all()
|
||||||
|
for blob in blobs:
|
||||||
|
inspection = inspect_blob(
|
||||||
|
blob,
|
||||||
|
backend=backend,
|
||||||
|
verify_checksum=scan.verify_checksums,
|
||||||
|
)
|
||||||
|
apply_blob_inspection(session, blob, inspection)
|
||||||
|
scan.scanned_blob_count += 1
|
||||||
|
if inspection.valid:
|
||||||
|
scan.verified_blob_count += 1
|
||||||
|
_resolve_scan_blob_findings(session, scan, blob)
|
||||||
|
else:
|
||||||
|
scan.quarantined_blob_count += 1
|
||||||
|
_record_blob_finding(session, scan, blob, inspection)
|
||||||
|
scan.blob_cursor = blob.id
|
||||||
|
if len(blobs) < scan.batch_size:
|
||||||
|
scan.phase = "objects"
|
||||||
|
scan.object_cursor = None
|
||||||
|
|
||||||
|
|
||||||
|
def _scan_object_batch(
|
||||||
|
session: Session,
|
||||||
|
scan: FileIntegrityScan,
|
||||||
|
backend: StorageBackend,
|
||||||
|
) -> None:
|
||||||
|
page = backend.list_objects(
|
||||||
|
prefix=scan.storage_prefix,
|
||||||
|
after=scan.object_cursor,
|
||||||
|
limit=scan.batch_size,
|
||||||
|
)
|
||||||
|
keys = [item.key for item in page.objects]
|
||||||
|
referenced_keys = {
|
||||||
|
row[0]
|
||||||
|
for row in session.query(FileBlob.storage_key)
|
||||||
|
.filter(
|
||||||
|
FileBlob.tenant_id == scan.tenant_id,
|
||||||
|
FileBlob.storage_backend == scan.storage_backend,
|
||||||
|
FileBlob.storage_key.in_(keys),
|
||||||
|
)
|
||||||
|
.all()
|
||||||
|
} if keys else set()
|
||||||
|
for item in page.objects:
|
||||||
|
scan.scanned_object_count += 1
|
||||||
|
if item.key in referenced_keys:
|
||||||
|
continue
|
||||||
|
scan.orphan_object_count += 1
|
||||||
|
_record_orphan_finding(
|
||||||
|
session,
|
||||||
|
scan,
|
||||||
|
storage_key=item.key,
|
||||||
|
size_bytes=item.size_bytes,
|
||||||
|
)
|
||||||
|
scan.object_cursor = page.next_cursor
|
||||||
|
if page.next_cursor is None:
|
||||||
|
scan.phase = "completed"
|
||||||
|
scan.status = "completed"
|
||||||
|
scan.completed_at = utcnow()
|
||||||
|
|
||||||
|
|
||||||
|
def _record_blob_finding(
|
||||||
|
session: Session,
|
||||||
|
scan: FileIntegrityScan,
|
||||||
|
blob: FileBlob,
|
||||||
|
inspection: BlobInspection,
|
||||||
|
) -> FileIntegrityFinding:
|
||||||
|
finding = (
|
||||||
|
session.query(FileIntegrityFinding)
|
||||||
|
.filter(
|
||||||
|
FileIntegrityFinding.scan_id == scan.id,
|
||||||
|
FileIntegrityFinding.blob_id == blob.id,
|
||||||
|
FileIntegrityFinding.kind == inspection.kind,
|
||||||
|
)
|
||||||
|
.one_or_none()
|
||||||
|
)
|
||||||
|
if finding is None:
|
||||||
|
finding = FileIntegrityFinding(
|
||||||
|
scan_id=scan.id,
|
||||||
|
tenant_id=scan.tenant_id,
|
||||||
|
kind=inspection.kind,
|
||||||
|
blob_id=blob.id,
|
||||||
|
storage_key=blob.storage_key,
|
||||||
|
expected_size_bytes=blob.storage_size_bytes if blob.storage_size_bytes is not None else blob.size_bytes,
|
||||||
|
expected_checksum_sha256=blob.storage_checksum_sha256 if blob.storage_checksum_sha256 is not None else blob.checksum_sha256,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
finding.revision += 1
|
||||||
|
_update_finding_from_inspection(finding, inspection)
|
||||||
|
session.add(finding)
|
||||||
|
return finding
|
||||||
|
|
||||||
|
|
||||||
|
def _record_orphan_finding(
|
||||||
|
session: Session,
|
||||||
|
scan: FileIntegrityScan,
|
||||||
|
*,
|
||||||
|
storage_key: str,
|
||||||
|
size_bytes: int,
|
||||||
|
) -> FileIntegrityFinding:
|
||||||
|
finding = (
|
||||||
|
session.query(FileIntegrityFinding)
|
||||||
|
.filter(
|
||||||
|
FileIntegrityFinding.scan_id == scan.id,
|
||||||
|
FileIntegrityFinding.kind == "orphan_object",
|
||||||
|
FileIntegrityFinding.storage_key == storage_key,
|
||||||
|
)
|
||||||
|
.one_or_none()
|
||||||
|
)
|
||||||
|
if finding is None:
|
||||||
|
finding = FileIntegrityFinding(
|
||||||
|
scan_id=scan.id,
|
||||||
|
tenant_id=scan.tenant_id,
|
||||||
|
kind="orphan_object",
|
||||||
|
storage_key=storage_key,
|
||||||
|
observed_size_bytes=size_bytes,
|
||||||
|
)
|
||||||
|
session.add(finding)
|
||||||
|
return finding
|
||||||
|
|
||||||
|
|
||||||
|
def _resolve_scan_blob_findings(
|
||||||
|
session: Session,
|
||||||
|
scan: FileIntegrityScan,
|
||||||
|
blob: FileBlob,
|
||||||
|
) -> None:
|
||||||
|
for finding in (
|
||||||
|
session.query(FileIntegrityFinding)
|
||||||
|
.filter(
|
||||||
|
FileIntegrityFinding.scan_id == scan.id,
|
||||||
|
FileIntegrityFinding.blob_id == blob.id,
|
||||||
|
FileIntegrityFinding.state == "open",
|
||||||
|
)
|
||||||
|
.all()
|
||||||
|
):
|
||||||
|
finding.state = "resolved"
|
||||||
|
finding.resolved_at = utcnow()
|
||||||
|
finding.revision += 1
|
||||||
|
session.add(finding)
|
||||||
|
|
||||||
|
|
||||||
|
def _update_finding_from_inspection(
|
||||||
|
finding: FileIntegrityFinding,
|
||||||
|
inspection: BlobInspection,
|
||||||
|
) -> None:
|
||||||
|
finding.observed_size_bytes = inspection.observed_size_bytes
|
||||||
|
finding.observed_checksum_sha256 = inspection.observed_checksum_sha256
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"BlobInspection",
|
||||||
|
"IntegrityActionResult",
|
||||||
|
"QUARANTINED_BLOB_STATUSES",
|
||||||
|
"apply_blob_inspection",
|
||||||
|
"cleanup_orphan_finding",
|
||||||
|
"create_integrity_scan",
|
||||||
|
"ensure_blob_is_readable",
|
||||||
|
"inspect_blob",
|
||||||
|
"mark_integrity_scan_failed",
|
||||||
|
"read_verified_blob_bytes",
|
||||||
|
"recheck_integrity_finding",
|
||||||
|
"run_integrity_scan_batch",
|
||||||
|
"storage_prefix_for_tenant",
|
||||||
|
]
|
||||||
@@ -0,0 +1,796 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from typing import Callable, Iterable
|
||||||
|
|
||||||
|
from sqlalchemy import func, inspect, or_
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from govoplan_core.core.recovery import (
|
||||||
|
RecoveryGuaranteeError,
|
||||||
|
RecoveryMode,
|
||||||
|
RecoveryPlan,
|
||||||
|
RecoveryStatus,
|
||||||
|
)
|
||||||
|
from govoplan_core.core.recovery_runtime import (
|
||||||
|
RecoveryOperationBusy,
|
||||||
|
RecoveryOperationStateConflict,
|
||||||
|
begin_durable_recovery_operation,
|
||||||
|
)
|
||||||
|
from govoplan_core.core.runtime_coordination import process_runtime_identity
|
||||||
|
from govoplan_core.db.session import get_database
|
||||||
|
from govoplan_files.backend.db.models import (
|
||||||
|
CampaignAttachmentUse,
|
||||||
|
FileAsset,
|
||||||
|
FileBlob,
|
||||||
|
FileConnectorSpace,
|
||||||
|
FileFolder,
|
||||||
|
FileFormEvidenceGrant,
|
||||||
|
FileShare,
|
||||||
|
FileVersion,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.storage.access import ensure_owner_access
|
||||||
|
from govoplan_files.backend.storage.backends import (
|
||||||
|
StorageBackendError,
|
||||||
|
get_storage_backend,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.storage.common import FileStorageError, utcnow
|
||||||
|
from govoplan_files.backend.storage.connector_spaces import connector_space_owner_id
|
||||||
|
from govoplan_files.backend.storage.paths import normalize_folder
|
||||||
|
from govoplan_files.backend.storage.share_state import effective_file_share_clause
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class PurgePreviewItem:
|
||||||
|
file_id: str
|
||||||
|
filename: str
|
||||||
|
lifecycle_revision: int
|
||||||
|
deleted_at: datetime | None
|
||||||
|
retained_until: datetime | None
|
||||||
|
legal_hold: bool
|
||||||
|
blockers: tuple[str, ...]
|
||||||
|
blob_ids: tuple[str, ...]
|
||||||
|
|
||||||
|
def digest_payload(self) -> dict[str, object]:
|
||||||
|
return {
|
||||||
|
"file_id": self.file_id,
|
||||||
|
"filename": self.filename,
|
||||||
|
"lifecycle_revision": self.lifecycle_revision,
|
||||||
|
"deleted_at": _iso(self.deleted_at),
|
||||||
|
"retained_until": _iso(self.retained_until),
|
||||||
|
"legal_hold": self.legal_hold,
|
||||||
|
"blockers": list(self.blockers),
|
||||||
|
"blob_ids": list(self.blob_ids),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class PurgePreview:
|
||||||
|
preview_sha256: str
|
||||||
|
items: tuple[PurgePreviewItem, ...]
|
||||||
|
|
||||||
|
@property
|
||||||
|
def eligible(self) -> bool:
|
||||||
|
return all(not item.blockers for item in self.items)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class PurgeResult:
|
||||||
|
recovery_operation_id: str
|
||||||
|
status: str
|
||||||
|
replayed: bool
|
||||||
|
purged_files: int
|
||||||
|
released_blobs: int
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class BlobGcResult:
|
||||||
|
inspected_blobs: int
|
||||||
|
deleted_blobs: int
|
||||||
|
unresolved_operation_ids: tuple[str, ...]
|
||||||
|
|
||||||
|
|
||||||
|
def set_asset_lifecycle(
|
||||||
|
session: Session,
|
||||||
|
asset: FileAsset,
|
||||||
|
*,
|
||||||
|
retained_until: datetime | None,
|
||||||
|
legal_hold: bool,
|
||||||
|
reason: str,
|
||||||
|
expected_revision: int,
|
||||||
|
) -> FileAsset:
|
||||||
|
if asset.lifecycle_revision != expected_revision:
|
||||||
|
raise FileStorageError("File lifecycle settings changed; reload before saving")
|
||||||
|
asset.retained_until = retained_until
|
||||||
|
asset.legal_hold = bool(legal_hold)
|
||||||
|
asset.lifecycle_reason = reason.strip()
|
||||||
|
asset.lifecycle_revision += 1
|
||||||
|
session.add(asset)
|
||||||
|
session.flush()
|
||||||
|
return asset
|
||||||
|
|
||||||
|
|
||||||
|
def get_asset_for_lifecycle(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
user_id: str,
|
||||||
|
asset_id: str,
|
||||||
|
is_admin: bool,
|
||||||
|
) -> FileAsset:
|
||||||
|
asset = session.get(FileAsset, asset_id)
|
||||||
|
if asset is None or asset.tenant_id != tenant_id:
|
||||||
|
raise FileStorageError("File not found")
|
||||||
|
ensure_owner_access(
|
||||||
|
session,
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
owner_type=asset.owner_type,
|
||||||
|
owner_id=_asset_owner_id(asset),
|
||||||
|
user_id=user_id,
|
||||||
|
is_admin=is_admin,
|
||||||
|
)
|
||||||
|
return asset
|
||||||
|
|
||||||
|
|
||||||
|
def restore_asset(session: Session, asset: FileAsset) -> bool:
|
||||||
|
if asset.deleted_at is None:
|
||||||
|
return False
|
||||||
|
collision = _asset_owner_query(session, asset).filter(
|
||||||
|
FileAsset.id != asset.id,
|
||||||
|
FileAsset.display_path == asset.display_path,
|
||||||
|
FileAsset.deleted_at.is_(None),
|
||||||
|
).first()
|
||||||
|
if collision is not None:
|
||||||
|
raise FileStorageError(
|
||||||
|
"The file cannot be restored because its path is already in use"
|
||||||
|
)
|
||||||
|
asset.deleted_at = None
|
||||||
|
asset.lifecycle_revision += 1
|
||||||
|
session.add(asset)
|
||||||
|
session.flush()
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def restore_folder(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
owner_type: str,
|
||||||
|
owner_id: str,
|
||||||
|
user_id: str,
|
||||||
|
path: str,
|
||||||
|
recursive: bool,
|
||||||
|
is_admin: bool,
|
||||||
|
) -> tuple[int, int]:
|
||||||
|
ensure_owner_access(
|
||||||
|
session,
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
owner_type=owner_type,
|
||||||
|
owner_id=owner_id,
|
||||||
|
user_id=user_id,
|
||||||
|
is_admin=is_admin,
|
||||||
|
)
|
||||||
|
normalized = normalize_folder(path)
|
||||||
|
if not normalized:
|
||||||
|
raise FileStorageError("Folder path is required")
|
||||||
|
prefix = f"{normalized}/"
|
||||||
|
folders = _folder_owner_query(
|
||||||
|
session, tenant_id=tenant_id, owner_type=owner_type, owner_id=owner_id
|
||||||
|
).filter(FileFolder.deleted_at.is_not(None))
|
||||||
|
assets = _asset_owner_query_by_values(
|
||||||
|
session, tenant_id=tenant_id, owner_type=owner_type, owner_id=owner_id
|
||||||
|
).filter(FileAsset.deleted_at.is_not(None))
|
||||||
|
if recursive:
|
||||||
|
folders = folders.filter(
|
||||||
|
(FileFolder.path == normalized) | FileFolder.path.like(f"{prefix}%")
|
||||||
|
)
|
||||||
|
assets = assets.filter(FileAsset.display_path.like(f"{prefix}%"))
|
||||||
|
else:
|
||||||
|
folders = folders.filter(FileFolder.path == normalized)
|
||||||
|
assets = assets.filter(False)
|
||||||
|
folder_rows = folders.order_by(FileFolder.path.asc()).all()
|
||||||
|
asset_rows = assets.order_by(FileAsset.display_path.asc()).all()
|
||||||
|
if not folder_rows:
|
||||||
|
raise FileStorageError("Deleted folder not found")
|
||||||
|
for folder in folder_rows:
|
||||||
|
collision = _folder_owner_query(
|
||||||
|
session,
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
owner_type=owner_type,
|
||||||
|
owner_id=owner_id,
|
||||||
|
).filter(
|
||||||
|
FileFolder.id != folder.id,
|
||||||
|
FileFolder.path == folder.path,
|
||||||
|
FileFolder.deleted_at.is_(None),
|
||||||
|
).first()
|
||||||
|
if collision is not None:
|
||||||
|
raise FileStorageError(
|
||||||
|
f"The folder cannot be restored because its path is in use: {folder.path}"
|
||||||
|
)
|
||||||
|
for asset in asset_rows:
|
||||||
|
collision = _asset_owner_query(session, asset).filter(
|
||||||
|
FileAsset.id != asset.id,
|
||||||
|
FileAsset.display_path == asset.display_path,
|
||||||
|
FileAsset.deleted_at.is_(None),
|
||||||
|
).first()
|
||||||
|
if collision is not None:
|
||||||
|
raise FileStorageError(
|
||||||
|
f"A file cannot be restored because its path is in use: {asset.display_path}"
|
||||||
|
)
|
||||||
|
for folder in folder_rows:
|
||||||
|
folder.deleted_at = None
|
||||||
|
session.add(folder)
|
||||||
|
for asset in asset_rows:
|
||||||
|
asset.deleted_at = None
|
||||||
|
asset.lifecycle_revision += 1
|
||||||
|
session.add(asset)
|
||||||
|
session.flush()
|
||||||
|
return len(folder_rows), len(asset_rows)
|
||||||
|
|
||||||
|
|
||||||
|
def restore_connector_space(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
user_id: str,
|
||||||
|
space_id: str,
|
||||||
|
is_admin: bool,
|
||||||
|
) -> FileConnectorSpace:
|
||||||
|
space = session.get(FileConnectorSpace, space_id)
|
||||||
|
if space is None or space.tenant_id != tenant_id or space.deleted_at is None:
|
||||||
|
raise FileStorageError("Deleted connector space not found")
|
||||||
|
ensure_owner_access(
|
||||||
|
session,
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
owner_type=space.owner_type,
|
||||||
|
owner_id=connector_space_owner_id(space),
|
||||||
|
user_id=user_id,
|
||||||
|
is_admin=is_admin,
|
||||||
|
)
|
||||||
|
collision = _connector_space_owner_query(session, space).filter(
|
||||||
|
FileConnectorSpace.id != space.id,
|
||||||
|
FileConnectorSpace.label == space.label,
|
||||||
|
FileConnectorSpace.deleted_at.is_(None),
|
||||||
|
).first()
|
||||||
|
if collision is not None:
|
||||||
|
raise FileStorageError(
|
||||||
|
"The connector space cannot be restored because its label is in use"
|
||||||
|
)
|
||||||
|
space.deleted_at = None
|
||||||
|
space.is_active = True
|
||||||
|
session.add(space)
|
||||||
|
session.flush()
|
||||||
|
return space
|
||||||
|
|
||||||
|
|
||||||
|
def preview_asset_purge(
|
||||||
|
session: Session, *, tenant_id: str, file_ids: Iterable[str]
|
||||||
|
) -> PurgePreview:
|
||||||
|
normalized_ids = sorted(set(str(value).strip() for value in file_ids if str(value).strip()))
|
||||||
|
if not normalized_ids or len(normalized_ids) > 100:
|
||||||
|
raise FileStorageError("Purge preview must contain between 1 and 100 files")
|
||||||
|
assets = (
|
||||||
|
session.query(FileAsset)
|
||||||
|
.filter(FileAsset.tenant_id == tenant_id, FileAsset.id.in_(normalized_ids))
|
||||||
|
.order_by(FileAsset.id.asc())
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
by_id = {asset.id: asset for asset in assets}
|
||||||
|
items: list[PurgePreviewItem] = []
|
||||||
|
for file_id in normalized_ids:
|
||||||
|
asset = by_id.get(file_id)
|
||||||
|
if asset is None:
|
||||||
|
items.append(
|
||||||
|
PurgePreviewItem(
|
||||||
|
file_id=file_id,
|
||||||
|
filename="",
|
||||||
|
lifecycle_revision=0,
|
||||||
|
deleted_at=None,
|
||||||
|
retained_until=None,
|
||||||
|
legal_hold=False,
|
||||||
|
blockers=("not_found",),
|
||||||
|
blob_ids=(),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
blockers = _asset_purge_blockers(session, asset)
|
||||||
|
blob_ids = tuple(
|
||||||
|
sorted(
|
||||||
|
row[0]
|
||||||
|
for row in session.query(FileVersion.blob_id)
|
||||||
|
.filter(FileVersion.file_asset_id == asset.id)
|
||||||
|
.distinct()
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
)
|
||||||
|
items.append(
|
||||||
|
PurgePreviewItem(
|
||||||
|
file_id=asset.id,
|
||||||
|
filename=asset.filename,
|
||||||
|
lifecycle_revision=asset.lifecycle_revision,
|
||||||
|
deleted_at=asset.deleted_at,
|
||||||
|
retained_until=asset.retained_until,
|
||||||
|
legal_hold=asset.legal_hold,
|
||||||
|
blockers=tuple(blockers),
|
||||||
|
blob_ids=blob_ids,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
payload = [item.digest_payload() for item in items]
|
||||||
|
digest = hashlib.sha256(
|
||||||
|
json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||||
|
).hexdigest()
|
||||||
|
return PurgePreview(preview_sha256=digest, items=tuple(items))
|
||||||
|
|
||||||
|
|
||||||
|
def execute_asset_purge(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
file_ids: Iterable[str],
|
||||||
|
preview_sha256: str,
|
||||||
|
idempotency_key: str,
|
||||||
|
approval_reference: str,
|
||||||
|
before_commit: Callable[[list[FileAsset]], None] | None = None,
|
||||||
|
) -> PurgeResult:
|
||||||
|
normalized_ids = sorted(set(str(value).strip() for value in file_ids if str(value).strip()))
|
||||||
|
if not normalized_ids or len(normalized_ids) > 100:
|
||||||
|
raise FileStorageError("Purge must contain between 1 and 100 files")
|
||||||
|
request = {
|
||||||
|
"tenant_id": tenant_id,
|
||||||
|
"file_ids": normalized_ids,
|
||||||
|
"preview_sha256": preview_sha256,
|
||||||
|
"approval_reference": approval_reference,
|
||||||
|
}
|
||||||
|
try:
|
||||||
|
started = begin_durable_recovery_operation(
|
||||||
|
get_database().SessionLocal,
|
||||||
|
identity=process_runtime_identity(),
|
||||||
|
module_id="files",
|
||||||
|
operation_type="asset-hard-purge",
|
||||||
|
idempotency_key=f"files-asset-purge:{idempotency_key}",
|
||||||
|
request=request,
|
||||||
|
recovery_plan=RecoveryPlan(
|
||||||
|
mode=RecoveryMode.IRREVERSIBLE,
|
||||||
|
preconditions=(
|
||||||
|
"the actor holds the dedicated Files purge permission",
|
||||||
|
"the signed-off preview still matches current lifecycle state",
|
||||||
|
"every target is soft-deleted and free of retention or legal-hold blockers",
|
||||||
|
),
|
||||||
|
verification_steps=(
|
||||||
|
"verify every target FileAsset and FileVersion row is absent",
|
||||||
|
"recalculate retained FileBlob reference counts before later garbage collection",
|
||||||
|
),
|
||||||
|
approval_reference=approval_reference,
|
||||||
|
),
|
||||||
|
precondition_evidence={
|
||||||
|
"preview_sha256": preview_sha256,
|
||||||
|
"target_count": len(normalized_ids),
|
||||||
|
},
|
||||||
|
lease_resource_key=f"files:purge:{tenant_id}",
|
||||||
|
lease_ttl_seconds=15 * 60,
|
||||||
|
resource_type="file_asset_batch",
|
||||||
|
resource_id=preview_sha256,
|
||||||
|
metadata={"resources": ["postgresql"], "bounded_target_count": len(normalized_ids)},
|
||||||
|
block_unresolved_resource=True,
|
||||||
|
)
|
||||||
|
except (RecoveryOperationBusy, RecoveryOperationStateConflict) as exc:
|
||||||
|
raise FileStorageError(
|
||||||
|
"Another purge or unresolved retry owns the tenant purge fence"
|
||||||
|
) from exc
|
||||||
|
except (RecoveryGuaranteeError, RuntimeError, ValueError) as exc:
|
||||||
|
raise FileStorageError("The Files recovery ledger is unavailable; nothing was purged") from exc
|
||||||
|
if started.replayed or started.operation is None:
|
||||||
|
return PurgeResult(
|
||||||
|
recovery_operation_id=started.operation_id,
|
||||||
|
status=started.status,
|
||||||
|
replayed=True,
|
||||||
|
purged_files=0,
|
||||||
|
released_blobs=0,
|
||||||
|
)
|
||||||
|
operation = started.operation
|
||||||
|
try:
|
||||||
|
preview = preview_asset_purge(session, tenant_id=tenant_id, file_ids=normalized_ids)
|
||||||
|
if preview.preview_sha256 != preview_sha256:
|
||||||
|
operation.reject(
|
||||||
|
summary="The purge preview became stale before execution",
|
||||||
|
evidence=_verified_evidence(
|
||||||
|
{"preview_matches": False},
|
||||||
|
expected_preview_sha256=preview_sha256,
|
||||||
|
observed_preview_sha256=preview.preview_sha256,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
raise FileStorageError("The purge preview is stale; preview again")
|
||||||
|
if not preview.eligible:
|
||||||
|
operation.reject(
|
||||||
|
summary="The purge was blocked by current lifecycle policy",
|
||||||
|
evidence=_verified_evidence(
|
||||||
|
{"lifecycle_policy_allows": False},
|
||||||
|
blocked_files=[
|
||||||
|
{"file_id": item.file_id, "blockers": list(item.blockers)}
|
||||||
|
for item in preview.items
|
||||||
|
if item.blockers
|
||||||
|
]
|
||||||
|
),
|
||||||
|
)
|
||||||
|
raise FileStorageError("One or more files are not eligible for purge")
|
||||||
|
assets = (
|
||||||
|
session.query(FileAsset)
|
||||||
|
.filter(FileAsset.tenant_id == tenant_id, FileAsset.id.in_(normalized_ids))
|
||||||
|
.with_for_update()
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
blob_ids = sorted({blob_id for item in preview.items for blob_id in item.blob_ids})
|
||||||
|
if before_commit is not None:
|
||||||
|
before_commit(assets)
|
||||||
|
session.query(FileShare).filter(
|
||||||
|
FileShare.file_asset_id.in_(normalized_ids)
|
||||||
|
).delete(synchronize_session=False)
|
||||||
|
session.query(FileVersion).filter(
|
||||||
|
FileVersion.file_asset_id.in_(normalized_ids)
|
||||||
|
).delete(synchronize_session=False)
|
||||||
|
for asset in assets:
|
||||||
|
session.delete(asset)
|
||||||
|
session.flush()
|
||||||
|
released_blobs = 0
|
||||||
|
for blob_id in blob_ids:
|
||||||
|
references = int(
|
||||||
|
session.query(func.count(FileVersion.id))
|
||||||
|
.filter(FileVersion.blob_id == blob_id)
|
||||||
|
.scalar()
|
||||||
|
or 0
|
||||||
|
)
|
||||||
|
blob = session.get(FileBlob, blob_id)
|
||||||
|
if blob is not None:
|
||||||
|
blob.ref_count = references
|
||||||
|
session.add(blob)
|
||||||
|
if references == 0:
|
||||||
|
released_blobs += 1
|
||||||
|
operation.commit_verified_success(
|
||||||
|
session,
|
||||||
|
evidence=_verified_evidence(
|
||||||
|
{
|
||||||
|
"assets_absent": True,
|
||||||
|
"versions_absent": True,
|
||||||
|
"blob_reference_counts_recalculated": True,
|
||||||
|
},
|
||||||
|
purged_file_ids=normalized_ids,
|
||||||
|
purged_files=len(assets),
|
||||||
|
released_blobs=released_blobs,
|
||||||
|
preview_sha256=preview_sha256,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
return PurgeResult(
|
||||||
|
recovery_operation_id=started.operation_id,
|
||||||
|
status=RecoveryStatus.SUCCEEDED.value,
|
||||||
|
replayed=False,
|
||||||
|
purged_files=len(assets),
|
||||||
|
released_blobs=released_blobs,
|
||||||
|
)
|
||||||
|
except FileStorageError:
|
||||||
|
session.rollback()
|
||||||
|
if not operation.closed:
|
||||||
|
operation.release_unresolved()
|
||||||
|
raise
|
||||||
|
except Exception as exc:
|
||||||
|
session.rollback()
|
||||||
|
if not operation.closed:
|
||||||
|
operation.reject(
|
||||||
|
summary="The purge failed before its database transaction committed",
|
||||||
|
evidence=_verified_evidence(
|
||||||
|
{"database_transaction_committed": False},
|
||||||
|
exception_type=type(exc).__name__,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def garbage_collect_unreferenced_blobs(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
limit: int,
|
||||||
|
approval_reference: str,
|
||||||
|
) -> BlobGcResult:
|
||||||
|
candidate_rows = (
|
||||||
|
session.query(FileBlob)
|
||||||
|
.filter(FileBlob.tenant_id == tenant_id)
|
||||||
|
.filter(
|
||||||
|
or_(FileBlob.retained_until.is_(None), FileBlob.retained_until <= utcnow())
|
||||||
|
)
|
||||||
|
.filter(
|
||||||
|
~session.query(FileVersion.id)
|
||||||
|
.filter(FileVersion.blob_id == FileBlob.id)
|
||||||
|
.exists()
|
||||||
|
)
|
||||||
|
.order_by(FileBlob.created_at.asc(), FileBlob.id.asc())
|
||||||
|
.limit(limit)
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
candidates = tuple(
|
||||||
|
(row.id, row.storage_key) for row in candidate_rows
|
||||||
|
)
|
||||||
|
# Close the read snapshot before the independent recovery transaction takes
|
||||||
|
# its durable SQLite/PostgreSQL write lock.
|
||||||
|
session.commit()
|
||||||
|
deleted = 0
|
||||||
|
unresolved: list[str] = []
|
||||||
|
backend = get_storage_backend()
|
||||||
|
for candidate_id, storage_key in candidates:
|
||||||
|
key_digest = hashlib.sha256(storage_key.encode("utf-8")).hexdigest()
|
||||||
|
request = {
|
||||||
|
"tenant_id": tenant_id,
|
||||||
|
"blob_id": candidate_id,
|
||||||
|
"storage_key_sha256": key_digest,
|
||||||
|
"approval_reference": approval_reference,
|
||||||
|
}
|
||||||
|
try:
|
||||||
|
started = begin_durable_recovery_operation(
|
||||||
|
get_database().SessionLocal,
|
||||||
|
identity=process_runtime_identity(),
|
||||||
|
module_id="files",
|
||||||
|
operation_type="blob-garbage-collection",
|
||||||
|
idempotency_key=(
|
||||||
|
f"files-blob-gc:{candidate_id}:{key_digest[:12]}:"
|
||||||
|
f"{hashlib.sha256(approval_reference.encode('utf-8')).hexdigest()[:12]}"
|
||||||
|
),
|
||||||
|
request=request,
|
||||||
|
recovery_plan=RecoveryPlan(
|
||||||
|
mode=RecoveryMode.FORWARD_RECOVERY,
|
||||||
|
preconditions=(
|
||||||
|
"the caller holds dedicated Files purge authority",
|
||||||
|
"a fresh database check shows no FileVersion reference",
|
||||||
|
),
|
||||||
|
forward_recovery_steps=(
|
||||||
|
"verify the object is absent",
|
||||||
|
"delete the unreferenced FileBlob row after absence is proven",
|
||||||
|
),
|
||||||
|
verification_steps=(
|
||||||
|
"recheck FileVersion references while holding the blob fence",
|
||||||
|
"probe the exact managed storage key after deletion",
|
||||||
|
),
|
||||||
|
approval_reference=approval_reference,
|
||||||
|
),
|
||||||
|
precondition_evidence={
|
||||||
|
"blob_id": candidate_id,
|
||||||
|
"storage_key_sha256": key_digest,
|
||||||
|
"observed_reference_count": 0,
|
||||||
|
},
|
||||||
|
lease_resource_key=f"files:blob:{tenant_id}:{candidate_id}",
|
||||||
|
lease_ttl_seconds=15 * 60,
|
||||||
|
resource_type="file_blob",
|
||||||
|
resource_id=candidate_id,
|
||||||
|
metadata={"resources": ["postgresql", "object-storage"], "storage_backend": backend.name},
|
||||||
|
block_unresolved_resource=True,
|
||||||
|
)
|
||||||
|
except (RecoveryOperationBusy, RecoveryOperationStateConflict):
|
||||||
|
continue
|
||||||
|
except (RecoveryGuaranteeError, RuntimeError, ValueError) as exc:
|
||||||
|
raise FileStorageError(
|
||||||
|
"The Files recovery ledger is unavailable; no blob bytes were deleted"
|
||||||
|
) from exc
|
||||||
|
if started.replayed or started.operation is None:
|
||||||
|
continue
|
||||||
|
operation = started.operation
|
||||||
|
try:
|
||||||
|
blob = (
|
||||||
|
session.query(FileBlob)
|
||||||
|
.filter(FileBlob.id == candidate_id, FileBlob.tenant_id == tenant_id)
|
||||||
|
.with_for_update()
|
||||||
|
.one_or_none()
|
||||||
|
)
|
||||||
|
references = int(
|
||||||
|
session.query(func.count(FileVersion.id))
|
||||||
|
.filter(FileVersion.blob_id == candidate_id)
|
||||||
|
.scalar()
|
||||||
|
or 0
|
||||||
|
)
|
||||||
|
blob_retained = bool(
|
||||||
|
blob is not None
|
||||||
|
and blob.retained_until is not None
|
||||||
|
and _as_utc(blob.retained_until) > utcnow()
|
||||||
|
)
|
||||||
|
except Exception as exc:
|
||||||
|
session.rollback()
|
||||||
|
if not operation.closed:
|
||||||
|
operation.reject(
|
||||||
|
summary="Blob eligibility could not be rechecked before deletion",
|
||||||
|
evidence=_verified_evidence(
|
||||||
|
{"external_effect_started": False},
|
||||||
|
exception_type=type(exc).__name__,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
raise FileStorageError(
|
||||||
|
"Blob eligibility could not be rechecked; no bytes were deleted"
|
||||||
|
) from exc
|
||||||
|
if blob is None or references or blob_retained:
|
||||||
|
operation.reject(
|
||||||
|
summary="Blob garbage collection was no longer eligible",
|
||||||
|
evidence=_verified_evidence(
|
||||||
|
{
|
||||||
|
"blob_present": blob is not None,
|
||||||
|
"reference_count_zero": references == 0,
|
||||||
|
"blob_retention_expired": not blob_retained,
|
||||||
|
},
|
||||||
|
reference_count=references,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
session.rollback()
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
backend.delete(blob.storage_key)
|
||||||
|
object_present = backend.exists(blob.storage_key)
|
||||||
|
except (StorageBackendError, OSError) as exc:
|
||||||
|
try:
|
||||||
|
object_present = backend.exists(blob.storage_key)
|
||||||
|
except (StorageBackendError, OSError):
|
||||||
|
object_present = None
|
||||||
|
session.rollback()
|
||||||
|
if object_present is True:
|
||||||
|
operation.fail(
|
||||||
|
summary="The unreferenced blob object was retained",
|
||||||
|
evidence={"object_present": True, "exception_type": type(exc).__name__},
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
operation.unresolved(
|
||||||
|
status=RecoveryStatus.OUTCOME_UNKNOWN,
|
||||||
|
summary="The blob deletion outcome could not be verified",
|
||||||
|
evidence={"object_present": object_present, "exception_type": type(exc).__name__},
|
||||||
|
failure_summary="Reconcile the exact blob key before retrying garbage collection",
|
||||||
|
)
|
||||||
|
unresolved.append(started.operation_id)
|
||||||
|
continue
|
||||||
|
if object_present:
|
||||||
|
session.rollback()
|
||||||
|
operation.fail(
|
||||||
|
summary="The object store did not delete the unreferenced blob",
|
||||||
|
evidence={"object_present": True},
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
session.delete(blob)
|
||||||
|
session.flush()
|
||||||
|
operation.commit_verified_success(
|
||||||
|
session,
|
||||||
|
evidence=_verified_evidence(
|
||||||
|
{
|
||||||
|
"object_absent": True,
|
||||||
|
"database_blob_absent": True,
|
||||||
|
"reference_count_zero": True,
|
||||||
|
},
|
||||||
|
blob_id=candidate_id,
|
||||||
|
storage_key_sha256=key_digest,
|
||||||
|
object_present=False,
|
||||||
|
database_blob_present=False,
|
||||||
|
reference_count=0,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
deleted += 1
|
||||||
|
except Exception as exc:
|
||||||
|
session.rollback()
|
||||||
|
if not operation.closed:
|
||||||
|
operation.unresolved(
|
||||||
|
status=RecoveryStatus.RECOVERY_REQUIRED,
|
||||||
|
summary="The object is absent but blob metadata still requires forward recovery",
|
||||||
|
evidence={"object_present": False, "exception_type": type(exc).__name__},
|
||||||
|
failure_summary="Delete the FileBlob row only after rechecking all FileVersion references",
|
||||||
|
)
|
||||||
|
unresolved.append(started.operation_id)
|
||||||
|
return BlobGcResult(
|
||||||
|
inspected_blobs=len(candidates),
|
||||||
|
deleted_blobs=deleted,
|
||||||
|
unresolved_operation_ids=tuple(unresolved),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _asset_purge_blockers(session: Session, asset: FileAsset) -> list[str]:
|
||||||
|
blockers: list[str] = []
|
||||||
|
if asset.deleted_at is None:
|
||||||
|
blockers.append("not_soft_deleted")
|
||||||
|
if asset.legal_hold:
|
||||||
|
blockers.append("legal_hold")
|
||||||
|
if asset.retained_until is not None and _as_utc(asset.retained_until) > utcnow():
|
||||||
|
blockers.append("retention_active")
|
||||||
|
if _table_exists(session, CampaignAttachmentUse.__tablename__) and session.query(
|
||||||
|
CampaignAttachmentUse.id
|
||||||
|
).filter(CampaignAttachmentUse.file_asset_id == asset.id).first() is not None:
|
||||||
|
blockers.append("campaign_evidence")
|
||||||
|
if _table_exists(session, FileFormEvidenceGrant.__tablename__) and session.query(
|
||||||
|
FileFormEvidenceGrant.id
|
||||||
|
).filter(FileFormEvidenceGrant.file_asset_id == asset.id).first() is not None:
|
||||||
|
blockers.append("form_evidence")
|
||||||
|
if session.query(FileShare.id).filter(
|
||||||
|
FileShare.file_asset_id == asset.id,
|
||||||
|
effective_file_share_clause(),
|
||||||
|
).first() is not None:
|
||||||
|
blockers.append("active_share")
|
||||||
|
return blockers
|
||||||
|
|
||||||
|
|
||||||
|
def _asset_owner_id(asset: FileAsset) -> str:
|
||||||
|
owner_id = asset.owner_user_id if asset.owner_type == "user" else asset.owner_group_id
|
||||||
|
if not owner_id:
|
||||||
|
raise FileStorageError("File has no valid owner")
|
||||||
|
return owner_id
|
||||||
|
|
||||||
|
|
||||||
|
def _asset_owner_query(session: Session, asset: FileAsset):
|
||||||
|
return _asset_owner_query_by_values(
|
||||||
|
session,
|
||||||
|
tenant_id=asset.tenant_id,
|
||||||
|
owner_type=asset.owner_type,
|
||||||
|
owner_id=_asset_owner_id(asset),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _asset_owner_query_by_values(
|
||||||
|
session: Session, *, tenant_id: str, owner_type: str, owner_id: str
|
||||||
|
):
|
||||||
|
query = session.query(FileAsset).filter(
|
||||||
|
FileAsset.tenant_id == tenant_id, FileAsset.owner_type == owner_type
|
||||||
|
)
|
||||||
|
if owner_type == "user":
|
||||||
|
return query.filter(FileAsset.owner_user_id == owner_id)
|
||||||
|
if owner_type == "group":
|
||||||
|
return query.filter(FileAsset.owner_group_id == owner_id)
|
||||||
|
raise FileStorageError("Files must be owned by a user or group")
|
||||||
|
|
||||||
|
|
||||||
|
def _folder_owner_query(
|
||||||
|
session: Session, *, tenant_id: str, owner_type: str, owner_id: str
|
||||||
|
):
|
||||||
|
query = session.query(FileFolder).filter(
|
||||||
|
FileFolder.tenant_id == tenant_id, FileFolder.owner_type == owner_type
|
||||||
|
)
|
||||||
|
if owner_type == "user":
|
||||||
|
return query.filter(FileFolder.owner_user_id == owner_id)
|
||||||
|
if owner_type == "group":
|
||||||
|
return query.filter(FileFolder.owner_group_id == owner_id)
|
||||||
|
raise FileStorageError("Folders must be owned by a user or group")
|
||||||
|
|
||||||
|
|
||||||
|
def _connector_space_owner_query(session: Session, space: FileConnectorSpace):
|
||||||
|
query = session.query(FileConnectorSpace).filter(
|
||||||
|
FileConnectorSpace.tenant_id == space.tenant_id,
|
||||||
|
FileConnectorSpace.owner_type == space.owner_type,
|
||||||
|
)
|
||||||
|
owner_id = connector_space_owner_id(space)
|
||||||
|
if space.owner_type == "user":
|
||||||
|
return query.filter(FileConnectorSpace.owner_user_id == owner_id)
|
||||||
|
return query.filter(FileConnectorSpace.owner_group_id == owner_id)
|
||||||
|
|
||||||
|
|
||||||
|
def _as_utc(value: datetime) -> datetime:
|
||||||
|
return value.replace(tzinfo=UTC) if value.tzinfo is None else value.astimezone(UTC)
|
||||||
|
|
||||||
|
|
||||||
|
def _iso(value: datetime | None) -> str | None:
|
||||||
|
return _as_utc(value).isoformat() if value is not None else None
|
||||||
|
|
||||||
|
|
||||||
|
def _table_exists(session: Session, table_name: str) -> bool:
|
||||||
|
return bool(session.bind is not None and inspect(session.bind).has_table(table_name))
|
||||||
|
|
||||||
|
|
||||||
|
def _verified_evidence(
|
||||||
|
checks: dict[str, object], **details: object
|
||||||
|
) -> dict[str, object]:
|
||||||
|
return {"verified": True, "checks": checks, **details}
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"BlobGcResult",
|
||||||
|
"PurgePreview",
|
||||||
|
"PurgePreviewItem",
|
||||||
|
"PurgeResult",
|
||||||
|
"execute_asset_purge",
|
||||||
|
"garbage_collect_unreferenced_blobs",
|
||||||
|
"get_asset_for_lifecycle",
|
||||||
|
"preview_asset_purge",
|
||||||
|
"restore_asset",
|
||||||
|
"restore_connector_space",
|
||||||
|
"restore_folder",
|
||||||
|
"set_asset_lifecycle",
|
||||||
|
]
|
||||||
@@ -0,0 +1,931 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from collections.abc import Callable
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
import hashlib
|
||||||
|
from threading import Lock
|
||||||
|
from typing import Protocol
|
||||||
|
|
||||||
|
from sqlalchemy import event, select
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from govoplan_core.core.recovery import (
|
||||||
|
RecoveryGuaranteeError,
|
||||||
|
RecoveryMode,
|
||||||
|
RecoveryOperation,
|
||||||
|
RecoveryPlan,
|
||||||
|
RecoveryStatus,
|
||||||
|
plan_recovery_operation,
|
||||||
|
prepare_recovery_operation,
|
||||||
|
start_recovery_operation,
|
||||||
|
verify_recovery_evidence_chain,
|
||||||
|
)
|
||||||
|
from govoplan_core.core.recovery_runtime import (
|
||||||
|
DurableRecoveryOperation,
|
||||||
|
DurableRecoveryStart,
|
||||||
|
RecoveryOperationBusy,
|
||||||
|
RecoveryOperationStateConflict,
|
||||||
|
begin_durable_recovery_operation,
|
||||||
|
)
|
||||||
|
from govoplan_core.core.runtime_coordination import (
|
||||||
|
RuntimeIdentity,
|
||||||
|
acquire_lease,
|
||||||
|
process_runtime_identity,
|
||||||
|
release_lease,
|
||||||
|
)
|
||||||
|
from govoplan_core.db.session import get_database
|
||||||
|
from govoplan_files.backend.db.models import (
|
||||||
|
FileBlob,
|
||||||
|
FileIntegrityFinding,
|
||||||
|
FileIntegrityScan,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.storage.backends import (
|
||||||
|
StorageBackend,
|
||||||
|
StorageBackendError,
|
||||||
|
StorageObjectMissing,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.storage.common import FileStorageError
|
||||||
|
|
||||||
|
|
||||||
|
_PENDING_EFFECTS_KEY = "govoplan_files_pending_recovery_effects"
|
||||||
|
_HOOKS_INSTALLED_KEY = "govoplan_files_recovery_hooks_installed"
|
||||||
|
_ROLLBACK_ERRORS_KEY = "govoplan_files_recovery_rollback_errors"
|
||||||
|
_SQLITE_FENCES: set[str] = set()
|
||||||
|
_SQLITE_FENCES_LOCK = Lock()
|
||||||
|
|
||||||
|
|
||||||
|
class _PendingEffect(Protocol):
|
||||||
|
def settle(self, *, committed: bool) -> None: ...
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(slots=True)
|
||||||
|
class PendingBlobWrite:
|
||||||
|
operation: DurableRecoveryOperation
|
||||||
|
backend: StorageBackend
|
||||||
|
tenant_id: str
|
||||||
|
blob_id: str
|
||||||
|
storage_key: str
|
||||||
|
semantic_checksum_sha256: str
|
||||||
|
semantic_size_bytes: int
|
||||||
|
protection_discriminator: str
|
||||||
|
created_new: bool
|
||||||
|
expected_storage_checksum_sha256: str | None = None
|
||||||
|
expected_storage_size_bytes: int | None = None
|
||||||
|
expected_envelope_id: str | None = None
|
||||||
|
sqlite_fence_key: str | None = None
|
||||||
|
restart_after_rollback: Callable[[], DurableRecoveryOperation] | None = None
|
||||||
|
|
||||||
|
def prepare_stored_bytes(
|
||||||
|
self,
|
||||||
|
data: bytes,
|
||||||
|
*,
|
||||||
|
envelope_id: str | None,
|
||||||
|
) -> None:
|
||||||
|
"""Retain process-local verification evidence without recording content."""
|
||||||
|
|
||||||
|
self.expected_storage_checksum_sha256 = hashlib.sha256(data).hexdigest()
|
||||||
|
self.expected_storage_size_bytes = len(data)
|
||||||
|
self.expected_envelope_id = envelope_id
|
||||||
|
|
||||||
|
def settle(self, *, committed: bool) -> None:
|
||||||
|
if not committed and self.restart_after_rollback is not None:
|
||||||
|
try:
|
||||||
|
self.operation = self.restart_after_rollback()
|
||||||
|
except Exception:
|
||||||
|
# SQLite cannot make the pre-effect ledger row independent of
|
||||||
|
# the caller transaction. If reconstructing the evidence row
|
||||||
|
# also fails, still avoid leaving a known new orphan behind.
|
||||||
|
if self.created_new:
|
||||||
|
try:
|
||||||
|
self.backend.delete(self.storage_key)
|
||||||
|
except (StorageBackendError, OSError):
|
||||||
|
pass
|
||||||
|
raise
|
||||||
|
evidence = _blob_write_evidence(self)
|
||||||
|
if _blob_write_complete(evidence):
|
||||||
|
self.operation.succeed(evidence=evidence)
|
||||||
|
return
|
||||||
|
|
||||||
|
if self.created_new and evidence.get("database_blob_present") is False:
|
||||||
|
self._settle_unreferenced_new_object(evidence)
|
||||||
|
return
|
||||||
|
|
||||||
|
if not self.created_new and _object_matches(evidence):
|
||||||
|
if _forward_complete_blob_repair(self):
|
||||||
|
completed = _blob_write_evidence(self)
|
||||||
|
if _blob_write_complete(completed):
|
||||||
|
self.operation.succeed(evidence=completed)
|
||||||
|
return
|
||||||
|
|
||||||
|
if evidence.get("database_blob_present") is True and not _object_matches(
|
||||||
|
evidence
|
||||||
|
):
|
||||||
|
_quarantine_blob_after_failed_verification(self, evidence)
|
||||||
|
evidence = _blob_write_evidence(self)
|
||||||
|
|
||||||
|
status = (
|
||||||
|
RecoveryStatus.OUTCOME_UNKNOWN
|
||||||
|
if not evidence.get("verified")
|
||||||
|
else RecoveryStatus.RECOVERY_REQUIRED
|
||||||
|
)
|
||||||
|
transaction = "committed" if committed else "rolled back"
|
||||||
|
self.operation.unresolved(
|
||||||
|
status=status,
|
||||||
|
summary=f"Managed Files blob write remained unresolved after the database transaction {transaction}",
|
||||||
|
evidence=evidence,
|
||||||
|
failure_summary=(
|
||||||
|
"The managed object and Files blob metadata require reconciliation"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
def _settle_unreferenced_new_object(self, evidence: dict[str, object]) -> None:
|
||||||
|
object_present = evidence.get("object_present")
|
||||||
|
if object_present is False:
|
||||||
|
self.operation.reject(
|
||||||
|
summary="The Files blob write left no durable object or database row",
|
||||||
|
evidence=evidence,
|
||||||
|
)
|
||||||
|
return
|
||||||
|
if object_present is not True:
|
||||||
|
self.operation.unresolved(
|
||||||
|
status=RecoveryStatus.OUTCOME_UNKNOWN,
|
||||||
|
summary="The unreferenced Files object could not be probed",
|
||||||
|
evidence=evidence,
|
||||||
|
failure_summary="Object storage availability prevented upload compensation",
|
||||||
|
)
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
self.backend.delete(self.storage_key)
|
||||||
|
except (StorageBackendError, OSError):
|
||||||
|
self.operation.unresolved(
|
||||||
|
status=RecoveryStatus.RECOVERY_REQUIRED,
|
||||||
|
summary="An unreferenced Files object could not be compensated",
|
||||||
|
evidence=evidence,
|
||||||
|
failure_summary="Delete the unreferenced managed object after verifying that no FileBlob references it",
|
||||||
|
)
|
||||||
|
return
|
||||||
|
recovered = _blob_write_evidence(self)
|
||||||
|
if (
|
||||||
|
recovered.get("verified") is True
|
||||||
|
and recovered.get("database_blob_present") is False
|
||||||
|
and recovered.get("object_present") is False
|
||||||
|
):
|
||||||
|
self.operation.compensate(
|
||||||
|
failure_summary="The Files database transaction did not retain the new blob",
|
||||||
|
failure_evidence=evidence,
|
||||||
|
recovery_evidence=recovered,
|
||||||
|
)
|
||||||
|
return
|
||||||
|
self.operation.unresolved(
|
||||||
|
status=RecoveryStatus.RECOVERY_REQUIRED,
|
||||||
|
summary="Files upload compensation could not be verified",
|
||||||
|
evidence=recovered,
|
||||||
|
failure_summary="The unreferenced managed object requires operator reconciliation",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(slots=True)
|
||||||
|
class PendingOrphanCleanup:
|
||||||
|
operation: DurableRecoveryOperation
|
||||||
|
backend: StorageBackend
|
||||||
|
finding_id: str
|
||||||
|
tenant_id: str
|
||||||
|
storage_key: str
|
||||||
|
resolved_by_user_id: str
|
||||||
|
|
||||||
|
def settle(self, *, committed: bool) -> None:
|
||||||
|
del committed
|
||||||
|
evidence = _orphan_cleanup_evidence(self)
|
||||||
|
if _orphan_cleanup_complete(evidence):
|
||||||
|
self.operation.succeed(evidence=evidence)
|
||||||
|
return
|
||||||
|
if (
|
||||||
|
evidence.get("verified") is True
|
||||||
|
and evidence.get("object_present") is True
|
||||||
|
and evidence.get("finding_deleted") is False
|
||||||
|
):
|
||||||
|
self.operation.reject(
|
||||||
|
summary="The orphan object was retained and the cleanup finding stayed open",
|
||||||
|
evidence=evidence,
|
||||||
|
)
|
||||||
|
return
|
||||||
|
if evidence.get("object_present") is False:
|
||||||
|
if _forward_complete_orphan_finding(self):
|
||||||
|
completed = _orphan_cleanup_evidence(self)
|
||||||
|
if _orphan_cleanup_complete(completed):
|
||||||
|
self.operation.succeed(evidence=completed)
|
||||||
|
return
|
||||||
|
status = (
|
||||||
|
RecoveryStatus.OUTCOME_UNKNOWN
|
||||||
|
if not evidence.get("verified")
|
||||||
|
else RecoveryStatus.RECOVERY_REQUIRED
|
||||||
|
)
|
||||||
|
self.operation.unresolved(
|
||||||
|
status=status,
|
||||||
|
summary="Files orphan cleanup requires reconciliation",
|
||||||
|
evidence=evidence,
|
||||||
|
failure_summary="Recheck the object and integrity-finding state before another cleanup attempt",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def begin_blob_write_recovery(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
backend: StorageBackend,
|
||||||
|
tenant_id: str,
|
||||||
|
blob_id: str,
|
||||||
|
storage_key: str,
|
||||||
|
semantic_checksum_sha256: str,
|
||||||
|
semantic_size_bytes: int,
|
||||||
|
protection_discriminator: str,
|
||||||
|
created_new: bool,
|
||||||
|
repair_token: str | None = None,
|
||||||
|
) -> PendingBlobWrite:
|
||||||
|
disposition = "create" if created_new else "repair"
|
||||||
|
state_token = repair_token or blob_id
|
||||||
|
key_digest = hashlib.sha256(storage_key.encode("utf-8")).hexdigest()
|
||||||
|
idempotency_key = (
|
||||||
|
f"files-blob-{disposition}:{blob_id}:{state_token[:48]}"
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
identity = process_runtime_identity()
|
||||||
|
except RuntimeError as exc:
|
||||||
|
raise FileStorageError(
|
||||||
|
"The Files recovery ledger is unavailable; no object was written"
|
||||||
|
) from exc
|
||||||
|
lease_resource_key = f"files:blob:{tenant_id}:{blob_id}"
|
||||||
|
request = {
|
||||||
|
"tenant_id": tenant_id,
|
||||||
|
"blob_id": blob_id,
|
||||||
|
"storage_key": storage_key if created_new else None,
|
||||||
|
"storage_key_sha256": key_digest,
|
||||||
|
"semantic_checksum_sha256": semantic_checksum_sha256,
|
||||||
|
"semantic_size_bytes": semantic_size_bytes,
|
||||||
|
"protection_discriminator": protection_discriminator,
|
||||||
|
"disposition": disposition,
|
||||||
|
}
|
||||||
|
recovery_plan = RecoveryPlan(
|
||||||
|
mode=(
|
||||||
|
RecoveryMode.COMPENSATION
|
||||||
|
if created_new
|
||||||
|
else RecoveryMode.FORWARD_RECOVERY
|
||||||
|
),
|
||||||
|
preconditions=(
|
||||||
|
"the caller has Files write authority for the target owner",
|
||||||
|
"the storage key belongs to the tenant Files namespace",
|
||||||
|
"the request records digests rather than file contents",
|
||||||
|
),
|
||||||
|
compensation_steps=(
|
||||||
|
"verify that no FileBlob references the newly reserved key",
|
||||||
|
"delete only that unreferenced key and verify absence",
|
||||||
|
)
|
||||||
|
if created_new
|
||||||
|
else (),
|
||||||
|
forward_recovery_steps=(
|
||||||
|
"verify the expected bytes at the existing blob key",
|
||||||
|
"forward-complete matching integrity metadata or quarantine the blob",
|
||||||
|
)
|
||||||
|
if not created_new
|
||||||
|
else (),
|
||||||
|
verification_steps=(
|
||||||
|
"reload FileBlob metadata through an independent session",
|
||||||
|
"stream and hash the managed object independently",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
precondition_evidence = {
|
||||||
|
"blob_id": blob_id,
|
||||||
|
"storage_key_sha256": key_digest,
|
||||||
|
"semantic_checksum_sha256": semantic_checksum_sha256,
|
||||||
|
"semantic_size_bytes": semantic_size_bytes,
|
||||||
|
"created_new": created_new,
|
||||||
|
}
|
||||||
|
metadata = {
|
||||||
|
"resources": ["postgresql", "object-storage"],
|
||||||
|
"storage_backend": backend.name,
|
||||||
|
"durability_mode": "independent_transaction",
|
||||||
|
}
|
||||||
|
session_factory = get_database().SessionLocal
|
||||||
|
sqlite_mode = session.get_bind().dialect.name == "sqlite"
|
||||||
|
sqlite_fence_key: str | None = None
|
||||||
|
|
||||||
|
def start_independent(*, reconstructed_after_rollback: bool = False) -> DurableRecoveryStart:
|
||||||
|
return begin_durable_recovery_operation(
|
||||||
|
session_factory,
|
||||||
|
identity=identity,
|
||||||
|
module_id="files",
|
||||||
|
operation_type=f"blob-{disposition}",
|
||||||
|
idempotency_key=idempotency_key,
|
||||||
|
request=request,
|
||||||
|
recovery_plan=recovery_plan,
|
||||||
|
precondition_evidence={
|
||||||
|
**precondition_evidence,
|
||||||
|
**(
|
||||||
|
{
|
||||||
|
"sqlite_caller_transaction_rolled_back": True,
|
||||||
|
"effect_may_have_preceded_durable_intent": True,
|
||||||
|
}
|
||||||
|
if reconstructed_after_rollback
|
||||||
|
else {}
|
||||||
|
),
|
||||||
|
},
|
||||||
|
lease_resource_key=lease_resource_key,
|
||||||
|
lease_ttl_seconds=15 * 60,
|
||||||
|
resource_type="file_blob",
|
||||||
|
resource_id=blob_id,
|
||||||
|
metadata={
|
||||||
|
**metadata,
|
||||||
|
"durability_mode": (
|
||||||
|
"sqlite_post_rollback_reconstruction"
|
||||||
|
if reconstructed_after_rollback
|
||||||
|
else "independent_transaction"
|
||||||
|
),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
if sqlite_mode:
|
||||||
|
_reserve_sqlite_fence(lease_resource_key)
|
||||||
|
sqlite_fence_key = lease_resource_key
|
||||||
|
started = _begin_caller_transaction_recovery_operation(
|
||||||
|
session,
|
||||||
|
session_factory=session_factory,
|
||||||
|
identity=identity,
|
||||||
|
module_id="files",
|
||||||
|
operation_type=f"blob-{disposition}",
|
||||||
|
idempotency_key=idempotency_key,
|
||||||
|
request=request,
|
||||||
|
recovery_plan=recovery_plan,
|
||||||
|
precondition_evidence={
|
||||||
|
**precondition_evidence,
|
||||||
|
"sqlite_caller_transaction": True,
|
||||||
|
"reduced_crash_durability": True,
|
||||||
|
},
|
||||||
|
lease_resource_key=lease_resource_key,
|
||||||
|
lease_ttl_seconds=15 * 60,
|
||||||
|
resource_type="file_blob",
|
||||||
|
resource_id=blob_id,
|
||||||
|
metadata={
|
||||||
|
**metadata,
|
||||||
|
"resources": ["sqlite", "object-storage"],
|
||||||
|
"durability_mode": "sqlite_caller_transaction",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
started = start_independent()
|
||||||
|
except (RecoveryOperationBusy, RecoveryOperationStateConflict) as exc:
|
||||||
|
if sqlite_fence_key is not None:
|
||||||
|
_release_sqlite_fence(sqlite_fence_key)
|
||||||
|
raise FileStorageError(
|
||||||
|
"This managed blob is already owned by another recovery operation"
|
||||||
|
) from exc
|
||||||
|
except (RecoveryGuaranteeError, RuntimeError) as exc:
|
||||||
|
if sqlite_fence_key is not None:
|
||||||
|
_release_sqlite_fence(sqlite_fence_key)
|
||||||
|
raise FileStorageError(
|
||||||
|
"The Files recovery ledger is unavailable; no object was written"
|
||||||
|
) from exc
|
||||||
|
if started.replayed or started.operation is None:
|
||||||
|
if sqlite_fence_key is not None:
|
||||||
|
_release_sqlite_fence(sqlite_fence_key)
|
||||||
|
raise FileStorageError(
|
||||||
|
"The matching Files blob operation was already completed; reload before retrying"
|
||||||
|
)
|
||||||
|
pending = PendingBlobWrite(
|
||||||
|
operation=started.operation,
|
||||||
|
backend=backend,
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
blob_id=blob_id,
|
||||||
|
storage_key=storage_key,
|
||||||
|
semantic_checksum_sha256=semantic_checksum_sha256,
|
||||||
|
semantic_size_bytes=semantic_size_bytes,
|
||||||
|
protection_discriminator=protection_discriminator,
|
||||||
|
created_new=created_new,
|
||||||
|
sqlite_fence_key=sqlite_fence_key,
|
||||||
|
restart_after_rollback=(
|
||||||
|
lambda: _require_started_operation(
|
||||||
|
start_independent(reconstructed_after_rollback=True)
|
||||||
|
)
|
||||||
|
if sqlite_mode
|
||||||
|
else None
|
||||||
|
),
|
||||||
|
)
|
||||||
|
_register_pending_effect(session, pending)
|
||||||
|
return pending
|
||||||
|
|
||||||
|
|
||||||
|
def begin_orphan_cleanup_recovery(
|
||||||
|
session: Session,
|
||||||
|
finding: FileIntegrityFinding,
|
||||||
|
*,
|
||||||
|
backend: StorageBackend,
|
||||||
|
user_id: str,
|
||||||
|
) -> PendingOrphanCleanup:
|
||||||
|
key_digest = hashlib.sha256(finding.storage_key.encode("utf-8")).hexdigest()
|
||||||
|
try:
|
||||||
|
started = begin_durable_recovery_operation(
|
||||||
|
get_database().SessionLocal,
|
||||||
|
identity=process_runtime_identity(),
|
||||||
|
module_id="files",
|
||||||
|
operation_type="integrity-orphan-cleanup",
|
||||||
|
idempotency_key=f"files-orphan-cleanup:{finding.id}",
|
||||||
|
request={
|
||||||
|
"tenant_id": finding.tenant_id,
|
||||||
|
"finding_id": finding.id,
|
||||||
|
"storage_key_sha256": key_digest,
|
||||||
|
},
|
||||||
|
recovery_plan=RecoveryPlan(
|
||||||
|
mode=RecoveryMode.FORWARD_RECOVERY,
|
||||||
|
preconditions=(
|
||||||
|
"the integrity finding identifies an unreferenced object",
|
||||||
|
"the object key remains inside the completed scan scope",
|
||||||
|
"a fresh database reference check found no FileBlob",
|
||||||
|
),
|
||||||
|
forward_recovery_steps=(
|
||||||
|
"verify object absence independently",
|
||||||
|
"mark the durable finding deleted only after absence is proven",
|
||||||
|
),
|
||||||
|
verification_steps=(
|
||||||
|
"reload the finding and its scan through an independent session",
|
||||||
|
"probe the original storage key through the configured backend",
|
||||||
|
),
|
||||||
|
),
|
||||||
|
precondition_evidence={
|
||||||
|
"finding_id": finding.id,
|
||||||
|
"scan_id": finding.scan_id,
|
||||||
|
"storage_key_sha256": key_digest,
|
||||||
|
"finding_state": finding.state,
|
||||||
|
},
|
||||||
|
lease_resource_key=(
|
||||||
|
f"files:orphan-cleanup:{finding.tenant_id}:{key_digest[:40]}"
|
||||||
|
),
|
||||||
|
lease_ttl_seconds=15 * 60,
|
||||||
|
resource_type="file_integrity_finding",
|
||||||
|
resource_id=finding.id,
|
||||||
|
metadata={
|
||||||
|
"resources": ["postgresql", "object-storage"],
|
||||||
|
"storage_backend": backend.name,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
except (RecoveryOperationBusy, RecoveryOperationStateConflict) as exc:
|
||||||
|
raise FileStorageError(
|
||||||
|
"This orphan object is already owned by another recovery operation"
|
||||||
|
) from exc
|
||||||
|
except (RecoveryGuaranteeError, RuntimeError) as exc:
|
||||||
|
raise FileStorageError(
|
||||||
|
"The Files recovery ledger is unavailable; no object was deleted"
|
||||||
|
) from exc
|
||||||
|
if started.replayed or started.operation is None:
|
||||||
|
raise FileStorageError(
|
||||||
|
"This orphan cleanup was already completed; reload the finding"
|
||||||
|
)
|
||||||
|
pending = PendingOrphanCleanup(
|
||||||
|
operation=started.operation,
|
||||||
|
backend=backend,
|
||||||
|
finding_id=finding.id,
|
||||||
|
tenant_id=finding.tenant_id,
|
||||||
|
storage_key=finding.storage_key,
|
||||||
|
resolved_by_user_id=user_id,
|
||||||
|
)
|
||||||
|
_register_pending_effect(session, pending)
|
||||||
|
return pending
|
||||||
|
|
||||||
|
|
||||||
|
def _register_pending_effect(session: Session, effect: _PendingEffect) -> None:
|
||||||
|
if not session.in_transaction():
|
||||||
|
session.begin()
|
||||||
|
pending = session.info.setdefault(_PENDING_EFFECTS_KEY, [])
|
||||||
|
pending.append(effect)
|
||||||
|
if session.info.get(_HOOKS_INSTALLED_KEY):
|
||||||
|
return
|
||||||
|
event.listen(session, "after_commit", _after_session_commit)
|
||||||
|
event.listen(session, "after_rollback", _after_session_rollback)
|
||||||
|
session.info[_HOOKS_INSTALLED_KEY] = True
|
||||||
|
|
||||||
|
|
||||||
|
def _after_session_commit(session: Session) -> None:
|
||||||
|
# SQLAlchemy also emits after_commit for a released SAVEPOINT. A Files
|
||||||
|
# batch may open one while creating a later recovery row; settle only when
|
||||||
|
# the outer business transaction has actually become visible.
|
||||||
|
if session.in_nested_transaction():
|
||||||
|
return
|
||||||
|
_settle_pending_effects(session, committed=True)
|
||||||
|
|
||||||
|
|
||||||
|
def _after_session_rollback(session: Session) -> None:
|
||||||
|
if session.in_nested_transaction():
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
_settle_pending_effects(session, committed=False)
|
||||||
|
except RecoveryGuaranteeError as exc:
|
||||||
|
session.info.setdefault(_ROLLBACK_ERRORS_KEY, []).append(str(exc))
|
||||||
|
|
||||||
|
|
||||||
|
def _settle_pending_effects(session: Session, *, committed: bool) -> None:
|
||||||
|
pending = list(session.info.pop(_PENDING_EFFECTS_KEY, []))
|
||||||
|
failures: list[Exception] = []
|
||||||
|
for effect in pending:
|
||||||
|
try:
|
||||||
|
effect.settle(committed=committed)
|
||||||
|
except Exception as exc: # preserve every effect's chance to settle
|
||||||
|
failures.append(exc)
|
||||||
|
operation = getattr(effect, "operation", None)
|
||||||
|
if operation is not None and not operation.closed:
|
||||||
|
try:
|
||||||
|
operation.release_unresolved()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
finally:
|
||||||
|
sqlite_fence_key = getattr(effect, "sqlite_fence_key", None)
|
||||||
|
if sqlite_fence_key:
|
||||||
|
_release_sqlite_fence(sqlite_fence_key)
|
||||||
|
if failures:
|
||||||
|
raise RecoveryGuaranteeError(
|
||||||
|
f"{len(failures)} Files recovery operation(s) could not be finalized"
|
||||||
|
) from failures[0]
|
||||||
|
|
||||||
|
|
||||||
|
def _reserve_sqlite_fence(resource_key: str) -> None:
|
||||||
|
with _SQLITE_FENCES_LOCK:
|
||||||
|
if resource_key in _SQLITE_FENCES:
|
||||||
|
raise RecoveryOperationBusy(
|
||||||
|
f"Another local SQLite transaction owns {resource_key}"
|
||||||
|
)
|
||||||
|
_SQLITE_FENCES.add(resource_key)
|
||||||
|
|
||||||
|
|
||||||
|
def _release_sqlite_fence(resource_key: str) -> None:
|
||||||
|
with _SQLITE_FENCES_LOCK:
|
||||||
|
_SQLITE_FENCES.discard(resource_key)
|
||||||
|
|
||||||
|
|
||||||
|
def _require_started_operation(started: DurableRecoveryStart) -> DurableRecoveryOperation:
|
||||||
|
if started.replayed or started.operation is None:
|
||||||
|
raise RecoveryOperationStateConflict(started.operation_id, started.status)
|
||||||
|
return started.operation
|
||||||
|
|
||||||
|
|
||||||
|
def _begin_caller_transaction_recovery_operation(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
session_factory: Callable[[], Session],
|
||||||
|
identity: RuntimeIdentity,
|
||||||
|
module_id: str,
|
||||||
|
operation_type: str,
|
||||||
|
idempotency_key: str,
|
||||||
|
request: dict[str, object],
|
||||||
|
recovery_plan: RecoveryPlan,
|
||||||
|
precondition_evidence: dict[str, object],
|
||||||
|
lease_resource_key: str,
|
||||||
|
lease_ttl_seconds: int,
|
||||||
|
resource_type: str,
|
||||||
|
resource_id: str,
|
||||||
|
metadata: dict[str, object],
|
||||||
|
) -> DurableRecoveryStart:
|
||||||
|
"""Start SQLite recovery evidence inside the caller transaction.
|
||||||
|
|
||||||
|
SQLite permits only one writer, so opening the normal independent ledger
|
||||||
|
transaction after an earlier member of a batch has written will deadlock
|
||||||
|
until the busy timeout. The caller-transaction mode preserves fencing,
|
||||||
|
request hashing, checkpoints, commit verification, and rollback
|
||||||
|
compensation, but cannot make the intent survive a hard process loss
|
||||||
|
before the caller commits. PostgreSQL never uses this reduced mode.
|
||||||
|
"""
|
||||||
|
|
||||||
|
claim = acquire_lease(
|
||||||
|
session,
|
||||||
|
installation_id=identity.installation_id,
|
||||||
|
resource_key=lease_resource_key,
|
||||||
|
holder_node_id=identity.node_id,
|
||||||
|
holder_incarnation=identity.incarnation,
|
||||||
|
ttl_seconds=lease_ttl_seconds,
|
||||||
|
metadata={
|
||||||
|
"module_id": module_id,
|
||||||
|
"operation_type": operation_type,
|
||||||
|
"durability_mode": "sqlite_caller_transaction",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
if claim is None:
|
||||||
|
raise RecoveryOperationBusy(
|
||||||
|
f"Another runtime owns the recovery fence for {lease_resource_key}"
|
||||||
|
)
|
||||||
|
existing = session.execute(
|
||||||
|
select(RecoveryOperation).where(
|
||||||
|
RecoveryOperation.installation_id == identity.installation_id,
|
||||||
|
RecoveryOperation.module_id == module_id,
|
||||||
|
RecoveryOperation.idempotency_key == idempotency_key,
|
||||||
|
)
|
||||||
|
).scalar_one_or_none()
|
||||||
|
operation = plan_recovery_operation(
|
||||||
|
session,
|
||||||
|
installation_id=identity.installation_id,
|
||||||
|
module_id=module_id,
|
||||||
|
operation_type=operation_type,
|
||||||
|
idempotency_key=idempotency_key,
|
||||||
|
request=request,
|
||||||
|
recovery_plan=recovery_plan,
|
||||||
|
resource_type=resource_type,
|
||||||
|
resource_id=resource_id,
|
||||||
|
lease_claim=claim,
|
||||||
|
metadata=metadata,
|
||||||
|
)
|
||||||
|
if existing is not None:
|
||||||
|
if operation.status == RecoveryStatus.SUCCEEDED.value:
|
||||||
|
release_lease(session, claim)
|
||||||
|
return DurableRecoveryStart(
|
||||||
|
operation_id=operation.id,
|
||||||
|
status=operation.status,
|
||||||
|
replayed=True,
|
||||||
|
operation=None,
|
||||||
|
)
|
||||||
|
raise RecoveryOperationStateConflict(operation.id, operation.status)
|
||||||
|
prepare_recovery_operation(
|
||||||
|
session,
|
||||||
|
operation,
|
||||||
|
evidence=precondition_evidence,
|
||||||
|
lease_claim=claim,
|
||||||
|
)
|
||||||
|
start_recovery_operation(
|
||||||
|
session,
|
||||||
|
operation,
|
||||||
|
evidence={"lease_resource_key": lease_resource_key},
|
||||||
|
lease_claim=claim,
|
||||||
|
)
|
||||||
|
if not verify_recovery_evidence_chain(session, operation.id):
|
||||||
|
raise RecoveryGuaranteeError(
|
||||||
|
"Recovery checkpoint chain verification failed before side effects"
|
||||||
|
)
|
||||||
|
return DurableRecoveryStart(
|
||||||
|
operation_id=operation.id,
|
||||||
|
status=operation.status,
|
||||||
|
replayed=False,
|
||||||
|
operation=DurableRecoveryOperation(
|
||||||
|
session_factory=session_factory,
|
||||||
|
operation_id=operation.id,
|
||||||
|
lease_claim=claim,
|
||||||
|
lease_ttl_seconds=lease_ttl_seconds,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _blob_write_evidence(effect: PendingBlobWrite) -> dict[str, object]:
|
||||||
|
database_blob_present: bool | None
|
||||||
|
database_matches: bool | None
|
||||||
|
database_integrity_verified: bool | None
|
||||||
|
try:
|
||||||
|
with effect.operation.session_factory() as session:
|
||||||
|
blob = session.get(FileBlob, effect.blob_id)
|
||||||
|
database_blob_present = blob is not None
|
||||||
|
database_matches = bool(
|
||||||
|
blob is not None
|
||||||
|
and blob.tenant_id == effect.tenant_id
|
||||||
|
and blob.storage_key == effect.storage_key
|
||||||
|
and blob.checksum_sha256 == effect.semantic_checksum_sha256
|
||||||
|
and blob.size_bytes == effect.semantic_size_bytes
|
||||||
|
and blob.protection_discriminator
|
||||||
|
== effect.protection_discriminator
|
||||||
|
and blob.encryption_envelope_id == effect.expected_envelope_id
|
||||||
|
and (
|
||||||
|
effect.expected_storage_checksum_sha256
|
||||||
|
== (
|
||||||
|
blob.storage_checksum_sha256
|
||||||
|
or blob.checksum_sha256
|
||||||
|
)
|
||||||
|
)
|
||||||
|
) if blob is not None else False
|
||||||
|
database_integrity_verified = bool(
|
||||||
|
blob is not None
|
||||||
|
and blob.integrity_status == "verified"
|
||||||
|
and blob.quarantined_at is None
|
||||||
|
) if blob is not None else False
|
||||||
|
except Exception:
|
||||||
|
database_blob_present = None
|
||||||
|
database_matches = None
|
||||||
|
database_integrity_verified = None
|
||||||
|
|
||||||
|
object_present: bool | None
|
||||||
|
observed_size: int | None = None
|
||||||
|
observed_checksum: str | None = None
|
||||||
|
try:
|
||||||
|
digest = hashlib.sha256()
|
||||||
|
observed_size = 0
|
||||||
|
for chunk in effect.backend.iter_bytes(effect.storage_key):
|
||||||
|
observed_size += len(chunk)
|
||||||
|
digest.update(chunk)
|
||||||
|
observed_checksum = digest.hexdigest()
|
||||||
|
object_present = True
|
||||||
|
except StorageObjectMissing:
|
||||||
|
object_present = False
|
||||||
|
except (StorageBackendError, OSError):
|
||||||
|
object_present = None
|
||||||
|
|
||||||
|
object_size_matches = (
|
||||||
|
observed_size == effect.expected_storage_size_bytes
|
||||||
|
if object_present is True and effect.expected_storage_size_bytes is not None
|
||||||
|
else False if object_present is False else None
|
||||||
|
)
|
||||||
|
object_checksum_matches = (
|
||||||
|
observed_checksum == effect.expected_storage_checksum_sha256
|
||||||
|
if object_present is True
|
||||||
|
and effect.expected_storage_checksum_sha256 is not None
|
||||||
|
else False if object_present is False else None
|
||||||
|
)
|
||||||
|
verified = database_blob_present is not None and object_present is not None
|
||||||
|
return {
|
||||||
|
"verified": verified,
|
||||||
|
"checks": {
|
||||||
|
"database_reloaded": database_blob_present is not None,
|
||||||
|
"object_probed": object_present is not None,
|
||||||
|
"database_matches_request": database_matches,
|
||||||
|
"database_integrity_verified": database_integrity_verified,
|
||||||
|
"object_size_matches": object_size_matches,
|
||||||
|
"object_checksum_matches": object_checksum_matches,
|
||||||
|
},
|
||||||
|
"blob_id": effect.blob_id,
|
||||||
|
"database_blob_present": database_blob_present,
|
||||||
|
"database_matches_request": database_matches,
|
||||||
|
"database_integrity_verified": database_integrity_verified,
|
||||||
|
"object_present": object_present,
|
||||||
|
"observed_size_bytes": observed_size,
|
||||||
|
"observed_checksum_sha256": observed_checksum,
|
||||||
|
"expected_storage_size_bytes": effect.expected_storage_size_bytes,
|
||||||
|
"expected_storage_checksum_sha256": effect.expected_storage_checksum_sha256,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _blob_write_complete(evidence: dict[str, object]) -> bool:
|
||||||
|
return bool(
|
||||||
|
evidence.get("verified") is True
|
||||||
|
and evidence.get("database_matches_request") is True
|
||||||
|
and evidence.get("database_integrity_verified") is True
|
||||||
|
and _object_matches(evidence)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _object_matches(evidence: dict[str, object]) -> bool:
|
||||||
|
checks = evidence.get("checks")
|
||||||
|
return bool(
|
||||||
|
isinstance(checks, dict)
|
||||||
|
and evidence.get("object_present") is True
|
||||||
|
and checks.get("object_size_matches") is True
|
||||||
|
and checks.get("object_checksum_matches") is True
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _forward_complete_blob_repair(effect: PendingBlobWrite) -> bool:
|
||||||
|
try:
|
||||||
|
with effect.operation.session_factory() as session:
|
||||||
|
blob = session.get(FileBlob, effect.blob_id)
|
||||||
|
if (
|
||||||
|
blob is None
|
||||||
|
or blob.tenant_id != effect.tenant_id
|
||||||
|
or blob.storage_key != effect.storage_key
|
||||||
|
or blob.checksum_sha256 != effect.semantic_checksum_sha256
|
||||||
|
or blob.size_bytes != effect.semantic_size_bytes
|
||||||
|
or blob.protection_discriminator
|
||||||
|
!= effect.protection_discriminator
|
||||||
|
or blob.encryption_envelope_id != effect.expected_envelope_id
|
||||||
|
):
|
||||||
|
return False
|
||||||
|
blob.storage_checksum_sha256 = (
|
||||||
|
effect.expected_storage_checksum_sha256
|
||||||
|
if effect.expected_envelope_id
|
||||||
|
else None
|
||||||
|
)
|
||||||
|
blob.storage_size_bytes = (
|
||||||
|
effect.expected_storage_size_bytes
|
||||||
|
if effect.expected_envelope_id
|
||||||
|
else None
|
||||||
|
)
|
||||||
|
blob.integrity_status = "verified"
|
||||||
|
blob.integrity_checked_at = datetime.now(UTC)
|
||||||
|
blob.integrity_failure = None
|
||||||
|
blob.quarantined_at = None
|
||||||
|
session.add(blob)
|
||||||
|
session.commit()
|
||||||
|
return True
|
||||||
|
except Exception:
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _quarantine_blob_after_failed_verification(
|
||||||
|
effect: PendingBlobWrite,
|
||||||
|
evidence: dict[str, object],
|
||||||
|
) -> None:
|
||||||
|
try:
|
||||||
|
with effect.operation.session_factory() as session:
|
||||||
|
blob = session.get(FileBlob, effect.blob_id)
|
||||||
|
if blob is None or blob.storage_key != effect.storage_key:
|
||||||
|
return
|
||||||
|
blob.integrity_status = (
|
||||||
|
"missing"
|
||||||
|
if evidence.get("object_present") is False
|
||||||
|
else "checksum_mismatch"
|
||||||
|
)
|
||||||
|
blob.integrity_checked_at = datetime.now(UTC)
|
||||||
|
blob.integrity_failure = "recovery_verification_failed"
|
||||||
|
blob.quarantined_at = datetime.now(UTC)
|
||||||
|
session.add(blob)
|
||||||
|
session.commit()
|
||||||
|
except Exception:
|
||||||
|
return
|
||||||
|
|
||||||
|
|
||||||
|
def _orphan_cleanup_evidence(effect: PendingOrphanCleanup) -> dict[str, object]:
|
||||||
|
try:
|
||||||
|
object_present: bool | None = effect.backend.exists(effect.storage_key)
|
||||||
|
except (StorageBackendError, OSError):
|
||||||
|
object_present = None
|
||||||
|
finding_present: bool | None
|
||||||
|
finding_deleted: bool | None
|
||||||
|
still_unreferenced: bool | None
|
||||||
|
try:
|
||||||
|
with effect.operation.session_factory() as session:
|
||||||
|
finding = session.get(FileIntegrityFinding, effect.finding_id)
|
||||||
|
finding_present = finding is not None
|
||||||
|
finding_deleted = bool(
|
||||||
|
finding is not None and finding.state == "deleted"
|
||||||
|
) if finding is not None else False
|
||||||
|
referenced = (
|
||||||
|
session.query(FileBlob.id)
|
||||||
|
.filter(
|
||||||
|
FileBlob.tenant_id == effect.tenant_id,
|
||||||
|
FileBlob.storage_key == effect.storage_key,
|
||||||
|
)
|
||||||
|
.first()
|
||||||
|
)
|
||||||
|
still_unreferenced = referenced is None
|
||||||
|
except Exception:
|
||||||
|
finding_present = None
|
||||||
|
finding_deleted = None
|
||||||
|
still_unreferenced = None
|
||||||
|
verified = object_present is not None and finding_present is not None
|
||||||
|
return {
|
||||||
|
"verified": verified,
|
||||||
|
"checks": {
|
||||||
|
"database_reloaded": finding_present is not None,
|
||||||
|
"object_probed": object_present is not None,
|
||||||
|
"finding_marked_deleted": finding_deleted,
|
||||||
|
"object_absent": (
|
||||||
|
not object_present if object_present is not None else None
|
||||||
|
),
|
||||||
|
"still_unreferenced": still_unreferenced,
|
||||||
|
},
|
||||||
|
"finding_id": effect.finding_id,
|
||||||
|
"finding_present": finding_present,
|
||||||
|
"finding_deleted": finding_deleted,
|
||||||
|
"object_present": object_present,
|
||||||
|
"still_unreferenced": still_unreferenced,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _orphan_cleanup_complete(evidence: dict[str, object]) -> bool:
|
||||||
|
return bool(
|
||||||
|
evidence.get("verified") is True
|
||||||
|
and evidence.get("finding_deleted") is True
|
||||||
|
and evidence.get("object_present") is False
|
||||||
|
and evidence.get("still_unreferenced") is True
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _forward_complete_orphan_finding(effect: PendingOrphanCleanup) -> bool:
|
||||||
|
try:
|
||||||
|
with effect.operation.session_factory() as session:
|
||||||
|
finding = session.get(FileIntegrityFinding, effect.finding_id)
|
||||||
|
if (
|
||||||
|
finding is None
|
||||||
|
or finding.tenant_id != effect.tenant_id
|
||||||
|
or finding.storage_key != effect.storage_key
|
||||||
|
):
|
||||||
|
return False
|
||||||
|
scan = session.get(FileIntegrityScan, finding.scan_id)
|
||||||
|
if scan is None or not finding.storage_key.startswith(
|
||||||
|
scan.storage_prefix
|
||||||
|
):
|
||||||
|
return False
|
||||||
|
referenced = (
|
||||||
|
session.query(FileBlob.id)
|
||||||
|
.filter(
|
||||||
|
FileBlob.tenant_id == effect.tenant_id,
|
||||||
|
FileBlob.storage_key == effect.storage_key,
|
||||||
|
)
|
||||||
|
.first()
|
||||||
|
)
|
||||||
|
if referenced:
|
||||||
|
return False
|
||||||
|
finding.state = "deleted"
|
||||||
|
finding.resolved_at = datetime.now(UTC)
|
||||||
|
finding.resolved_by_user_id = effect.resolved_by_user_id
|
||||||
|
session.add(finding)
|
||||||
|
session.commit()
|
||||||
|
return True
|
||||||
|
except Exception:
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"PendingBlobWrite",
|
||||||
|
"PendingOrphanCleanup",
|
||||||
|
"begin_blob_write_recovery",
|
||||||
|
"begin_orphan_cleanup_recovery",
|
||||||
|
]
|
||||||
@@ -0,0 +1,331 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import inspect
|
||||||
|
import socket
|
||||||
|
import threading
|
||||||
|
from functools import lru_cache
|
||||||
|
from importlib import import_module
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from govoplan_core.security.outbound_http import (
|
||||||
|
OutboundHttpError,
|
||||||
|
create_outbound_connection,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class SdkPeerPinningError(RuntimeError):
|
||||||
|
"""Raised when an optional SDK cannot be bound to the pinned transport."""
|
||||||
|
|
||||||
|
|
||||||
|
_BOTOCORE_CLIENT_CREATION_LOCK = threading.Lock()
|
||||||
|
|
||||||
|
|
||||||
|
def create_pinned_s3_client(**kwargs: Any) -> Any:
|
||||||
|
"""Construct an S3 client whose first and every later socket is pinned.
|
||||||
|
|
||||||
|
Botocore does not expose the HTTP-session class through boto3's public
|
||||||
|
client API. Its endpoint creator does expose that seam, so this function
|
||||||
|
replaces the creator only for the bounded client-construction operation.
|
||||||
|
The lock avoids an unsafe interleaving with another GovOPlaN client
|
||||||
|
construction. Any unrelated botocore client created during the short
|
||||||
|
replacement window also receives the stricter transport.
|
||||||
|
"""
|
||||||
|
|
||||||
|
try:
|
||||||
|
boto3 = import_module("boto3")
|
||||||
|
botocore_args = import_module("botocore.args")
|
||||||
|
except ImportError as exc:
|
||||||
|
raise SdkPeerPinningError(
|
||||||
|
"S3 connector browsing requires the optional boto3 dependency"
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
pinned_session_cls = _botocore_transport_types()[0]
|
||||||
|
original_creator = getattr(botocore_args, "EndpointCreator", None)
|
||||||
|
if original_creator is None or not hasattr(original_creator, "create_endpoint"):
|
||||||
|
raise SdkPeerPinningError(
|
||||||
|
"The installed botocore release does not expose the endpoint transport seam required for peer pinning"
|
||||||
|
)
|
||||||
|
|
||||||
|
class PinnedEndpointCreator(original_creator): # type: ignore[misc, valid-type]
|
||||||
|
def create_endpoint(self, *args: Any, **endpoint_kwargs: Any) -> Any:
|
||||||
|
endpoint_kwargs["http_session_cls"] = pinned_session_cls
|
||||||
|
return super().create_endpoint(*args, **endpoint_kwargs)
|
||||||
|
|
||||||
|
with _BOTOCORE_CLIENT_CREATION_LOCK:
|
||||||
|
current_creator = getattr(botocore_args, "EndpointCreator", None)
|
||||||
|
if current_creator is not original_creator:
|
||||||
|
raise SdkPeerPinningError(
|
||||||
|
"Botocore endpoint construction changed concurrently; refusing to create an unproven S3 client"
|
||||||
|
)
|
||||||
|
botocore_args.EndpointCreator = PinnedEndpointCreator
|
||||||
|
try:
|
||||||
|
session_type = getattr(getattr(boto3, "session", None), "Session", None)
|
||||||
|
if session_type is None:
|
||||||
|
raise SdkPeerPinningError(
|
||||||
|
"The installed boto3 release does not expose its isolated session constructor"
|
||||||
|
)
|
||||||
|
client = session_type().client("s3", **kwargs)
|
||||||
|
finally:
|
||||||
|
if getattr(botocore_args, "EndpointCreator", None) is PinnedEndpointCreator:
|
||||||
|
botocore_args.EndpointCreator = original_creator
|
||||||
|
|
||||||
|
http_session = getattr(getattr(client, "_endpoint", None), "http_session", None)
|
||||||
|
if http_session is None or not isinstance(http_session, pinned_session_cls):
|
||||||
|
close = getattr(client, "close", None)
|
||||||
|
if callable(close):
|
||||||
|
close()
|
||||||
|
raise SdkPeerPinningError(
|
||||||
|
"Botocore did not install the required pinned HTTP transport; the S3 client was discarded"
|
||||||
|
)
|
||||||
|
return client
|
||||||
|
|
||||||
|
|
||||||
|
@lru_cache(maxsize=1)
|
||||||
|
def _botocore_transport_types() -> tuple[type[Any], type[Any], type[Any]]:
|
||||||
|
try:
|
||||||
|
awsrequest = import_module("botocore.awsrequest")
|
||||||
|
httpsession = import_module("botocore.httpsession")
|
||||||
|
urllib3_exceptions = import_module("urllib3.exceptions")
|
||||||
|
except ImportError as exc:
|
||||||
|
raise SdkPeerPinningError(
|
||||||
|
"S3 connector browsing requires a compatible botocore HTTP transport"
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
required = (
|
||||||
|
"AWSHTTPConnection",
|
||||||
|
"AWSHTTPSConnection",
|
||||||
|
"AWSHTTPConnectionPool",
|
||||||
|
"AWSHTTPSConnectionPool",
|
||||||
|
)
|
||||||
|
if any(not hasattr(awsrequest, name) for name in required) or not hasattr(
|
||||||
|
httpsession, "URLLib3Session"
|
||||||
|
):
|
||||||
|
raise SdkPeerPinningError(
|
||||||
|
"The installed botocore release is missing the connection classes required for peer pinning"
|
||||||
|
)
|
||||||
|
|
||||||
|
def pinned_new_connection(connection: Any) -> socket.socket:
|
||||||
|
hostname = str(getattr(connection, "_dns_host", "") or "").strip()
|
||||||
|
port = int(getattr(connection, "port", 0) or 0)
|
||||||
|
if not hostname or not port:
|
||||||
|
raise urllib3_exceptions.NewConnectionError(
|
||||||
|
connection, "Pinned S3 connection is missing its target authority"
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
return create_outbound_connection(
|
||||||
|
hostname,
|
||||||
|
port,
|
||||||
|
timeout=getattr(connection, "timeout", None),
|
||||||
|
source_address=getattr(connection, "source_address", None),
|
||||||
|
socket_options=getattr(connection, "socket_options", None),
|
||||||
|
label="S3 connector peer",
|
||||||
|
)
|
||||||
|
except socket.timeout as exc:
|
||||||
|
raise urllib3_exceptions.ConnectTimeoutError(
|
||||||
|
connection,
|
||||||
|
f"Connection to {hostname} timed out while selecting an approved peer",
|
||||||
|
) from exc
|
||||||
|
except (OSError, OutboundHttpError, ValueError) as exc:
|
||||||
|
raise urllib3_exceptions.NewConnectionError(
|
||||||
|
connection,
|
||||||
|
f"S3 connector peer was rejected: {exc}",
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
class PinnedAWSHTTPConnection(awsrequest.AWSHTTPConnection):
|
||||||
|
_new_conn = pinned_new_connection
|
||||||
|
|
||||||
|
class PinnedAWSHTTPSConnection(awsrequest.AWSHTTPSConnection):
|
||||||
|
_new_conn = pinned_new_connection
|
||||||
|
|
||||||
|
class PinnedAWSHTTPConnectionPool(awsrequest.AWSHTTPConnectionPool):
|
||||||
|
ConnectionCls = PinnedAWSHTTPConnection
|
||||||
|
|
||||||
|
class PinnedAWSHTTPSConnectionPool(awsrequest.AWSHTTPSConnectionPool):
|
||||||
|
ConnectionCls = PinnedAWSHTTPSConnection
|
||||||
|
|
||||||
|
class PinnedURLLib3Session(httpsession.URLLib3Session):
|
||||||
|
def __init__(self, *args: Any, **kwargs: Any) -> None:
|
||||||
|
# A proxy would select the final target outside this process. Until
|
||||||
|
# proxy peer delegation is modeled, connector traffic is direct.
|
||||||
|
kwargs["proxies"] = {}
|
||||||
|
super().__init__(*args, **kwargs)
|
||||||
|
self._pool_classes_by_scheme = {
|
||||||
|
"http": PinnedAWSHTTPConnectionPool,
|
||||||
|
"https": PinnedAWSHTTPSConnectionPool,
|
||||||
|
}
|
||||||
|
manager = getattr(self, "_manager", None)
|
||||||
|
if manager is None or not hasattr(manager, "pool_classes_by_scheme"):
|
||||||
|
raise SdkPeerPinningError(
|
||||||
|
"The installed botocore pool manager cannot enforce pinned connection classes"
|
||||||
|
)
|
||||||
|
manager.pool_classes_by_scheme = self._pool_classes_by_scheme
|
||||||
|
|
||||||
|
return PinnedURLLib3Session, PinnedAWSHTTPConnection, PinnedAWSHTTPSConnection
|
||||||
|
|
||||||
|
|
||||||
|
@lru_cache(maxsize=1)
|
||||||
|
def pinned_smb_connection_cache() -> dict[str, Any]:
|
||||||
|
"""Return the Files-owned cache; unproven process-global sessions are never reused."""
|
||||||
|
|
||||||
|
return {}
|
||||||
|
|
||||||
|
|
||||||
|
def install_pinned_smb_transport(smbclient: Any) -> Any:
|
||||||
|
"""Install a process-wide fail-closed smbclient session factory.
|
||||||
|
|
||||||
|
smbclient routes initial connections, reconnects, and DFS targets through
|
||||||
|
``smbclient._pool.register_session``. Replacing that single factory with a
|
||||||
|
behavior-compatible implementation ensures every target uses a socket
|
||||||
|
selected by Core's deployment-wide outbound policy.
|
||||||
|
"""
|
||||||
|
|
||||||
|
try:
|
||||||
|
pool = import_module("smbclient._pool")
|
||||||
|
connection_module = import_module("smbprotocol.connection")
|
||||||
|
session_module = import_module("smbprotocol.session")
|
||||||
|
transport_module = import_module("smbprotocol.transport")
|
||||||
|
except ImportError as exc:
|
||||||
|
raise SdkPeerPinningError(
|
||||||
|
"SMB connector browsing requires the optional smbprotocol dependency"
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
current = getattr(pool, "register_session", None)
|
||||||
|
if getattr(current, "__govoplan_peer_pinned__", False):
|
||||||
|
expected_tcp = getattr(current, "__govoplan_pinned_tcp__", None)
|
||||||
|
if expected_tcp is None or getattr(connection_module, "Tcp", None) is not expected_tcp:
|
||||||
|
raise SdkPeerPinningError(
|
||||||
|
"The installed SMB transport changed after peer pinning; refusing to reuse the session factory"
|
||||||
|
)
|
||||||
|
return smbclient
|
||||||
|
required_parameters = {
|
||||||
|
"server",
|
||||||
|
"username",
|
||||||
|
"password",
|
||||||
|
"port",
|
||||||
|
"encrypt",
|
||||||
|
"connection_timeout",
|
||||||
|
"connection_cache",
|
||||||
|
"auth_protocol",
|
||||||
|
"require_signing",
|
||||||
|
}
|
||||||
|
if current is None or not required_parameters.issubset(
|
||||||
|
inspect.signature(current).parameters
|
||||||
|
):
|
||||||
|
raise SdkPeerPinningError(
|
||||||
|
"The installed smbprotocol release does not expose the session seam required for peer pinning"
|
||||||
|
)
|
||||||
|
|
||||||
|
class PinnedTcp(transport_module.Tcp):
|
||||||
|
def connect(self) -> None:
|
||||||
|
with self._sock_lock:
|
||||||
|
if self.connected:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
self._sock = create_outbound_connection(
|
||||||
|
self.server,
|
||||||
|
int(self.port),
|
||||||
|
timeout=self.timeout,
|
||||||
|
label="SMB connector peer",
|
||||||
|
)
|
||||||
|
except (OSError, OutboundHttpError, ValueError) as exc:
|
||||||
|
raise ValueError(
|
||||||
|
f"SMB connector peer '{self.server}:{self.port}' was rejected: {exc}"
|
||||||
|
) from exc
|
||||||
|
self._sock.settimeout(None)
|
||||||
|
self.connected = True
|
||||||
|
|
||||||
|
# Connection.connect() instantiates the module-level Tcp symbol on every
|
||||||
|
# reconnect. Replacing it process-wide is deliberate: an SMB connection
|
||||||
|
# created by another code path must become stricter, never bypass Files'
|
||||||
|
# peer boundary.
|
||||||
|
if not hasattr(connection_module, "Tcp"):
|
||||||
|
raise SdkPeerPinningError(
|
||||||
|
"The installed smbprotocol release cannot install the pinned TCP transport"
|
||||||
|
)
|
||||||
|
connection_module.Tcp = PinnedTcp
|
||||||
|
|
||||||
|
def pinned_register_session(
|
||||||
|
server: str,
|
||||||
|
username: str | None = None,
|
||||||
|
password: str | None = None,
|
||||||
|
port: int = 445,
|
||||||
|
encrypt: bool | None = None,
|
||||||
|
connection_timeout: float = 60,
|
||||||
|
connection_cache: dict[str, Any] | None = None,
|
||||||
|
auth_protocol: str = "negotiate",
|
||||||
|
require_signing: bool = True,
|
||||||
|
) -> Any:
|
||||||
|
cache = pinned_smb_connection_cache() if connection_cache is None else connection_cache
|
||||||
|
connection_key = f"{server.lower()}:{port}"
|
||||||
|
connection = cache.get(connection_key)
|
||||||
|
transport = getattr(connection, "transport", None)
|
||||||
|
if connection is not None and not isinstance(transport, PinnedTcp):
|
||||||
|
disconnect = getattr(connection, "disconnect", None)
|
||||||
|
if callable(disconnect):
|
||||||
|
try:
|
||||||
|
disconnect(close=True)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
cache.pop(connection_key, None)
|
||||||
|
connection = None
|
||||||
|
if connection is None or not getattr(connection.transport, "connected", False):
|
||||||
|
connection = connection_module.Connection(
|
||||||
|
pool.ClientConfig().client_guid,
|
||||||
|
server,
|
||||||
|
port,
|
||||||
|
require_signing=require_signing,
|
||||||
|
)
|
||||||
|
connection.transport = PinnedTcp(server, port)
|
||||||
|
connection.connect(timeout=connection_timeout)
|
||||||
|
if not isinstance(connection.transport, PinnedTcp):
|
||||||
|
disconnect = getattr(connection, "disconnect", None)
|
||||||
|
if callable(disconnect):
|
||||||
|
try:
|
||||||
|
disconnect(close=True)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
raise SdkPeerPinningError(
|
||||||
|
"smbprotocol replaced the required pinned TCP transport during connection setup"
|
||||||
|
)
|
||||||
|
cache[connection_key] = connection
|
||||||
|
|
||||||
|
session = next(
|
||||||
|
(
|
||||||
|
item
|
||||||
|
for item in connection.session_table.values()
|
||||||
|
if username is None or item.username == username
|
||||||
|
),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
if session is None:
|
||||||
|
session = session_module.Session(
|
||||||
|
connection,
|
||||||
|
username=username,
|
||||||
|
password=password,
|
||||||
|
require_encryption=(encrypt is True),
|
||||||
|
auth_protocol=auth_protocol,
|
||||||
|
)
|
||||||
|
session.connect()
|
||||||
|
elif encrypt is not None:
|
||||||
|
if session.encrypt_data and not encrypt:
|
||||||
|
raise ValueError(
|
||||||
|
"Cannot disable encryption on an already negotiated session."
|
||||||
|
)
|
||||||
|
if not session.encrypt_data and encrypt:
|
||||||
|
session.encrypt = True
|
||||||
|
return session
|
||||||
|
|
||||||
|
pinned_register_session.__govoplan_peer_pinned__ = True # type: ignore[attr-defined]
|
||||||
|
pinned_register_session.__govoplan_pinned_tcp__ = PinnedTcp # type: ignore[attr-defined]
|
||||||
|
pool.register_session = pinned_register_session
|
||||||
|
if hasattr(smbclient, "register_session"):
|
||||||
|
smbclient.register_session = pinned_register_session
|
||||||
|
return smbclient
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"SdkPeerPinningError",
|
||||||
|
"create_pinned_s3_client",
|
||||||
|
"install_pinned_smb_transport",
|
||||||
|
"pinned_smb_connection_cache",
|
||||||
|
]
|
||||||
@@ -0,0 +1,34 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
|
from sqlalchemy import and_, or_
|
||||||
|
|
||||||
|
from govoplan_files.backend.db.models import FileShare
|
||||||
|
from govoplan_files.backend.storage.common import utcnow
|
||||||
|
|
||||||
|
|
||||||
|
def effective_file_share_clause(*, at: datetime | None = None):
|
||||||
|
checked_at = at or utcnow()
|
||||||
|
return and_(
|
||||||
|
FileShare.revoked_at.is_(None),
|
||||||
|
or_(FileShare.expires_at.is_(None), FileShare.expires_at > checked_at),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def file_share_is_active(share: FileShare, *, at: datetime | None = None) -> bool:
|
||||||
|
if share.revoked_at is not None:
|
||||||
|
return False
|
||||||
|
if share.expires_at is None:
|
||||||
|
return True
|
||||||
|
checked_at = _aware_utc(at or utcnow())
|
||||||
|
return _aware_utc(share.expires_at) > checked_at
|
||||||
|
|
||||||
|
|
||||||
|
def _aware_utc(value: datetime) -> datetime:
|
||||||
|
if value.tzinfo is None:
|
||||||
|
return value.replace(tzinfo=timezone.utc)
|
||||||
|
return value.astimezone(timezone.utc)
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["effective_file_share_clause", "file_share_is_active"]
|
||||||
@@ -1,15 +1,23 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import unittest
|
import unittest
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
from sqlalchemy import create_engine
|
from sqlalchemy import create_engine, text
|
||||||
from sqlalchemy.orm import sessionmaker
|
from sqlalchemy.orm import sessionmaker
|
||||||
|
|
||||||
from govoplan_access.backend.db.models import Account, Group, User
|
from govoplan_access.backend.db.models import Account, Group, User
|
||||||
from govoplan_core.core.access import PrincipalRef
|
from govoplan_core.core.access import PrincipalRef
|
||||||
|
from govoplan_core.core.change_sequence import ChangeSequenceEntry
|
||||||
from govoplan_core.db.base import Base
|
from govoplan_core.db.base import Base
|
||||||
from govoplan_files.backend.capabilities import FilesAccessService, virtual_folder_resource_id
|
from govoplan_files.backend.capabilities import FilesAccessService, virtual_folder_resource_id
|
||||||
from govoplan_files.backend.db.models import FileAsset, FileFolder, FileShare
|
from govoplan_files.backend.db.models import FileAsset, FileFolder, FileShare
|
||||||
|
from govoplan_files.backend.storage.files import (
|
||||||
|
count_assets_for_user,
|
||||||
|
list_assets_for_user,
|
||||||
|
list_recent_assets_for_user,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
TENANT_ID = "tenant-1"
|
TENANT_ID = "tenant-1"
|
||||||
@@ -19,6 +27,54 @@ GROUP_ID = "group-1"
|
|||||||
|
|
||||||
|
|
||||||
class FilesAccessProviderTests(unittest.TestCase):
|
class FilesAccessProviderTests(unittest.TestCase):
|
||||||
|
def test_recent_projection_is_bounded_ordered_and_access_filtered(self) -> None:
|
||||||
|
session = _session()
|
||||||
|
self.addCleanup(_close_session, session)
|
||||||
|
_seed_access_subjects(session)
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
session.add_all(
|
||||||
|
[
|
||||||
|
FileAsset(
|
||||||
|
id="owned-older",
|
||||||
|
tenant_id=TENANT_ID,
|
||||||
|
owner_type="user",
|
||||||
|
owner_user_id=USER_ID,
|
||||||
|
display_path="older.pdf",
|
||||||
|
filename="older.pdf",
|
||||||
|
updated_at=now - timedelta(hours=2),
|
||||||
|
),
|
||||||
|
FileAsset(
|
||||||
|
id="owned-newer",
|
||||||
|
tenant_id=TENANT_ID,
|
||||||
|
owner_type="user",
|
||||||
|
owner_user_id=USER_ID,
|
||||||
|
display_path="newer.pdf",
|
||||||
|
filename="newer.pdf",
|
||||||
|
updated_at=now - timedelta(hours=1),
|
||||||
|
),
|
||||||
|
FileAsset(
|
||||||
|
id="other-newest",
|
||||||
|
tenant_id=TENANT_ID,
|
||||||
|
owner_type="user",
|
||||||
|
owner_user_id=OTHER_USER_ID,
|
||||||
|
display_path="private.pdf",
|
||||||
|
filename="private.pdf",
|
||||||
|
updated_at=now,
|
||||||
|
),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
|
||||||
|
with patch("govoplan_files.backend.storage.files.user_group_ids", return_value=[]):
|
||||||
|
recent = list_recent_assets_for_user(
|
||||||
|
session,
|
||||||
|
tenant_id=TENANT_ID,
|
||||||
|
user_id=USER_ID,
|
||||||
|
limit=1,
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(["owned-newer"], [asset.id for asset in recent])
|
||||||
|
|
||||||
def test_file_access_provider_explains_owner_share_admin_and_missing_resources(self) -> None:
|
def test_file_access_provider_explains_owner_share_admin_and_missing_resources(self) -> None:
|
||||||
session = _session()
|
session = _session()
|
||||||
self.addCleanup(_close_session, session)
|
self.addCleanup(_close_session, session)
|
||||||
@@ -75,10 +131,123 @@ class FilesAccessProviderTests(unittest.TestCase):
|
|||||||
self.assertTrue(any(item.kind == "owner" and item.id == GROUP_ID for item in virtual_items))
|
self.assertTrue(any(item.kind == "owner" and item.id == GROUP_ID for item in virtual_items))
|
||||||
self.assertEqual("files.not_found", missing_items[0].source)
|
self.assertEqual("files.not_found", missing_items[0].source)
|
||||||
|
|
||||||
|
def test_file_property_filters_cover_campaign_and_audit_usage(self) -> None:
|
||||||
|
session = _session()
|
||||||
|
self.addCleanup(_close_session, session)
|
||||||
|
_seed_access_subjects(session)
|
||||||
|
assets = [
|
||||||
|
FileAsset(
|
||||||
|
id=f"file-{index}",
|
||||||
|
tenant_id=TENANT_ID,
|
||||||
|
owner_type="user",
|
||||||
|
owner_user_id=USER_ID,
|
||||||
|
display_path=f"{index}.pdf",
|
||||||
|
filename=f"{index}.pdf",
|
||||||
|
)
|
||||||
|
for index in range(1, 4)
|
||||||
|
]
|
||||||
|
session.add_all([
|
||||||
|
*assets,
|
||||||
|
FileShare(
|
||||||
|
id="share-campaign",
|
||||||
|
tenant_id=TENANT_ID,
|
||||||
|
file_asset_id=assets[0].id,
|
||||||
|
target_type="campaign",
|
||||||
|
target_id="campaign-1",
|
||||||
|
permission="read",
|
||||||
|
),
|
||||||
|
FileShare(
|
||||||
|
id="share-campaign-expired",
|
||||||
|
tenant_id=TENANT_ID,
|
||||||
|
file_asset_id=assets[2].id,
|
||||||
|
target_type="campaign",
|
||||||
|
target_id="campaign-1",
|
||||||
|
permission="read",
|
||||||
|
expires_at=datetime.now(timezone.utc) - timedelta(seconds=1),
|
||||||
|
),
|
||||||
|
])
|
||||||
|
session.commit()
|
||||||
|
session.execute(
|
||||||
|
text(
|
||||||
|
"INSERT INTO campaign_attachment_uses "
|
||||||
|
"(id, tenant_id, file_asset_id, use_stage) "
|
||||||
|
"VALUES (:id, :tenant_id, :file_asset_id, :use_stage)"
|
||||||
|
),
|
||||||
|
{
|
||||||
|
"id": "attachment-use-1",
|
||||||
|
"tenant_id": TENANT_ID,
|
||||||
|
"file_asset_id": assets[1].id,
|
||||||
|
"use_stage": "sent",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
|
||||||
|
linked = list_assets_for_user(
|
||||||
|
session,
|
||||||
|
tenant_id=TENANT_ID,
|
||||||
|
user_id=USER_ID,
|
||||||
|
owner_type="user",
|
||||||
|
owner_id=USER_ID,
|
||||||
|
campaign_usage="linked",
|
||||||
|
)
|
||||||
|
unlinked = list_assets_for_user(
|
||||||
|
session,
|
||||||
|
tenant_id=TENANT_ID,
|
||||||
|
user_id=USER_ID,
|
||||||
|
owner_type="user",
|
||||||
|
owner_id=USER_ID,
|
||||||
|
campaign_usage="unlinked",
|
||||||
|
)
|
||||||
|
audit_relevant = list_assets_for_user(
|
||||||
|
session,
|
||||||
|
tenant_id=TENANT_ID,
|
||||||
|
user_id=USER_ID,
|
||||||
|
owner_type="user",
|
||||||
|
owner_id=USER_ID,
|
||||||
|
audit_relevant=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual([asset.id for asset in linked], ["file-1", "file-2"])
|
||||||
|
self.assertEqual([asset.id for asset in unlinked], ["file-3"])
|
||||||
|
self.assertEqual([asset.id for asset in audit_relevant], ["file-2"])
|
||||||
|
self.assertEqual(
|
||||||
|
count_assets_for_user(
|
||||||
|
session,
|
||||||
|
tenant_id=TENANT_ID,
|
||||||
|
user_id=USER_ID,
|
||||||
|
owner_type="user",
|
||||||
|
owner_id=USER_ID,
|
||||||
|
campaign_usage="unlinked",
|
||||||
|
),
|
||||||
|
1,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _session():
|
def _session():
|
||||||
engine = create_engine("sqlite:///:memory:", future=True)
|
engine = create_engine("sqlite:///:memory:", future=True)
|
||||||
Base.metadata.create_all(bind=engine, tables=[Account.__table__, User.__table__, Group.__table__, FileAsset.__table__, FileFolder.__table__, FileShare.__table__])
|
Base.metadata.create_all(
|
||||||
|
bind=engine,
|
||||||
|
tables=[
|
||||||
|
Account.__table__,
|
||||||
|
User.__table__,
|
||||||
|
Group.__table__,
|
||||||
|
ChangeSequenceEntry.__table__,
|
||||||
|
FileAsset.__table__,
|
||||||
|
FileFolder.__table__,
|
||||||
|
FileShare.__table__,
|
||||||
|
],
|
||||||
|
)
|
||||||
|
with engine.begin() as connection:
|
||||||
|
connection.execute(
|
||||||
|
text(
|
||||||
|
"CREATE TABLE campaign_attachment_uses ("
|
||||||
|
"id VARCHAR(36) PRIMARY KEY, "
|
||||||
|
"tenant_id VARCHAR(36) NOT NULL, "
|
||||||
|
"file_asset_id VARCHAR(36) NOT NULL, "
|
||||||
|
"use_stage VARCHAR(20) NOT NULL"
|
||||||
|
")"
|
||||||
|
)
|
||||||
|
)
|
||||||
return sessionmaker(bind=engine, future=True)()
|
return sessionmaker(bind=engine, future=True)()
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,101 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import io
|
||||||
|
import hashlib
|
||||||
|
import unittest
|
||||||
|
import zipfile
|
||||||
|
from types import SimpleNamespace
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from fastapi import UploadFile
|
||||||
|
|
||||||
|
from govoplan_files.backend.routes.uploads import (
|
||||||
|
_validate_archive_preview_token,
|
||||||
|
preview_archive_upload,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.storage.common import FileStorageError
|
||||||
|
|
||||||
|
|
||||||
|
def _archive_bytes() -> bytes:
|
||||||
|
target = io.BytesIO()
|
||||||
|
with zipfile.ZipFile(target, "w", compression=zipfile.ZIP_DEFLATED) as archive:
|
||||||
|
archive.writestr("folder/report.txt", b"report")
|
||||||
|
return target.getvalue()
|
||||||
|
|
||||||
|
|
||||||
|
class ArchivePreviewTokenTests(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
self.settings = SimpleNamespace(
|
||||||
|
file_upload_zip_max_bytes=10 * 1024 * 1024,
|
||||||
|
file_archive_max_entries=10_000,
|
||||||
|
file_archive_max_expanded_bytes=2 * 1024 * 1024 * 1024,
|
||||||
|
file_archive_max_expansion_ratio=100,
|
||||||
|
file_archive_preview_ttl_seconds=30 * 60,
|
||||||
|
)
|
||||||
|
self.principal = SimpleNamespace(
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
user=SimpleNamespace(id="user-1"),
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_preview_token_is_bound_to_actor_archive_and_destination(self) -> None:
|
||||||
|
archive_data = _archive_bytes()
|
||||||
|
upload = UploadFile(
|
||||||
|
file=io.BytesIO(archive_data),
|
||||||
|
filename="monthly.zip",
|
||||||
|
)
|
||||||
|
with patch(
|
||||||
|
"govoplan_files.backend.routes.uploads.settings",
|
||||||
|
self.settings,
|
||||||
|
):
|
||||||
|
preview = preview_archive_upload(
|
||||||
|
file=upload,
|
||||||
|
owner_type="user",
|
||||||
|
owner_id="user-1",
|
||||||
|
path="incoming",
|
||||||
|
campaign_id=None,
|
||||||
|
password=None,
|
||||||
|
principal=self.principal,
|
||||||
|
)
|
||||||
|
_validate_archive_preview_token(
|
||||||
|
preview.preview_token,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
user_id="user-1",
|
||||||
|
owner_type="user",
|
||||||
|
owner_id="user-1",
|
||||||
|
path="incoming",
|
||||||
|
campaign_id=None,
|
||||||
|
archive_format="zip",
|
||||||
|
archive_sha256=hashlib.sha256(archive_data).hexdigest(),
|
||||||
|
)
|
||||||
|
with self.assertRaisesRegex(
|
||||||
|
FileStorageError, "does not match this upload destination"
|
||||||
|
):
|
||||||
|
_validate_archive_preview_token(
|
||||||
|
preview.preview_token,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
user_id="user-1",
|
||||||
|
owner_type="user",
|
||||||
|
owner_id="user-1",
|
||||||
|
path="elsewhere",
|
||||||
|
campaign_id=None,
|
||||||
|
archive_format="zip",
|
||||||
|
archive_sha256=hashlib.sha256(archive_data).hexdigest(),
|
||||||
|
)
|
||||||
|
with self.assertRaisesRegex(
|
||||||
|
FileStorageError, "contents changed"
|
||||||
|
):
|
||||||
|
_validate_archive_preview_token(
|
||||||
|
preview.preview_token,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
user_id="user-1",
|
||||||
|
owner_type="user",
|
||||||
|
owner_id="user-1",
|
||||||
|
path="incoming",
|
||||||
|
campaign_id=None,
|
||||||
|
archive_format="zip",
|
||||||
|
archive_sha256="0" * 64,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,170 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import io
|
||||||
|
import tarfile
|
||||||
|
import unittest
|
||||||
|
import zipfile
|
||||||
|
from types import SimpleNamespace
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
import pyzipper
|
||||||
|
|
||||||
|
from govoplan_files.backend.storage.archives import (
|
||||||
|
ArchivePasswordError,
|
||||||
|
extract_archive_upload,
|
||||||
|
inspect_archive,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.storage.common import FileStorageError
|
||||||
|
|
||||||
|
|
||||||
|
def _zip_bytes(entries: dict[str, bytes]) -> bytes:
|
||||||
|
target = io.BytesIO()
|
||||||
|
with zipfile.ZipFile(target, "w", compression=zipfile.ZIP_DEFLATED) as archive:
|
||||||
|
for path, data in entries.items():
|
||||||
|
archive.writestr(path, data)
|
||||||
|
return target.getvalue()
|
||||||
|
|
||||||
|
|
||||||
|
def _tar_bytes(entries: dict[str, bytes], mode: str = "w:gz") -> bytes:
|
||||||
|
target = io.BytesIO()
|
||||||
|
with tarfile.open(fileobj=target, mode=mode) as archive:
|
||||||
|
for path, data in entries.items():
|
||||||
|
info = tarfile.TarInfo(path)
|
||||||
|
info.size = len(data)
|
||||||
|
archive.addfile(info, io.BytesIO(data))
|
||||||
|
return target.getvalue()
|
||||||
|
|
||||||
|
|
||||||
|
def _encrypted_zip_bytes(password: str) -> bytes:
|
||||||
|
target = io.BytesIO()
|
||||||
|
with pyzipper.AESZipFile(
|
||||||
|
target,
|
||||||
|
"w",
|
||||||
|
compression=zipfile.ZIP_DEFLATED,
|
||||||
|
encryption=pyzipper.WZ_AES,
|
||||||
|
) as archive:
|
||||||
|
archive.setpassword(password.encode("utf-8"))
|
||||||
|
archive.writestr("secure/report.txt", b"classified")
|
||||||
|
return target.getvalue()
|
||||||
|
|
||||||
|
|
||||||
|
class ArchiveInspectionTests(unittest.TestCase):
|
||||||
|
def test_zip_preview_derives_folders_and_sizes(self) -> None:
|
||||||
|
inspection = inspect_archive(
|
||||||
|
_zip_bytes(
|
||||||
|
{
|
||||||
|
"department/one.txt": b"one",
|
||||||
|
"department/nested/two.csv": b"two",
|
||||||
|
}
|
||||||
|
),
|
||||||
|
filename="monthly.zip",
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual("zip", inspection.archive_format)
|
||||||
|
self.assertEqual(2, inspection.file_count)
|
||||||
|
self.assertEqual(2, inspection.directory_count)
|
||||||
|
self.assertEqual(6, inspection.expanded_size_bytes)
|
||||||
|
self.assertEqual(
|
||||||
|
[
|
||||||
|
"department",
|
||||||
|
"department/nested",
|
||||||
|
"department/nested/two.csv",
|
||||||
|
"department/one.txt",
|
||||||
|
],
|
||||||
|
[entry.path for entry in inspection.entries],
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_tar_compression_variants_are_inspected(self) -> None:
|
||||||
|
for filename, mode in (
|
||||||
|
("monthly.tar", "w"),
|
||||||
|
("monthly.tar.gz", "w:gz"),
|
||||||
|
("monthly.tar.bz2", "w:bz2"),
|
||||||
|
("monthly.tar.xz", "w:xz"),
|
||||||
|
):
|
||||||
|
with self.subTest(filename=filename):
|
||||||
|
inspection = inspect_archive(
|
||||||
|
_tar_bytes({"report.txt": b"report"}, mode=mode),
|
||||||
|
filename=filename,
|
||||||
|
)
|
||||||
|
self.assertEqual(1, inspection.file_count)
|
||||||
|
self.assertEqual(filename.removeprefix("monthly."), inspection.archive_format)
|
||||||
|
|
||||||
|
def test_unsafe_member_path_is_rejected(self) -> None:
|
||||||
|
with self.assertRaisesRegex(FileStorageError, "Unsafe archive member"):
|
||||||
|
inspect_archive(
|
||||||
|
_zip_bytes({"../escape.txt": b"no"}),
|
||||||
|
filename="unsafe.zip",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_archive_bomb_ratio_is_rejected(self) -> None:
|
||||||
|
with self.assertRaisesRegex(FileStorageError, "expansion ratio"):
|
||||||
|
inspect_archive(
|
||||||
|
_zip_bytes({"zeros.bin": b"\0" * 50_000}),
|
||||||
|
filename="bomb.zip",
|
||||||
|
max_expansion_ratio=2,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_encrypted_zip_reports_and_verifies_password(self) -> None:
|
||||||
|
archive = _encrypted_zip_bytes("correct horse")
|
||||||
|
|
||||||
|
missing = inspect_archive(archive, filename="secure.zip")
|
||||||
|
self.assertTrue(missing.requires_password)
|
||||||
|
self.assertFalse(missing.password_verified)
|
||||||
|
|
||||||
|
verified = inspect_archive(
|
||||||
|
archive,
|
||||||
|
filename="secure.zip",
|
||||||
|
password="correct horse",
|
||||||
|
)
|
||||||
|
self.assertTrue(verified.requires_password)
|
||||||
|
self.assertTrue(verified.password_verified)
|
||||||
|
|
||||||
|
with self.assertRaisesRegex(ArchivePasswordError, "incorrect"):
|
||||||
|
inspect_archive(
|
||||||
|
archive,
|
||||||
|
filename="secure.zip",
|
||||||
|
password="wrong",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_folder_selection_extracts_only_descendants(self) -> None:
|
||||||
|
archive = _zip_bytes(
|
||||||
|
{
|
||||||
|
"selected/one.txt": b"one",
|
||||||
|
"selected/two.txt": b"two",
|
||||||
|
"other/three.txt": b"three",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
stored = SimpleNamespace(
|
||||||
|
asset=SimpleNamespace(id="asset"),
|
||||||
|
version=SimpleNamespace(id="version"),
|
||||||
|
blob=SimpleNamespace(id="blob"),
|
||||||
|
)
|
||||||
|
with patch(
|
||||||
|
"govoplan_files.backend.storage.archives.create_file_asset",
|
||||||
|
return_value=stored,
|
||||||
|
) as create_asset:
|
||||||
|
result = extract_archive_upload(
|
||||||
|
object(), # type: ignore[arg-type]
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
owner_type="user",
|
||||||
|
owner_id="user-1",
|
||||||
|
user_id="user-1",
|
||||||
|
archive_data=archive,
|
||||||
|
filename="selection.zip",
|
||||||
|
folder="imports",
|
||||||
|
campaign_id=None,
|
||||||
|
selected_paths=["selected"],
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(2, len(result))
|
||||||
|
self.assertEqual(
|
||||||
|
{"imports/selected/one.txt", "imports/selected/two.txt"},
|
||||||
|
{
|
||||||
|
call.kwargs["display_path"]
|
||||||
|
for call in create_asset.call_args_list
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,89 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from types import SimpleNamespace
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from govoplan_core.auth import ApiPrincipal
|
||||||
|
from govoplan_core.core.access import PrincipalRef
|
||||||
|
from govoplan_core.core.files import ManagedArtifactStore, ManagedArtifactWriteRequest
|
||||||
|
from govoplan_files.backend.capabilities import FilesArtifactStore
|
||||||
|
|
||||||
|
|
||||||
|
def principal() -> ApiPrincipal:
|
||||||
|
return ApiPrincipal(
|
||||||
|
principal=PrincipalRef(
|
||||||
|
account_id="account-1",
|
||||||
|
membership_id="membership-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
scopes=frozenset({"files:file:upload"}),
|
||||||
|
),
|
||||||
|
account=object(),
|
||||||
|
user=SimpleNamespace(id="user-1"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class _Session:
|
||||||
|
def query(self):
|
||||||
|
raise AssertionError("not used")
|
||||||
|
|
||||||
|
def flush(self):
|
||||||
|
raise AssertionError("not used")
|
||||||
|
|
||||||
|
|
||||||
|
def stored():
|
||||||
|
return SimpleNamespace(
|
||||||
|
asset=SimpleNamespace(
|
||||||
|
id="file-1",
|
||||||
|
display_path="Generated/Templates/result.html",
|
||||||
|
owner_type="user",
|
||||||
|
),
|
||||||
|
version=SimpleNamespace(
|
||||||
|
id="version-1",
|
||||||
|
filename_at_upload="result.html",
|
||||||
|
content_type="text/html",
|
||||||
|
size_bytes=6,
|
||||||
|
checksum_sha256="0" * 64,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class FilesArtifactStoreTests(unittest.TestCase):
|
||||||
|
def test_plain_write_uses_files_authority_and_returns_provider_neutral_ref(self) -> None:
|
||||||
|
service = FilesArtifactStore()
|
||||||
|
self.assertIsInstance(service, ManagedArtifactStore)
|
||||||
|
request = ManagedArtifactWriteRequest(
|
||||||
|
filename="result.html",
|
||||||
|
payload=b"result",
|
||||||
|
content_type="text/html",
|
||||||
|
folder="Generated/Templates",
|
||||||
|
metadata={"producer_module": "templates"},
|
||||||
|
)
|
||||||
|
with patch("govoplan_files.backend.capabilities.create_file_asset", return_value=stored()) as create:
|
||||||
|
result = service.store_artifact(_Session(), principal(), request=request)
|
||||||
|
self.assertEqual("file-1", result.file_asset_id)
|
||||||
|
self.assertEqual("version-1", result.file_version_id)
|
||||||
|
self.assertEqual("tenant-1", create.call_args.kwargs["tenant_id"])
|
||||||
|
self.assertEqual("user-1", create.call_args.kwargs["owner_id"])
|
||||||
|
self.assertEqual(b"result", create.call_args.kwargs["data"])
|
||||||
|
|
||||||
|
def test_idempotent_write_uses_source_provenance_without_payload_metadata(self) -> None:
|
||||||
|
request = ManagedArtifactWriteRequest(
|
||||||
|
filename="result.html",
|
||||||
|
payload=b"secret body",
|
||||||
|
content_type="text/html",
|
||||||
|
idempotency_key="render-1",
|
||||||
|
metadata={"producer_module": "templates", "output_sha256": "a" * 64},
|
||||||
|
)
|
||||||
|
with patch(
|
||||||
|
"govoplan_files.backend.capabilities.sync_file_asset_from_source",
|
||||||
|
return_value=(stored(), "unchanged", None),
|
||||||
|
) as sync:
|
||||||
|
FilesArtifactStore().store_artifact(_Session(), principal(), request=request)
|
||||||
|
metadata = sync.call_args.kwargs["metadata"]
|
||||||
|
self.assertEqual("render-1", metadata["source_provenance"]["external_id"])
|
||||||
|
self.assertNotIn("secret body", str(metadata))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,219 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from types import SimpleNamespace
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from govoplan_core.core.configuration_packages import (
|
||||||
|
ConfigurationPackageFragment,
|
||||||
|
ConfigurationPreflightContext,
|
||||||
|
)
|
||||||
|
from govoplan_core.core.infrastructure_capabilities import (
|
||||||
|
infrastructure_capability_receipt_from_mapping,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.configuration_provider import (
|
||||||
|
FILES_CONFIGURATION_CAPABILITY,
|
||||||
|
FilesConfigurationProvider,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.manifest import manifest
|
||||||
|
|
||||||
|
|
||||||
|
def _receipt(*, source: str = "host-local", state: str = "configured"):
|
||||||
|
endpoint = (
|
||||||
|
{"kind": "filesystem", "reference": "volume:files-data"}
|
||||||
|
if source == "host-local"
|
||||||
|
else {"scheme": "http", "host": "garage", "port": 3900}
|
||||||
|
)
|
||||||
|
secret_refs = (
|
||||||
|
[]
|
||||||
|
if source == "host-local"
|
||||||
|
else [
|
||||||
|
"env:FILE_STORAGE_S3_ACCESS_KEY_ID",
|
||||||
|
"env:FILE_STORAGE_S3_SECRET_ACCESS_KEY",
|
||||||
|
"env:GARAGE_RPC_SECRET",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
return infrastructure_capability_receipt_from_mapping(
|
||||||
|
{
|
||||||
|
"schema_version": 1,
|
||||||
|
"installation_id": "files-provider-test",
|
||||||
|
"profile": "evaluation",
|
||||||
|
"capabilities": [
|
||||||
|
{
|
||||||
|
"id": "files.storage",
|
||||||
|
"label": "Managed file content storage",
|
||||||
|
"state": state,
|
||||||
|
"source": source,
|
||||||
|
"detail": "Deployment-owned storage binding.",
|
||||||
|
"endpoint": endpoint,
|
||||||
|
"secret_refs": secret_refs,
|
||||||
|
"dependent_modules": ["files"],
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"post_install_tasks": [],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _local_settings(**overrides):
|
||||||
|
values = {
|
||||||
|
"file_storage_backend": "local",
|
||||||
|
"file_storage_local_root": "/var/lib/govoplan/files",
|
||||||
|
}
|
||||||
|
values.update(overrides)
|
||||||
|
return SimpleNamespace(**values)
|
||||||
|
|
||||||
|
|
||||||
|
def _s3_settings(**overrides):
|
||||||
|
values = {
|
||||||
|
"file_storage_backend": "s3",
|
||||||
|
"file_storage_s3_endpoint_url": "http://garage:3900",
|
||||||
|
"file_storage_s3_bucket": "files",
|
||||||
|
"file_storage_s3_deployment_managed": True,
|
||||||
|
"file_storage_s3_endpoint_trusted": False,
|
||||||
|
}
|
||||||
|
values.update(overrides)
|
||||||
|
return SimpleNamespace(**values)
|
||||||
|
|
||||||
|
|
||||||
|
class FilesConfigurationProviderTests(unittest.TestCase):
|
||||||
|
def test_provider_is_registered(self) -> None:
|
||||||
|
self.assertIn(FILES_CONFIGURATION_CAPABILITY, manifest.capability_factories)
|
||||||
|
|
||||||
|
def test_matching_local_storage_is_an_idempotent_noop(self) -> None:
|
||||||
|
provider = FilesConfigurationProvider(
|
||||||
|
settings=_local_settings(),
|
||||||
|
environment={},
|
||||||
|
)
|
||||||
|
context = ConfigurationPreflightContext(
|
||||||
|
infrastructure_receipt=_receipt(),
|
||||||
|
)
|
||||||
|
fragment = ConfigurationPackageFragment(
|
||||||
|
module_id="files",
|
||||||
|
fragment_type="managed_storage",
|
||||||
|
fragment_id="files-storage",
|
||||||
|
payload={
|
||||||
|
"expected_backend": "local",
|
||||||
|
"expected_source": "host-local",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
first = provider.preflight(fragment, context)
|
||||||
|
applied = provider.apply(fragment, {}, context)
|
||||||
|
second = provider.preflight(fragment, context)
|
||||||
|
|
||||||
|
self.assertEqual("skip", first.plan[0].action)
|
||||||
|
self.assertEqual("skip", second.plan[0].action)
|
||||||
|
self.assertEqual((), applied.diagnostics)
|
||||||
|
self.assertEqual({}, applied.created_refs)
|
||||||
|
self.assertEqual({}, applied.updated_refs)
|
||||||
|
|
||||||
|
def test_runtime_backend_mismatch_blocks_without_rewriting_settings(self) -> None:
|
||||||
|
settings = _local_settings()
|
||||||
|
provider = FilesConfigurationProvider(settings=settings, environment={})
|
||||||
|
context = ConfigurationPreflightContext(
|
||||||
|
infrastructure_receipt=_receipt(source="installer-managed-garage"),
|
||||||
|
)
|
||||||
|
fragment = ConfigurationPackageFragment(
|
||||||
|
module_id="files",
|
||||||
|
fragment_type="managed_storage",
|
||||||
|
payload={},
|
||||||
|
)
|
||||||
|
|
||||||
|
result = provider.preflight(fragment, context)
|
||||||
|
|
||||||
|
self.assertEqual("blocked", result.plan[0].action)
|
||||||
|
self.assertIn(
|
||||||
|
"files_storage_runtime_mismatch",
|
||||||
|
{item.code for item in result.diagnostics},
|
||||||
|
)
|
||||||
|
self.assertEqual("local", settings.file_storage_backend)
|
||||||
|
|
||||||
|
def test_garage_binding_requires_only_files_secret_references(self) -> None:
|
||||||
|
fragment = ConfigurationPackageFragment(
|
||||||
|
module_id="files",
|
||||||
|
fragment_type="managed_storage",
|
||||||
|
payload={},
|
||||||
|
)
|
||||||
|
context = ConfigurationPreflightContext(
|
||||||
|
infrastructure_receipt=_receipt(source="installer-managed-garage"),
|
||||||
|
)
|
||||||
|
missing = FilesConfigurationProvider(
|
||||||
|
settings=_s3_settings(),
|
||||||
|
environment={},
|
||||||
|
).preflight(fragment, context)
|
||||||
|
available = FilesConfigurationProvider(
|
||||||
|
settings=_s3_settings(),
|
||||||
|
environment={
|
||||||
|
"FILE_STORAGE_S3_ACCESS_KEY_ID": "reference-resolved",
|
||||||
|
"FILE_STORAGE_S3_SECRET_ACCESS_KEY": "reference-resolved",
|
||||||
|
},
|
||||||
|
).preflight(fragment, context)
|
||||||
|
|
||||||
|
self.assertEqual("blocked", missing.plan[0].action)
|
||||||
|
self.assertEqual(
|
||||||
|
2,
|
||||||
|
sum(
|
||||||
|
item.code == "files_storage_secret_reference_unresolved"
|
||||||
|
for item in missing.diagnostics
|
||||||
|
),
|
||||||
|
)
|
||||||
|
self.assertEqual("skip", available.plan[0].action)
|
||||||
|
|
||||||
|
def test_operator_supplied_s3_requires_explicit_endpoint_trust(self) -> None:
|
||||||
|
receipt = _receipt(source="operator-supplied-s3", state="externally_supplied")
|
||||||
|
fragment = ConfigurationPackageFragment(
|
||||||
|
module_id="files",
|
||||||
|
fragment_type="managed_storage",
|
||||||
|
payload={},
|
||||||
|
)
|
||||||
|
context = ConfigurationPreflightContext(infrastructure_receipt=receipt)
|
||||||
|
environment = {
|
||||||
|
"FILE_STORAGE_S3_ACCESS_KEY_ID": "reference-resolved",
|
||||||
|
"FILE_STORAGE_S3_SECRET_ACCESS_KEY": "reference-resolved",
|
||||||
|
}
|
||||||
|
|
||||||
|
blocked = FilesConfigurationProvider(
|
||||||
|
settings=_s3_settings(
|
||||||
|
file_storage_s3_deployment_managed=False,
|
||||||
|
file_storage_s3_endpoint_trusted=False,
|
||||||
|
),
|
||||||
|
environment=environment,
|
||||||
|
).preflight(fragment, context)
|
||||||
|
ready = FilesConfigurationProvider(
|
||||||
|
settings=_s3_settings(
|
||||||
|
file_storage_s3_deployment_managed=False,
|
||||||
|
file_storage_s3_endpoint_trusted=True,
|
||||||
|
),
|
||||||
|
environment=environment,
|
||||||
|
).preflight(fragment, context)
|
||||||
|
|
||||||
|
self.assertEqual("blocked", blocked.plan[0].action)
|
||||||
|
self.assertIn(
|
||||||
|
"files_storage_trust_boundary_missing",
|
||||||
|
{item.code for item in blocked.diagnostics},
|
||||||
|
)
|
||||||
|
self.assertEqual("skip", ready.plan[0].action)
|
||||||
|
|
||||||
|
def test_inline_storage_secret_is_rejected(self) -> None:
|
||||||
|
provider = FilesConfigurationProvider(
|
||||||
|
settings=_local_settings(),
|
||||||
|
environment={},
|
||||||
|
)
|
||||||
|
result = provider.preflight(
|
||||||
|
ConfigurationPackageFragment(
|
||||||
|
module_id="files",
|
||||||
|
fragment_type="managed_storage",
|
||||||
|
payload={"secret_access_key": "inline"},
|
||||||
|
),
|
||||||
|
ConfigurationPreflightContext(infrastructure_receipt=_receipt()),
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual("blocked", result.plan[0].action)
|
||||||
|
self.assertIn(
|
||||||
|
"files_configuration_secret_forbidden",
|
||||||
|
{item.code for item in result.diagnostics},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -12,7 +12,8 @@ from govoplan_core.core.change_sequence import ChangeSequenceEntry
|
|||||||
from govoplan_core.db.base import Base
|
from govoplan_core.db.base import Base
|
||||||
from govoplan_core.security.secrets import encrypt_secret
|
from govoplan_core.security.secrets import encrypt_secret
|
||||||
from govoplan_files.backend.db.models import FileConnectorCredential, FileConnectorProfile
|
from govoplan_files.backend.db.models import FileConnectorCredential, FileConnectorProfile
|
||||||
from govoplan_files.backend.router import deactivate_connector_credential, deactivate_connector_profile
|
from govoplan_files.backend.routes.connector_profiles import deactivate_connector_profile
|
||||||
|
from govoplan_files.backend.routes.connector_settings import deactivate_connector_credential
|
||||||
|
|
||||||
|
|
||||||
class Principal:
|
class Principal:
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ from unittest.mock import MagicMock
|
|||||||
|
|
||||||
from fastapi import HTTPException
|
from fastapi import HTTPException
|
||||||
|
|
||||||
from govoplan_files.backend.router import discover_connector_endpoint
|
from govoplan_files.backend.routes.connector_settings import discover_connector_endpoint
|
||||||
from govoplan_files.backend.schemas import FileConnectorDiscoveryRequest
|
from govoplan_files.backend.schemas import FileConnectorDiscoveryRequest
|
||||||
from govoplan_files.backend.storage.connector_browse import ConnectorBrowseError, _profile_password, _s3_verify
|
from govoplan_files.backend.storage.connector_browse import ConnectorBrowseError, _profile_password, _s3_verify
|
||||||
from govoplan_files.backend.storage.connector_deployment import (
|
from govoplan_files.backend.storage.connector_deployment import (
|
||||||
@@ -182,7 +182,7 @@ class ConnectorDiscoveryBoundaryTests(unittest.TestCase):
|
|||||||
credential_mode="basic",
|
credential_mode="basic",
|
||||||
credentials={"username": "admin", "password_env": "MASTER_KEY_B64"},
|
credentials={"username": "admin", "password_env": "MASTER_KEY_B64"},
|
||||||
)
|
)
|
||||||
with patch("govoplan_files.backend.router.browse_connector_profile") as browse, self.assertRaises(
|
with patch("govoplan_files.backend.routes.connector_settings.browse_connector_profile") as browse, self.assertRaises(
|
||||||
HTTPException
|
HTTPException
|
||||||
) as raised:
|
) as raised:
|
||||||
discover_connector_endpoint(payload, session=object(), principal=self._principal()) # type: ignore[arg-type]
|
discover_connector_endpoint(payload, session=object(), principal=self._principal()) # type: ignore[arg-type]
|
||||||
@@ -212,10 +212,10 @@ class ConnectorDiscoveryBoundaryTests(unittest.TestCase):
|
|||||||
events.append("io")
|
events.append("io")
|
||||||
return []
|
return []
|
||||||
|
|
||||||
with patch("govoplan_files.backend.router._ensure_connector_configuration_allowed", side_effect=ensure), patch(
|
with patch("govoplan_files.backend.routes.connector_settings._ensure_connector_configuration_allowed", side_effect=ensure), patch(
|
||||||
"govoplan_files.backend.router._audit_connector_discovery_attempt",
|
"govoplan_files.backend.routes.connector_settings._audit_connector_discovery_attempt",
|
||||||
side_effect=audit,
|
side_effect=audit,
|
||||||
), patch("govoplan_files.backend.router.browse_connector_profile", side_effect=browse):
|
), patch("govoplan_files.backend.routes.connector_settings.browse_connector_profile", side_effect=browse):
|
||||||
response = discover_connector_endpoint(payload, session=object(), principal=self._principal()) # type: ignore[arg-type]
|
response = discover_connector_endpoint(payload, session=object(), principal=self._principal()) # type: ignore[arg-type]
|
||||||
|
|
||||||
self.assertEqual("usable", response.status)
|
self.assertEqual("usable", response.status)
|
||||||
|
|||||||
@@ -0,0 +1,303 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import unittest
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from types import SimpleNamespace
|
||||||
|
from unittest.mock import MagicMock, Mock, call, patch
|
||||||
|
|
||||||
|
from govoplan_files.backend.routes.connector_io import sync_connector_space_folder
|
||||||
|
from govoplan_files.backend.schemas import (
|
||||||
|
FileAssetResponse,
|
||||||
|
FileConnectorFolderSyncRequest,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.storage.common import FileStorageError, UploadedStoredFile
|
||||||
|
from govoplan_files.backend.storage.connector_browse import ConnectorBrowseItem
|
||||||
|
from govoplan_files.backend.storage.connector_folder_sync import (
|
||||||
|
connector_relative_path,
|
||||||
|
discover_connector_folder,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.storage.connector_imports import ConnectorDownloadedFile
|
||||||
|
from govoplan_files.backend.storage.connector_policy import ConnectorPolicyDenied
|
||||||
|
from govoplan_files.backend.storage.connector_profiles import ConnectorProfile
|
||||||
|
|
||||||
|
|
||||||
|
def _item(kind: str, path: str, *, token: str | None = None) -> ConnectorBrowseItem:
|
||||||
|
return ConnectorBrowseItem(
|
||||||
|
kind=kind,
|
||||||
|
name=path.rsplit("/", 1)[-1],
|
||||||
|
path=path,
|
||||||
|
external_id=f"external:{path}",
|
||||||
|
etag=f"etag:{path}",
|
||||||
|
metadata={"next_continuation_token": token} if token else {},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorFolderDiscoveryTests(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
self.profile = ConnectorProfile(
|
||||||
|
id="profile-1",
|
||||||
|
label="Documents",
|
||||||
|
provider="webdav",
|
||||||
|
capabilities=("browse", "sync"),
|
||||||
|
)
|
||||||
|
|
||||||
|
@patch("govoplan_files.backend.storage.connector_folder_sync.browse_connector_profile")
|
||||||
|
def test_recursively_discovers_files_across_pages_in_deterministic_order(
|
||||||
|
self, browse: Mock
|
||||||
|
) -> None:
|
||||||
|
browse.side_effect = [
|
||||||
|
[_item("file", "root/z.txt", token="page-2")],
|
||||||
|
[_item("folder", "root/nested"), _item("file", "root/a.txt")],
|
||||||
|
[_item("file", "root/nested/b.txt")],
|
||||||
|
]
|
||||||
|
|
||||||
|
result = discover_connector_folder(
|
||||||
|
self.profile,
|
||||||
|
path="root",
|
||||||
|
library_id="library-1",
|
||||||
|
recursive=True,
|
||||||
|
max_files=10,
|
||||||
|
max_depth=3,
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(
|
||||||
|
["root/a.txt", "root/nested/b.txt", "root/z.txt"],
|
||||||
|
[item.path for item in result.files],
|
||||||
|
)
|
||||||
|
self.assertFalse(result.truncated)
|
||||||
|
self.assertEqual([], result.skipped)
|
||||||
|
self.assertEqual(
|
||||||
|
[
|
||||||
|
call(
|
||||||
|
self.profile,
|
||||||
|
path="root",
|
||||||
|
library_id="library-1",
|
||||||
|
continuation_token=None,
|
||||||
|
),
|
||||||
|
call(
|
||||||
|
self.profile,
|
||||||
|
path="root",
|
||||||
|
library_id="library-1",
|
||||||
|
continuation_token="page-2",
|
||||||
|
),
|
||||||
|
call(
|
||||||
|
self.profile,
|
||||||
|
path="root/nested",
|
||||||
|
library_id="library-1",
|
||||||
|
continuation_token=None,
|
||||||
|
),
|
||||||
|
],
|
||||||
|
browse.call_args_list,
|
||||||
|
)
|
||||||
|
|
||||||
|
@patch("govoplan_files.backend.storage.connector_folder_sync.browse_connector_profile")
|
||||||
|
def test_non_recursive_and_file_limit_skips_are_explicit(self, browse: Mock) -> None:
|
||||||
|
browse.return_value = [
|
||||||
|
_item("folder", "root/nested"),
|
||||||
|
_item("file", "root/a.txt"),
|
||||||
|
_item("file", "root/b.txt"),
|
||||||
|
]
|
||||||
|
|
||||||
|
result = discover_connector_folder(
|
||||||
|
self.profile,
|
||||||
|
path="root",
|
||||||
|
library_id=None,
|
||||||
|
recursive=False,
|
||||||
|
max_files=1,
|
||||||
|
max_depth=0,
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(["root/a.txt"], [item.path for item in result.files])
|
||||||
|
self.assertTrue(result.truncated)
|
||||||
|
self.assertEqual(
|
||||||
|
["root/nested", "root/b.txt"],
|
||||||
|
[item.path for item in result.skipped],
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_relative_path_rejects_provider_results_outside_linked_root(self) -> None:
|
||||||
|
self.assertEqual(
|
||||||
|
"nested/report.csv",
|
||||||
|
connector_relative_path(
|
||||||
|
space_root="linked/root", item_path="linked/root/nested/report.csv"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
with self.assertRaisesRegex(ValueError, "outside"):
|
||||||
|
connector_relative_path(
|
||||||
|
space_root="linked/root", item_path="another/root/report.csv"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorFolderSyncRouteTests(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
self.session = MagicMock()
|
||||||
|
self.space = SimpleNamespace(
|
||||||
|
id="space-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
owner_type="user",
|
||||||
|
owner_user_id="user-1",
|
||||||
|
owner_group_id=None,
|
||||||
|
connector_profile_id="profile-1",
|
||||||
|
provider="webdav",
|
||||||
|
library_id="library-1",
|
||||||
|
remote_path="linked/root",
|
||||||
|
sync_mode="manual",
|
||||||
|
)
|
||||||
|
self.profile = ConnectorProfile(
|
||||||
|
id="profile-1",
|
||||||
|
label="Documents",
|
||||||
|
provider="webdav",
|
||||||
|
capabilities=("browse", "sync"),
|
||||||
|
)
|
||||||
|
self.principal = SimpleNamespace(
|
||||||
|
tenant_id="tenant-1", user=SimpleNamespace(id="user-1")
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _stored(index: int) -> UploadedStoredFile:
|
||||||
|
return UploadedStoredFile(
|
||||||
|
asset=SimpleNamespace(id=f"asset-{index}"),
|
||||||
|
version=SimpleNamespace(id=f"version-{index}"),
|
||||||
|
blob=SimpleNamespace(id=f"blob-{index}"),
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _response(index: int, action: str) -> FileAssetResponse:
|
||||||
|
now = datetime(2026, 8, 21, tzinfo=UTC).isoformat()
|
||||||
|
return FileAssetResponse(
|
||||||
|
id=f"asset-{index}",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
owner_type="user",
|
||||||
|
owner_id="user-1",
|
||||||
|
display_path=f"Imports/file-{index}.txt",
|
||||||
|
filename=f"file-{index}.txt",
|
||||||
|
size_bytes=index,
|
||||||
|
checksum_sha256=str(index) * 64,
|
||||||
|
version_id=f"version-{index}",
|
||||||
|
created_at=now,
|
||||||
|
updated_at=now,
|
||||||
|
source_revision=f"revision-{action}",
|
||||||
|
)
|
||||||
|
|
||||||
|
@patch("govoplan_files.backend.routes.connector_io.audit_from_principal")
|
||||||
|
@patch("govoplan_files.backend.routes.connector_io._asset_response")
|
||||||
|
@patch("govoplan_files.backend.routes.connector_io._audit_connector_sync")
|
||||||
|
@patch("govoplan_files.backend.routes.connector_io.sync_file_asset_from_source")
|
||||||
|
@patch("govoplan_files.backend.routes.connector_io._download_connector_payload")
|
||||||
|
@patch("govoplan_files.backend.routes.connector_io.discover_connector_folder")
|
||||||
|
@patch("govoplan_files.backend.routes.connector_io._connector_space_policy_decision")
|
||||||
|
@patch("govoplan_files.backend.routes.connector_io._visible_connector_profile")
|
||||||
|
@patch("govoplan_files.backend.routes.connector_io.get_connector_space_for_user")
|
||||||
|
@patch("govoplan_files.backend.routes.connector_io._is_admin", return_value=False)
|
||||||
|
def test_returns_reviewable_per_item_results_and_audits_the_summary(
|
||||||
|
self,
|
||||||
|
_is_admin: Mock,
|
||||||
|
get_space: Mock,
|
||||||
|
visible_profile: Mock,
|
||||||
|
policy_decision: Mock,
|
||||||
|
discover: Mock,
|
||||||
|
download: Mock,
|
||||||
|
sync_file: Mock,
|
||||||
|
audit_sync: Mock,
|
||||||
|
asset_response: Mock,
|
||||||
|
audit: Mock,
|
||||||
|
) -> None:
|
||||||
|
get_space.return_value = self.space
|
||||||
|
visible_profile.return_value = self.profile
|
||||||
|
policy_decision.return_value = SimpleNamespace(allowed=True)
|
||||||
|
sources = [_item("file", f"linked/root/file-{index}.txt") for index in range(1, 6)]
|
||||||
|
discover.return_value = SimpleNamespace(
|
||||||
|
files=sources,
|
||||||
|
skipped=[SimpleNamespace(path="linked/root/ignored", reason="Unsupported item")],
|
||||||
|
truncated=False,
|
||||||
|
)
|
||||||
|
downloads = [
|
||||||
|
(
|
||||||
|
source.path,
|
||||||
|
ConnectorDownloadedFile(
|
||||||
|
filename=source.name,
|
||||||
|
data=source.name.encode(),
|
||||||
|
revision=source.etag,
|
||||||
|
),
|
||||||
|
{"source_provenance": {"connector_id": "profile-1"}},
|
||||||
|
)
|
||||||
|
for source in sources
|
||||||
|
]
|
||||||
|
denied_decision = SimpleNamespace(
|
||||||
|
reason="Path denied",
|
||||||
|
to_dict=Mock(return_value={"allowed": False, "reason": "Path denied"}),
|
||||||
|
)
|
||||||
|
downloads[-1] = ConnectorPolicyDenied(denied_decision) # type: ignore[assignment]
|
||||||
|
download.side_effect = downloads
|
||||||
|
sync_file.side_effect = [
|
||||||
|
(self._stored(1), "created", None),
|
||||||
|
(self._stored(2), "updated", "previous-2"),
|
||||||
|
(self._stored(3), "unchanged", "previous-3"),
|
||||||
|
FileStorageError("Target file already exists: Imports/file-4.txt"),
|
||||||
|
]
|
||||||
|
asset_response.side_effect = [
|
||||||
|
self._response(1, "created"),
|
||||||
|
self._response(2, "updated"),
|
||||||
|
self._response(3, "unchanged"),
|
||||||
|
]
|
||||||
|
|
||||||
|
response = sync_connector_space_folder(
|
||||||
|
"space-1",
|
||||||
|
FileConnectorFolderSyncRequest(target_folder="Imports"),
|
||||||
|
session=self.session,
|
||||||
|
principal=self.principal, # type: ignore[arg-type]
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(
|
||||||
|
{
|
||||||
|
"discovered": 5,
|
||||||
|
"created": 1,
|
||||||
|
"updated": 1,
|
||||||
|
"unchanged": 1,
|
||||||
|
"skipped": 1,
|
||||||
|
"conflicts": 1,
|
||||||
|
"policy_denied": 1,
|
||||||
|
"failed": 0,
|
||||||
|
},
|
||||||
|
response.summary.model_dump(),
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
[
|
||||||
|
"skipped",
|
||||||
|
"created",
|
||||||
|
"updated",
|
||||||
|
"unchanged",
|
||||||
|
"conflict",
|
||||||
|
"policy_denied",
|
||||||
|
],
|
||||||
|
[item.action for item in response.items],
|
||||||
|
)
|
||||||
|
self.assertEqual("Imports/file-1.txt", response.items[1].target_path)
|
||||||
|
self.assertEqual(
|
||||||
|
{"allowed": False, "reason": "Path denied"},
|
||||||
|
response.items[-1].policy_decision,
|
||||||
|
)
|
||||||
|
self.session.commit.assert_called_once_with()
|
||||||
|
audit_sync.assert_has_calls(
|
||||||
|
[
|
||||||
|
call(
|
||||||
|
self.session,
|
||||||
|
self.principal,
|
||||||
|
self._stored(index).asset,
|
||||||
|
sync_action=action,
|
||||||
|
previous_version_id=previous,
|
||||||
|
)
|
||||||
|
for index, action, previous in (
|
||||||
|
(1, "created", None),
|
||||||
|
(2, "updated", "previous-2"),
|
||||||
|
(3, "unchanged", "previous-3"),
|
||||||
|
)
|
||||||
|
]
|
||||||
|
)
|
||||||
|
summary_details = audit.call_args.kwargs["details"]
|
||||||
|
self.assertEqual(response.summary.model_dump(), summary_details["summary"])
|
||||||
|
self.assertEqual(6, len(summary_details["results"]))
|
||||||
|
self.assertNotIn("detail", summary_details["results"][-1])
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,66 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from govoplan_files.backend.storage.connector_policy import (
|
||||||
|
ConnectorAccessRequest,
|
||||||
|
ConnectorPolicySource,
|
||||||
|
connector_policy_decision,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorPolicyPatternTests(unittest.TestCase):
|
||||||
|
def test_resource_reference_allow_patterns_match(self) -> None:
|
||||||
|
decision = connector_policy_decision(
|
||||||
|
ConnectorAccessRequest(
|
||||||
|
connector_id="dev-smb",
|
||||||
|
credential_id="credential-primary",
|
||||||
|
provider="smb",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
ConnectorPolicySource(
|
||||||
|
scope_type="tenant",
|
||||||
|
scope_id="tenant-1",
|
||||||
|
label="Tenant",
|
||||||
|
policy={
|
||||||
|
"allow": {
|
||||||
|
"connectors": ["dev-*"],
|
||||||
|
"credentials": ["credential-*"],
|
||||||
|
"providers": ["s?b"],
|
||||||
|
}
|
||||||
|
},
|
||||||
|
),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertTrue(decision.allowed)
|
||||||
|
|
||||||
|
def test_resource_reference_deny_patterns_take_precedence(self) -> None:
|
||||||
|
decision = connector_policy_decision(
|
||||||
|
ConnectorAccessRequest(
|
||||||
|
connector_id="dev-smb",
|
||||||
|
credential_id="credential-legacy",
|
||||||
|
provider="smb",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
ConnectorPolicySource(
|
||||||
|
scope_type="tenant",
|
||||||
|
scope_id="tenant-1",
|
||||||
|
label="Tenant",
|
||||||
|
policy={
|
||||||
|
"allow": {"connectors": ["dev-*"]},
|
||||||
|
"deny": {"credentials": ["*-legacy"]},
|
||||||
|
},
|
||||||
|
),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertFalse(decision.allowed)
|
||||||
|
self.assertEqual(
|
||||||
|
("connector_policy_denylist",),
|
||||||
|
decision.requirements,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -1,5 +1,6 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
import unittest
|
import unittest
|
||||||
from unittest.mock import patch
|
from unittest.mock import patch
|
||||||
|
|
||||||
@@ -8,6 +9,7 @@ from sqlalchemy.orm import sessionmaker
|
|||||||
|
|
||||||
from govoplan_access.backend.db import models as access_models # noqa: F401 - resolve Files user foreign keys
|
from govoplan_access.backend.db import models as access_models # noqa: F401 - resolve Files user foreign keys
|
||||||
from govoplan_core.db.base import Base
|
from govoplan_core.db.base import Base
|
||||||
|
from govoplan_core.security.credential_envelopes import CredentialEnvelope
|
||||||
from govoplan_core.security.secrets import decrypt_secret, encrypt_secret
|
from govoplan_core.security.secrets import decrypt_secret, encrypt_secret
|
||||||
from govoplan_files.backend.db.models import FileConnectorProfile
|
from govoplan_files.backend.db.models import FileConnectorProfile
|
||||||
from govoplan_files.backend.storage.common import FileStorageError
|
from govoplan_files.backend.storage.common import FileStorageError
|
||||||
@@ -20,7 +22,13 @@ from govoplan_files.backend.storage.connector_profile_store import (
|
|||||||
class ConnectorProfileStoreUpdateTests(unittest.TestCase):
|
class ConnectorProfileStoreUpdateTests(unittest.TestCase):
|
||||||
def setUp(self) -> None:
|
def setUp(self) -> None:
|
||||||
self.engine = create_engine("sqlite:///:memory:")
|
self.engine = create_engine("sqlite:///:memory:")
|
||||||
Base.metadata.create_all(self.engine, tables=[FileConnectorProfile.__table__])
|
Base.metadata.create_all(
|
||||||
|
self.engine,
|
||||||
|
tables=[
|
||||||
|
FileConnectorProfile.__table__,
|
||||||
|
CredentialEnvelope.__table__,
|
||||||
|
],
|
||||||
|
)
|
||||||
self.session = sessionmaker(bind=self.engine)()
|
self.session = sessionmaker(bind=self.engine)()
|
||||||
self.row = FileConnectorProfile(
|
self.row = FileConnectorProfile(
|
||||||
id="profile-1",
|
id="profile-1",
|
||||||
@@ -171,6 +179,50 @@ class ConnectorProfileStoreUpdateTests(unittest.TestCase):
|
|||||||
self.assertEqual([self.row.id], [profile.id for profile in profiles])
|
self.assertEqual([self.row.id], [profile.id for profile in profiles])
|
||||||
self.assertEqual("original-password", profiles[0].password_value)
|
self.assertEqual("original-password", profiles[0].password_value)
|
||||||
|
|
||||||
|
def test_reusable_credential_resolves_and_revocation_keeps_profile_visible(self) -> None:
|
||||||
|
credential = CredentialEnvelope(
|
||||||
|
id="shared-credential",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
scope_type="tenant",
|
||||||
|
scope_id="tenant-1",
|
||||||
|
name="Shared WebDAV login",
|
||||||
|
credential_kind="username_password",
|
||||||
|
public_data={"username": "ada"},
|
||||||
|
secret_data_encrypted=encrypt_secret(json.dumps({"password": "secret"})),
|
||||||
|
secret_keys=["password"],
|
||||||
|
allowed_modules=["files"],
|
||||||
|
allowed_server_refs=[],
|
||||||
|
inherit_to_lower_scopes=True,
|
||||||
|
is_active=True,
|
||||||
|
revision="revision-1",
|
||||||
|
)
|
||||||
|
self.row.credential_profile_id = "credential-envelope:shared-credential"
|
||||||
|
self.row.username = None
|
||||||
|
self.row.password_encrypted = None
|
||||||
|
self.row.token_encrypted = None
|
||||||
|
self.session.add(credential)
|
||||||
|
self.session.commit()
|
||||||
|
|
||||||
|
profile = list_database_connector_profiles(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
)[0]
|
||||||
|
|
||||||
|
self.assertEqual("ada", profile.username)
|
||||||
|
self.assertEqual("secret", profile.password_value)
|
||||||
|
self.assertTrue(profile.credentials_configured)
|
||||||
|
|
||||||
|
credential.is_active = False
|
||||||
|
self.session.commit()
|
||||||
|
profile = list_database_connector_profiles(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
)[0]
|
||||||
|
|
||||||
|
self.assertIsNone(profile.password_value)
|
||||||
|
self.assertFalse(profile.credentials_configured)
|
||||||
|
self.assertTrue(profile.metadata["credential_unavailable"])
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ from datetime import UTC, datetime
|
|||||||
from unittest.mock import patch
|
from unittest.mock import patch
|
||||||
|
|
||||||
from govoplan_files.backend.storage.connector_browse import ConnectorBrowseError, _smb_location, browse_connector_profile
|
from govoplan_files.backend.storage.connector_browse import ConnectorBrowseError, _smb_location, browse_connector_profile
|
||||||
from govoplan_files.backend.storage.connector_imports import ConnectorImportError, read_connector_file
|
from govoplan_files.backend.storage.connector_imports import read_connector_file
|
||||||
from govoplan_files.backend.storage.connector_profiles import ConnectorProfile, connector_profiles_from_payload
|
from govoplan_files.backend.storage.connector_profiles import ConnectorProfile, connector_profiles_from_payload
|
||||||
from govoplan_files.backend.storage.connector_providers import connector_provider_descriptors
|
from govoplan_files.backend.storage.connector_providers import connector_provider_descriptors
|
||||||
|
|
||||||
@@ -23,6 +23,10 @@ class FakeS3Client:
|
|||||||
self.list_objects_request: dict[str, object] | None = None
|
self.list_objects_request: dict[str, object] | None = None
|
||||||
self.head_request: dict[str, object] | None = None
|
self.head_request: dict[str, object] | None = None
|
||||||
self.get_request: dict[str, object] | None = None
|
self.get_request: dict[str, object] | None = None
|
||||||
|
self.closed = False
|
||||||
|
|
||||||
|
def close(self) -> None:
|
||||||
|
self.closed = True
|
||||||
|
|
||||||
def list_objects_v2(self, **kwargs: object) -> dict[str, object]:
|
def list_objects_v2(self, **kwargs: object) -> dict[str, object]:
|
||||||
self.list_objects_request = dict(kwargs)
|
self.list_objects_request = dict(kwargs)
|
||||||
@@ -75,42 +79,39 @@ def s3_profile(**overrides: object) -> ConnectorProfile:
|
|||||||
|
|
||||||
|
|
||||||
class ConnectorProviderTests(unittest.TestCase):
|
class ConnectorProviderTests(unittest.TestCase):
|
||||||
def test_smb_sdk_transport_fails_closed_before_client_creation_in_all_modes(self) -> None:
|
def test_smb_browse_uses_the_files_owned_pinned_connection_cache(self) -> None:
|
||||||
profile = ConnectorProfile(
|
profile = ConnectorProfile(
|
||||||
id="smb",
|
id="smb",
|
||||||
label="SMB",
|
label="SMB",
|
||||||
provider="smb",
|
provider="smb",
|
||||||
endpoint_url="smb://files.example.test/share",
|
endpoint_url="smb://files.example.test/share",
|
||||||
)
|
)
|
||||||
for allow_private, address in ((False, "93.184.216.34"), (True, "10.0.0.5")):
|
with patch.dict(
|
||||||
with self.subTest(allow_private=allow_private), patch.dict(
|
"os.environ",
|
||||||
"os.environ",
|
{"APP_ENV": "production", "GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS": "false"},
|
||||||
{
|
), patch(
|
||||||
"APP_ENV": "production",
|
"govoplan_core.security.outbound_http.socket.getaddrinfo",
|
||||||
"GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS": str(allow_private).lower(),
|
return_value=[(2, 1, 6, "", ("93.184.216.34", 445))],
|
||||||
},
|
), patch("govoplan_files.backend.storage.connector_browse._smbclient_module") as sdk:
|
||||||
), patch(
|
sdk.return_value.scandir.return_value.__enter__.return_value = iter(())
|
||||||
"govoplan_core.security.outbound_http.socket.getaddrinfo",
|
self.assertEqual([], browse_connector_profile(profile, path=""))
|
||||||
return_value=[(2, 1, 6, "", (address, 445))],
|
|
||||||
), patch("govoplan_files.backend.storage.connector_browse._smbclient_module") as sdk, self.assertRaisesRegex(
|
|
||||||
ConnectorBrowseError,
|
|
||||||
"redirects/referrals.*DNS/IP pinning",
|
|
||||||
):
|
|
||||||
browse_connector_profile(profile, path="")
|
|
||||||
sdk.assert_not_called()
|
|
||||||
|
|
||||||
def test_smb_explicit_ip_still_fails_closed_because_the_sdk_may_follow_referrals(self) -> None:
|
kwargs = sdk.return_value.scandir.call_args.kwargs
|
||||||
|
self.assertIsInstance(kwargs["connection_cache"], dict)
|
||||||
|
self.assertTrue(kwargs["require_signing"])
|
||||||
|
|
||||||
|
def test_smb_endpoint_preflight_applies_private_network_policy(self) -> None:
|
||||||
profile = ConnectorProfile(id="smb", label="SMB", provider="smb", endpoint_url="smb://10.0.0.5/share")
|
profile = ConnectorProfile(id="smb", label="SMB", provider="smb", endpoint_url="smb://10.0.0.5/share")
|
||||||
with patch.dict(
|
with patch.dict(
|
||||||
"os.environ",
|
"os.environ",
|
||||||
{"APP_ENV": "production", "GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS": "true"},
|
{"APP_ENV": "production", "GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS": "false"},
|
||||||
), patch(
|
), patch(
|
||||||
"govoplan_core.security.outbound_http.socket.getaddrinfo",
|
"govoplan_core.security.outbound_http.socket.getaddrinfo",
|
||||||
return_value=[(2, 1, 6, "", ("10.0.0.5", 445))],
|
return_value=[(2, 1, 6, "", ("10.0.0.5", 445))],
|
||||||
), self.assertRaisesRegex(ConnectorBrowseError, "redirects/referrals.*DNS/IP pinning"):
|
), self.assertRaisesRegex(ConnectorBrowseError, "non-public network"):
|
||||||
_smb_location(profile)
|
_smb_location(profile)
|
||||||
|
|
||||||
def test_smb_import_surfaces_fail_closed_policy_as_an_import_error(self) -> None:
|
def test_smb_import_uses_the_same_pinned_connection_cache(self) -> None:
|
||||||
profile = ConnectorProfile(id="smb", label="SMB", provider="smb", endpoint_url="smb://10.0.0.5/share")
|
profile = ConnectorProfile(id="smb", label="SMB", provider="smb", endpoint_url="smb://10.0.0.5/share")
|
||||||
with patch.dict(
|
with patch.dict(
|
||||||
"os.environ",
|
"os.environ",
|
||||||
@@ -118,12 +119,13 @@ class ConnectorProviderTests(unittest.TestCase):
|
|||||||
), patch(
|
), patch(
|
||||||
"govoplan_core.security.outbound_http.socket.getaddrinfo",
|
"govoplan_core.security.outbound_http.socket.getaddrinfo",
|
||||||
return_value=[(2, 1, 6, "", ("10.0.0.5", 445))],
|
return_value=[(2, 1, 6, "", ("10.0.0.5", 445))],
|
||||||
), patch("govoplan_files.backend.storage.connector_imports._smbclient_module") as sdk, self.assertRaisesRegex(
|
), patch("govoplan_files.backend.storage.connector_imports._smbclient_module") as sdk:
|
||||||
ConnectorImportError,
|
sdk.return_value.stat.return_value.st_size = 4
|
||||||
"redirects/referrals.*DNS/IP pinning",
|
sdk.return_value.open_file.return_value.__enter__.return_value.read.return_value = b"test"
|
||||||
):
|
downloaded = read_connector_file(profile, library_id="", path="notice.txt", max_bytes=1024)
|
||||||
read_connector_file(profile, library_id="", path="notice.txt", max_bytes=1024)
|
|
||||||
sdk.assert_not_called()
|
self.assertEqual(b"test", downloaded.data)
|
||||||
|
self.assertIsInstance(sdk.return_value.stat.call_args.kwargs["connection_cache"], dict)
|
||||||
|
|
||||||
def test_provider_descriptors_include_s3_and_reserved_microsoft_providers(self) -> None:
|
def test_provider_descriptors_include_s3_and_reserved_microsoft_providers(self) -> None:
|
||||||
descriptors = {descriptor.provider: descriptor for descriptor in connector_provider_descriptors()}
|
descriptors = {descriptor.provider: descriptor for descriptor in connector_provider_descriptors()}
|
||||||
@@ -172,6 +174,7 @@ class ConnectorProviderTests(unittest.TestCase):
|
|||||||
self.assertEqual("govoplan:root/report.xlsx", items[1].external_id)
|
self.assertEqual("govoplan:root/report.xlsx", items[1].external_id)
|
||||||
self.assertEqual("root/report.xlsx", items[1].metadata["key"])
|
self.assertEqual("root/report.xlsx", items[1].metadata["key"])
|
||||||
self.assertEqual("next-page", items[1].metadata["next_continuation_token"])
|
self.assertEqual("next-page", items[1].metadata["next_continuation_token"])
|
||||||
|
self.assertTrue(client.closed)
|
||||||
|
|
||||||
def test_s3_browse_lists_buckets_when_profile_has_no_bucket(self) -> None:
|
def test_s3_browse_lists_buckets_when_profile_has_no_bucket(self) -> None:
|
||||||
client = FakeS3Client()
|
client = FakeS3Client()
|
||||||
@@ -181,27 +184,33 @@ class ConnectorProviderTests(unittest.TestCase):
|
|||||||
|
|
||||||
self.assertEqual(["archive"], [item.path for item in items])
|
self.assertEqual(["archive"], [item.path for item in items])
|
||||||
|
|
||||||
def test_s3_sdk_transport_fails_closed_before_client_creation_in_private_mode(self) -> None:
|
def test_s3_client_is_constructed_through_the_pinned_transport(self) -> None:
|
||||||
with patch.dict(
|
with patch.dict(
|
||||||
"os.environ",
|
"os.environ",
|
||||||
{"APP_ENV": "production", "GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS": "true"},
|
{"APP_ENV": "production", "GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS": "true"},
|
||||||
), patch(
|
), patch(
|
||||||
"govoplan_core.security.outbound_http.socket.getaddrinfo",
|
"govoplan_core.security.outbound_http.socket.getaddrinfo",
|
||||||
return_value=[(2, 1, 6, "", ("127.0.0.1", 9000))],
|
return_value=[(2, 1, 6, "", ("127.0.0.1", 9000))],
|
||||||
), patch("govoplan_files.backend.storage.connector_browse.import_module") as importer, self.assertRaisesRegex(
|
), patch(
|
||||||
ConnectorBrowseError,
|
"govoplan_files.backend.storage.connector_browse.create_pinned_s3_client",
|
||||||
"until that transport supports.*DNS/IP pinning",
|
return_value=FakeS3Client(),
|
||||||
):
|
) as factory:
|
||||||
browse_connector_profile(s3_profile(), path="")
|
browse_connector_profile(s3_profile(), path="")
|
||||||
importer.assert_not_called()
|
|
||||||
|
|
||||||
def test_s3_sdk_endpoint_discovery_fails_closed(self) -> None:
|
kwargs = factory.call_args.kwargs
|
||||||
with patch("govoplan_files.backend.storage.connector_browse.import_module") as importer, self.assertRaisesRegex(
|
self.assertEqual("http://127.0.0.1:9000", kwargs["endpoint_url"])
|
||||||
ConnectorBrowseError,
|
self.assertEqual("access-key", kwargs["aws_access_key_id"])
|
||||||
"endpoint discovery.*cannot guarantee.*DNS/IP pinning",
|
self.assertEqual("secret-key", kwargs["aws_secret_access_key"])
|
||||||
):
|
self.assertEqual({}, kwargs["config"].proxies)
|
||||||
|
|
||||||
|
def test_s3_endpoint_discovery_uses_the_same_pinned_transport(self) -> None:
|
||||||
|
with patch(
|
||||||
|
"govoplan_files.backend.storage.connector_browse.create_pinned_s3_client",
|
||||||
|
return_value=FakeS3Client(),
|
||||||
|
) as factory:
|
||||||
browse_connector_profile(s3_profile(endpoint_url=None), path="")
|
browse_connector_profile(s3_profile(endpoint_url=None), path="")
|
||||||
importer.assert_not_called()
|
|
||||||
|
self.assertNotIn("endpoint_url", factory.call_args.kwargs)
|
||||||
|
|
||||||
def test_s3_import_downloads_object_and_preserves_remote_identity(self) -> None:
|
def test_s3_import_downloads_object_and_preserves_remote_identity(self) -> None:
|
||||||
client = FakeS3Client()
|
client = FakeS3Client()
|
||||||
@@ -217,6 +226,7 @@ class ConnectorProviderTests(unittest.TestCase):
|
|||||||
self.assertEqual("govoplan:root/report.txt", downloaded.external_id)
|
self.assertEqual("govoplan:root/report.txt", downloaded.external_id)
|
||||||
self.assertEqual("s3://govoplan/root/report.txt", downloaded.external_url)
|
self.assertEqual("s3://govoplan/root/report.txt", downloaded.external_url)
|
||||||
self.assertEqual("checksum", downloaded.metadata["checksum_sha256"])
|
self.assertEqual("checksum", downloaded.metadata["checksum_sha256"])
|
||||||
|
self.assertTrue(client.closed)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|||||||
@@ -0,0 +1,101 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import unittest
|
||||||
|
from types import SimpleNamespace
|
||||||
|
from unittest.mock import Mock, patch
|
||||||
|
|
||||||
|
from govoplan_files.backend.storage.common import FileStorageError
|
||||||
|
from govoplan_files.backend.storage.connector_profiles import ConnectorProfile
|
||||||
|
from govoplan_files.backend.storage.connector_spaces import (
|
||||||
|
soft_delete_connector_space,
|
||||||
|
validate_connector_space_write_mode,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectorSpaceDeletionTests(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
self.session = Mock()
|
||||||
|
self.space = SimpleNamespace(
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
owner_type="user",
|
||||||
|
owner_user_id="owner-1",
|
||||||
|
owner_group_id=None,
|
||||||
|
deleted_at=None,
|
||||||
|
is_active=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
@patch("govoplan_files.backend.storage.connector_spaces.utcnow")
|
||||||
|
@patch("govoplan_files.backend.storage.connector_spaces.ensure_owner_access")
|
||||||
|
def test_owner_can_soft_delete_local_space_definition(
|
||||||
|
self, ensure_owner_access: Mock, utcnow: Mock
|
||||||
|
) -> None:
|
||||||
|
deleted_at = object()
|
||||||
|
utcnow.return_value = deleted_at
|
||||||
|
|
||||||
|
result = soft_delete_connector_space(
|
||||||
|
self.session, self.space, user_id="owner-1"
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertIs(result, self.space)
|
||||||
|
self.assertIs(deleted_at, self.space.deleted_at)
|
||||||
|
self.assertFalse(self.space.is_active)
|
||||||
|
ensure_owner_access.assert_called_once_with(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
owner_type="user",
|
||||||
|
owner_id="owner-1",
|
||||||
|
user_id="owner-1",
|
||||||
|
is_admin=False,
|
||||||
|
)
|
||||||
|
self.session.add.assert_called_once_with(self.space)
|
||||||
|
self.session.flush.assert_called_once_with()
|
||||||
|
|
||||||
|
@patch(
|
||||||
|
"govoplan_files.backend.storage.connector_spaces.ensure_owner_access",
|
||||||
|
side_effect=FileStorageError("File space access denied"),
|
||||||
|
)
|
||||||
|
def test_inaccessible_space_is_blocked_without_mutation(
|
||||||
|
self, _ensure_owner_access: Mock
|
||||||
|
) -> None:
|
||||||
|
with self.assertRaisesRegex(FileStorageError, "access denied"):
|
||||||
|
soft_delete_connector_space(
|
||||||
|
self.session, self.space, user_id="other-user"
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertIsNone(self.space.deleted_at)
|
||||||
|
self.assertTrue(self.space.is_active)
|
||||||
|
self.session.add.assert_not_called()
|
||||||
|
self.session.flush.assert_not_called()
|
||||||
|
|
||||||
|
def test_read_only_mode_does_not_require_remote_write_capability(self) -> None:
|
||||||
|
profile = ConnectorProfile(
|
||||||
|
id="webdav-read",
|
||||||
|
label="Read only",
|
||||||
|
provider="webdav",
|
||||||
|
capabilities=("browse", "import"),
|
||||||
|
)
|
||||||
|
|
||||||
|
validate_connector_space_write_mode(profile, read_only=True)
|
||||||
|
|
||||||
|
def test_two_way_mode_requires_explicit_supported_write_capability(self) -> None:
|
||||||
|
missing_capability = ConnectorProfile(
|
||||||
|
id="s3-read",
|
||||||
|
label="S3 read only",
|
||||||
|
provider="s3",
|
||||||
|
capabilities=("browse",),
|
||||||
|
)
|
||||||
|
unsupported_provider = ConnectorProfile(
|
||||||
|
id="webdav-write",
|
||||||
|
label="WebDAV write",
|
||||||
|
provider="webdav",
|
||||||
|
capabilities=("write",),
|
||||||
|
)
|
||||||
|
|
||||||
|
for profile in (missing_capability, unsupported_provider):
|
||||||
|
with self.subTest(profile=profile.id):
|
||||||
|
with self.assertRaisesRegex(FileStorageError, "requires an S3"):
|
||||||
|
validate_connector_space_write_mode(profile, read_only=False)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -10,6 +10,9 @@ from govoplan_files.backend.storage.connector_visibility import (
|
|||||||
connector_profile_usable_for_import,
|
connector_profile_usable_for_import,
|
||||||
visible_connector_profiles_for_actor,
|
visible_connector_profiles_for_actor,
|
||||||
)
|
)
|
||||||
|
from govoplan_files.backend.storage.connector_providers import (
|
||||||
|
connector_provider_descriptors,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _profile(
|
def _profile(
|
||||||
@@ -168,12 +171,16 @@ class ConnectorVisibilityTests(unittest.TestCase):
|
|||||||
self,
|
self,
|
||||||
) -> None:
|
) -> None:
|
||||||
self.assertTrue(connector_profile_usable_for_import(_profile("webdav")))
|
self.assertTrue(connector_profile_usable_for_import(_profile("webdav")))
|
||||||
self.assertFalse(
|
descriptors = {
|
||||||
connector_profile_usable_for_import(_profile("s3", provider="s3"))
|
item.provider: item for item in connector_provider_descriptors()
|
||||||
)
|
}
|
||||||
self.assertFalse(
|
for provider in ("s3", "smb"):
|
||||||
connector_profile_usable_for_import(_profile("smb", provider="smb"))
|
self.assertEqual(
|
||||||
)
|
descriptors[provider].installed,
|
||||||
|
connector_profile_usable_for_import(
|
||||||
|
_profile(provider, provider=provider)
|
||||||
|
),
|
||||||
|
)
|
||||||
self.assertFalse(
|
self.assertFalse(
|
||||||
connector_profile_usable_for_import(
|
connector_profile_usable_for_import(
|
||||||
_profile("sharepoint", provider="sharepoint")
|
_profile("sharepoint", provider="sharepoint")
|
||||||
|
|||||||
@@ -0,0 +1,149 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
import hashlib
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from sqlalchemy import create_engine
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from govoplan_core.core.encryption import (
|
||||||
|
CAPABILITY_ENCRYPTION_CONTENT_CIPHER,
|
||||||
|
ContentProtectionEnvelope,
|
||||||
|
ProtectedContent,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.db.models import FileBlob
|
||||||
|
from govoplan_files.backend.runtime import configure_runtime
|
||||||
|
from govoplan_files.backend.storage.common import FileStorageError
|
||||||
|
from govoplan_files.backend.storage.content_protection import protect_blob_content
|
||||||
|
from govoplan_files.backend.storage.integrity import read_verified_blob_bytes
|
||||||
|
|
||||||
|
|
||||||
|
class Registry:
|
||||||
|
def __init__(self, capability=None) -> None:
|
||||||
|
self.capability_value = capability
|
||||||
|
|
||||||
|
def has_capability(self, name: str) -> bool:
|
||||||
|
return (
|
||||||
|
name == CAPABILITY_ENCRYPTION_CONTENT_CIPHER
|
||||||
|
and self.capability_value is not None
|
||||||
|
)
|
||||||
|
|
||||||
|
def capability(self, _name: str):
|
||||||
|
return self.capability_value
|
||||||
|
|
||||||
|
|
||||||
|
class Cipher:
|
||||||
|
def protect_content(self, _session, *, request):
|
||||||
|
ciphertext = b"protected:" + request.plaintext
|
||||||
|
envelope = ContentProtectionEnvelope(
|
||||||
|
envelope_id="envelope-file-1",
|
||||||
|
tenant_id=request.tenant_id,
|
||||||
|
owner_module=request.owner_module,
|
||||||
|
resource_type=request.resource_type,
|
||||||
|
resource_id=request.resource_id,
|
||||||
|
profile_kind="server_envelope",
|
||||||
|
profile_id=request.profile_id,
|
||||||
|
provider_id="test",
|
||||||
|
vault_id=request.vault_id,
|
||||||
|
key_version=1,
|
||||||
|
algorithm_suite="AES-256-GCM",
|
||||||
|
ciphertext_ref=request.ciphertext_ref,
|
||||||
|
ciphertext_digest=f"sha256:{hashlib.sha256(ciphertext).hexdigest()}",
|
||||||
|
authenticated_context_digest="sha256:context",
|
||||||
|
state="active",
|
||||||
|
created_at=datetime.now(tz=UTC),
|
||||||
|
wrapped_key_refs=("wrapped:1",),
|
||||||
|
)
|
||||||
|
return ProtectedContent(envelope=envelope, ciphertext=ciphertext)
|
||||||
|
|
||||||
|
def unprotect_content(self, _session, *, request):
|
||||||
|
if not request.ciphertext.startswith(b"protected:"):
|
||||||
|
raise ValueError("invalid fixture ciphertext")
|
||||||
|
return request.ciphertext.removeprefix(b"protected:")
|
||||||
|
|
||||||
|
def execute_rewrap(self, *_args, **_kwargs):
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
|
def prepare_reencryption(self, *_args, **_kwargs):
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
|
|
||||||
|
class Backend:
|
||||||
|
def __init__(self, data: bytes) -> None:
|
||||||
|
self.data = data
|
||||||
|
|
||||||
|
def get_bytes(self, _key: str) -> bytes:
|
||||||
|
return self.data
|
||||||
|
|
||||||
|
|
||||||
|
class FileContentProtectionTests(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
self.engine = create_engine("sqlite+pysqlite:///:memory:")
|
||||||
|
FileBlob.__table__.create(self.engine)
|
||||||
|
self.session = Session(self.engine)
|
||||||
|
|
||||||
|
def tearDown(self) -> None:
|
||||||
|
self.session.close()
|
||||||
|
self.engine.dispose()
|
||||||
|
|
||||||
|
def test_encrypted_blob_is_opened_after_stored_object_integrity_check(self) -> None:
|
||||||
|
configure_runtime(registry=Registry(Cipher()), settings=object())
|
||||||
|
plaintext = b"monthly records"
|
||||||
|
protected = protect_blob_content(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
blob_id="blob-1",
|
||||||
|
vault_id="vault-1",
|
||||||
|
ciphertext_ref="tenants/tenant-1/files/blob-1",
|
||||||
|
plaintext=plaintext,
|
||||||
|
actor_id="account-1",
|
||||||
|
content_type="text/plain",
|
||||||
|
)
|
||||||
|
blob = FileBlob(
|
||||||
|
id="blob-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
storage_backend="test",
|
||||||
|
storage_key="tenants/tenant-1/files/blob-1",
|
||||||
|
checksum_sha256=hashlib.sha256(plaintext).hexdigest(),
|
||||||
|
size_bytes=len(plaintext),
|
||||||
|
protection_discriminator="vault:vault-1",
|
||||||
|
encryption_envelope_id=protected.envelope.envelope_id,
|
||||||
|
storage_checksum_sha256=hashlib.sha256(protected.ciphertext).hexdigest(),
|
||||||
|
storage_size_bytes=len(protected.ciphertext),
|
||||||
|
content_type="text/plain",
|
||||||
|
ref_count=1,
|
||||||
|
integrity_status="verified",
|
||||||
|
)
|
||||||
|
self.session.add(blob)
|
||||||
|
self.session.flush()
|
||||||
|
self.assertEqual(
|
||||||
|
plaintext,
|
||||||
|
read_verified_blob_bytes(blob, backend=Backend(protected.ciphertext)),
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_encrypted_blob_fails_closed_without_encryption_capability(self) -> None:
|
||||||
|
configure_runtime(registry=Registry(), settings=object())
|
||||||
|
ciphertext = b"protected:monthly records"
|
||||||
|
blob = FileBlob(
|
||||||
|
id="blob-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
storage_backend="test",
|
||||||
|
storage_key="tenants/tenant-1/files/blob-1",
|
||||||
|
checksum_sha256=hashlib.sha256(b"monthly records").hexdigest(),
|
||||||
|
size_bytes=len(b"monthly records"),
|
||||||
|
protection_discriminator="vault:vault-1",
|
||||||
|
encryption_envelope_id="envelope-file-1",
|
||||||
|
storage_checksum_sha256=hashlib.sha256(ciphertext).hexdigest(),
|
||||||
|
storage_size_bytes=len(ciphertext),
|
||||||
|
ref_count=1,
|
||||||
|
integrity_status="verified",
|
||||||
|
)
|
||||||
|
self.session.add(blob)
|
||||||
|
self.session.flush()
|
||||||
|
with self.assertRaisesRegex(FileStorageError, "Encryption is unavailable"):
|
||||||
|
read_verified_blob_bytes(blob, backend=Backend(ciphertext))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -49,6 +49,10 @@ class FilesRuntimeDocumentationTests(unittest.TestCase):
|
|||||||
settings = SimpleNamespace(
|
settings = SimpleNamespace(
|
||||||
file_upload_max_bytes=7 * 1024 * 1024,
|
file_upload_max_bytes=7 * 1024 * 1024,
|
||||||
file_upload_zip_max_bytes=19 * 1024 * 1024,
|
file_upload_zip_max_bytes=19 * 1024 * 1024,
|
||||||
|
file_archive_max_expanded_bytes=31 * 1024 * 1024,
|
||||||
|
file_archive_max_entries=321,
|
||||||
|
file_archive_max_expansion_ratio=42,
|
||||||
|
file_archive_preview_ttl_seconds=15 * 60,
|
||||||
)
|
)
|
||||||
with patch(
|
with patch(
|
||||||
"govoplan_files.backend.documentation.visible_connector_profiles_for_actor",
|
"govoplan_files.backend.documentation.visible_connector_profiles_for_actor",
|
||||||
@@ -73,8 +77,12 @@ class FilesRuntimeDocumentationTests(unittest.TestCase):
|
|||||||
|
|
||||||
archive = topics["files.workflow.upload-and-unpack-zip"]
|
archive = topics["files.workflow.upload-and-unpack-zip"]
|
||||||
self.assertIn("19 MiB (19,922,944 bytes)", archive.body)
|
self.assertIn("19 MiB (19,922,944 bytes)", archive.body)
|
||||||
|
self.assertIn("31 MiB (32,505,856 bytes)", archive.body)
|
||||||
self.assertIn("7 MiB (7,340,032 bytes)", archive.body)
|
self.assertIn("7 MiB (7,340,032 bytes)", archive.body)
|
||||||
self.assertIn("1,000", archive.body)
|
self.assertIn("321", archive.body)
|
||||||
|
self.assertIn("42:1", archive.body)
|
||||||
|
self.assertIn("15 minutes", archive.body)
|
||||||
|
self.assertIn("passwords remain request-only", archive.body)
|
||||||
self.assertIn("Actual extracted bytes are counted", archive.body)
|
self.assertIn("Actual extracted bytes are counted", archive.body)
|
||||||
|
|
||||||
def test_connector_task_requires_authority_and_a_visible_usable_profile(
|
def test_connector_task_requires_authority_and_a_visible_usable_profile(
|
||||||
@@ -134,6 +142,7 @@ class FilesRuntimeDocumentationTests(unittest.TestCase):
|
|||||||
base_path="/classified",
|
base_path="/classified",
|
||||||
credential_mode="basic",
|
credential_mode="basic",
|
||||||
password_value="credential-secret",
|
password_value="credential-secret",
|
||||||
|
secret_ref="runtime/connector-secret",
|
||||||
)
|
)
|
||||||
with patch(
|
with patch(
|
||||||
"govoplan_files.backend.documentation.visible_connector_profiles_for_actor",
|
"govoplan_files.backend.documentation.visible_connector_profiles_for_actor",
|
||||||
|
|||||||
@@ -0,0 +1,468 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import unittest
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
|
||||||
|
from sqlalchemy import create_engine
|
||||||
|
from sqlalchemy.orm import sessionmaker
|
||||||
|
|
||||||
|
from govoplan_access.backend.db.models import Account, Group, User
|
||||||
|
from govoplan_core.core.dsar import DsarProvider, DsarSubjectRef
|
||||||
|
from govoplan_core.core.change_sequence import ChangeSequenceEntry
|
||||||
|
from govoplan_core.db.base import Base
|
||||||
|
from govoplan_core.privacy.dsar_workflow import (
|
||||||
|
DataSubjectRequest,
|
||||||
|
create_data_subject_request,
|
||||||
|
execute_data_subject_erasure,
|
||||||
|
plan_data_subject_erasure,
|
||||||
|
search_data_subject_request,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.db.models import (
|
||||||
|
FileAsset,
|
||||||
|
FileBlob,
|
||||||
|
FileConnectorCredential,
|
||||||
|
FileConnectorPolicy,
|
||||||
|
FileConnectorProfile,
|
||||||
|
FileConnectorSpace,
|
||||||
|
FileFolder,
|
||||||
|
FileFormEvidenceGrant,
|
||||||
|
FileIntegrityFinding,
|
||||||
|
FileIntegrityScan,
|
||||||
|
FileShare,
|
||||||
|
FileVersion,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.dsar_provider import (
|
||||||
|
FILES_DSAR_CAPABILITY,
|
||||||
|
FilesDsarProvider,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.manifest import manifest
|
||||||
|
|
||||||
|
|
||||||
|
class _Registry:
|
||||||
|
def __init__(self, provider: FilesDsarProvider, *, files_active: bool = True) -> None:
|
||||||
|
self.provider = provider
|
||||||
|
self.files_active = files_active
|
||||||
|
|
||||||
|
def capability_names(self):
|
||||||
|
return (FILES_DSAR_CAPABILITY,)
|
||||||
|
|
||||||
|
def capability_owner(self, name):
|
||||||
|
self._assert_capability(name)
|
||||||
|
return "files"
|
||||||
|
|
||||||
|
def tenant_entitlement_resolver(self):
|
||||||
|
files_active = self.files_active
|
||||||
|
|
||||||
|
class _Resolver:
|
||||||
|
@staticmethod
|
||||||
|
def resolve(session, tenant_id):
|
||||||
|
del session, tenant_id
|
||||||
|
return type(
|
||||||
|
"State",
|
||||||
|
(),
|
||||||
|
{"effective_modules": ("files",) if files_active else ()},
|
||||||
|
)()
|
||||||
|
|
||||||
|
return _Resolver()
|
||||||
|
|
||||||
|
def require_tenant_capability(self, name, session, **kwargs):
|
||||||
|
del session, kwargs
|
||||||
|
self._assert_capability(name)
|
||||||
|
return self.provider
|
||||||
|
|
||||||
|
def manifests(self):
|
||||||
|
return (type("Manifest", (), {"id": "files"})(),)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _assert_capability(name: str) -> None:
|
||||||
|
if name != FILES_DSAR_CAPABILITY:
|
||||||
|
raise KeyError(name)
|
||||||
|
|
||||||
|
|
||||||
|
class FilesDsarProviderTests(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
self.engine = create_engine("sqlite:///:memory:", future=True)
|
||||||
|
Base.metadata.create_all(
|
||||||
|
bind=self.engine,
|
||||||
|
tables=[
|
||||||
|
Account.__table__,
|
||||||
|
User.__table__,
|
||||||
|
Group.__table__,
|
||||||
|
ChangeSequenceEntry.__table__,
|
||||||
|
DataSubjectRequest.__table__,
|
||||||
|
FileBlob.__table__,
|
||||||
|
FileAsset.__table__,
|
||||||
|
FileVersion.__table__,
|
||||||
|
FileFolder.__table__,
|
||||||
|
FileFormEvidenceGrant.__table__,
|
||||||
|
FileShare.__table__,
|
||||||
|
FileConnectorCredential.__table__,
|
||||||
|
FileConnectorPolicy.__table__,
|
||||||
|
FileConnectorProfile.__table__,
|
||||||
|
FileConnectorSpace.__table__,
|
||||||
|
FileIntegrityScan.__table__,
|
||||||
|
FileIntegrityFinding.__table__,
|
||||||
|
],
|
||||||
|
)
|
||||||
|
self.session = sessionmaker(bind=self.engine, future=True)()
|
||||||
|
self.account = Account(
|
||||||
|
id="account-1",
|
||||||
|
email="subject@example.test",
|
||||||
|
normalized_email="subject@example.test",
|
||||||
|
display_name="Subject",
|
||||||
|
password_hash="not-exported",
|
||||||
|
)
|
||||||
|
self.user = User(
|
||||||
|
id="membership-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
account_id=self.account.id,
|
||||||
|
email="subject@example.test",
|
||||||
|
display_name="Subject",
|
||||||
|
)
|
||||||
|
self.blob = FileBlob(
|
||||||
|
id="blob-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
storage_backend="local",
|
||||||
|
storage_key="private/storage/key-do-not-export",
|
||||||
|
checksum_sha256="a" * 64,
|
||||||
|
size_bytes=12,
|
||||||
|
ref_count=1,
|
||||||
|
)
|
||||||
|
self.asset = FileAsset(
|
||||||
|
id="asset-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
owner_type="user",
|
||||||
|
owner_user_id=self.user.id,
|
||||||
|
created_by_user_id=self.user.id,
|
||||||
|
current_version_id="version-1",
|
||||||
|
display_path="subjects/private.txt",
|
||||||
|
filename="private.txt",
|
||||||
|
description="Subject-provided document",
|
||||||
|
retained_until=datetime.now(timezone.utc) + timedelta(days=30),
|
||||||
|
lifecycle_reason="Pending proceeding",
|
||||||
|
metadata_={"password": "metadata-secret-do-not-export"},
|
||||||
|
)
|
||||||
|
self.version = FileVersion(
|
||||||
|
id="version-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
file_asset_id=self.asset.id,
|
||||||
|
blob_id=self.blob.id,
|
||||||
|
version_number=1,
|
||||||
|
filename_at_upload=self.asset.filename,
|
||||||
|
display_path_at_upload=self.asset.display_path,
|
||||||
|
content_type="text/plain",
|
||||||
|
size_bytes=12,
|
||||||
|
checksum_sha256="a" * 64,
|
||||||
|
created_by_user_id=self.user.id,
|
||||||
|
)
|
||||||
|
self.folder = FileFolder(
|
||||||
|
id="folder-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
owner_type="user",
|
||||||
|
owner_user_id=self.user.id,
|
||||||
|
path="subjects",
|
||||||
|
created_by_user_id=self.user.id,
|
||||||
|
)
|
||||||
|
self.share = FileShare(
|
||||||
|
id="share-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
file_asset_id=self.asset.id,
|
||||||
|
target_type="user",
|
||||||
|
target_id=self.user.id,
|
||||||
|
permission="read",
|
||||||
|
created_by_user_id=self.user.id,
|
||||||
|
)
|
||||||
|
self.form_evidence = FileFormEvidenceGrant(
|
||||||
|
id="evidence-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
form_instance_id="form-instance-1",
|
||||||
|
form_definition_id="application",
|
||||||
|
form_definition_revision="7",
|
||||||
|
token_sha256="b" * 64,
|
||||||
|
idempotency_key="evidence-key-1",
|
||||||
|
request_sha256="c" * 64,
|
||||||
|
custodian_user_id=self.user.id,
|
||||||
|
evidence_kind="attachment",
|
||||||
|
purpose="Submitted application evidence",
|
||||||
|
status="uploaded",
|
||||||
|
expires_at=datetime.now(timezone.utc) + timedelta(days=1),
|
||||||
|
max_size_bytes=1024,
|
||||||
|
allowed_content_types=["text/plain"],
|
||||||
|
file_asset_id=self.asset.id,
|
||||||
|
file_version_id=self.version.id,
|
||||||
|
metadata_={},
|
||||||
|
)
|
||||||
|
self.credential = FileConnectorCredential(
|
||||||
|
id="credential-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
scope_type="tenant",
|
||||||
|
scope_id="tenant-1",
|
||||||
|
label="Subject-created credential",
|
||||||
|
provider="s3",
|
||||||
|
credential_mode="database",
|
||||||
|
username="subject-user",
|
||||||
|
password_encrypted="encrypted-password-do-not-export",
|
||||||
|
token_encrypted="encrypted-token-do-not-export",
|
||||||
|
password_env="PASSWORD_ENV_DO_NOT_EXPORT",
|
||||||
|
secret_ref="vault://do-not-export",
|
||||||
|
created_by_user_id=self.user.id,
|
||||||
|
updated_by_user_id=self.user.id,
|
||||||
|
)
|
||||||
|
self.integrity_scan = FileIntegrityScan(
|
||||||
|
id="scan-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
storage_backend="local",
|
||||||
|
storage_prefix="private-prefix-do-not-export",
|
||||||
|
created_by_user_id=self.user.id,
|
||||||
|
)
|
||||||
|
tenant_two_asset = FileAsset(
|
||||||
|
id="asset-tenant-2",
|
||||||
|
tenant_id="tenant-2",
|
||||||
|
owner_type="user",
|
||||||
|
owner_user_id=self.user.id,
|
||||||
|
display_path="other-tenant.txt",
|
||||||
|
filename="other-tenant.txt",
|
||||||
|
)
|
||||||
|
self.session.add_all(
|
||||||
|
[
|
||||||
|
self.account,
|
||||||
|
self.user,
|
||||||
|
self.blob,
|
||||||
|
self.asset,
|
||||||
|
self.version,
|
||||||
|
self.folder,
|
||||||
|
self.share,
|
||||||
|
self.form_evidence,
|
||||||
|
self.credential,
|
||||||
|
self.integrity_scan,
|
||||||
|
tenant_two_asset,
|
||||||
|
]
|
||||||
|
)
|
||||||
|
self.session.commit()
|
||||||
|
self.provider = FilesDsarProvider()
|
||||||
|
self.subject = DsarSubjectRef(membership_id=self.user.id)
|
||||||
|
|
||||||
|
def tearDown(self) -> None:
|
||||||
|
self.session.close()
|
||||||
|
self.engine.dispose()
|
||||||
|
|
||||||
|
def test_manifest_publishes_protocol_conforming_provider(self) -> None:
|
||||||
|
provided_names = {item.name for item in manifest.provides_interfaces}
|
||||||
|
self.assertIn(FILES_DSAR_CAPABILITY, provided_names)
|
||||||
|
provider = manifest.capability_factories[FILES_DSAR_CAPABILITY](None)
|
||||||
|
self.assertIsInstance(provider, DsarProvider)
|
||||||
|
|
||||||
|
def test_search_is_tenant_scoped_and_excludes_bytes_and_secrets(self) -> None:
|
||||||
|
records = self.provider.search_subject(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
subject=self.subject,
|
||||||
|
)
|
||||||
|
resource_types = {record.resource_type for record in records}
|
||||||
|
self.assertTrue(
|
||||||
|
{
|
||||||
|
"file_asset",
|
||||||
|
"file_version",
|
||||||
|
"file_folder",
|
||||||
|
"file_share",
|
||||||
|
"file_form_evidence",
|
||||||
|
"connector_credential",
|
||||||
|
"file_integrity_scan",
|
||||||
|
}.issubset(resource_types)
|
||||||
|
)
|
||||||
|
serialized = repr([record.to_dict() for record in records])
|
||||||
|
self.assertNotIn("asset-tenant-2", serialized)
|
||||||
|
self.assertNotIn("private/storage/key-do-not-export", serialized)
|
||||||
|
self.assertNotIn("metadata-secret-do-not-export", serialized)
|
||||||
|
self.assertNotIn("encrypted-password-do-not-export", serialized)
|
||||||
|
self.assertNotIn("encrypted-token-do-not-export", serialized)
|
||||||
|
self.assertNotIn("subject-user", serialized)
|
||||||
|
self.assertNotIn("PASSWORD_ENV_DO_NOT_EXPORT", serialized)
|
||||||
|
self.assertNotIn("vault://do-not-export", serialized)
|
||||||
|
self.assertNotIn("private-prefix-do-not-export", serialized)
|
||||||
|
|
||||||
|
def test_conflicting_direct_subject_references_fail_closed(self) -> None:
|
||||||
|
records = self.provider.search_subject(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
subject=DsarSubjectRef(
|
||||||
|
membership_id=self.user.id,
|
||||||
|
external_references={"files.user": "another-membership"},
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual((), records)
|
||||||
|
|
||||||
|
def test_plan_classifies_retention_and_manual_review(self) -> None:
|
||||||
|
records = self.provider.search_subject(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
subject=self.subject,
|
||||||
|
)
|
||||||
|
actions = self.provider.plan_erasure(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
subject=self.subject,
|
||||||
|
records=records,
|
||||||
|
)
|
||||||
|
|
||||||
|
kinds = {action.kind for action in actions}
|
||||||
|
self.assertTrue({"retain", "manual_review", "revoke", "detach"}.issubset(kinds))
|
||||||
|
asset_retention = next(
|
||||||
|
action
|
||||||
|
for action in actions
|
||||||
|
if action.action_id == "files:retain:file_asset:asset-1"
|
||||||
|
)
|
||||||
|
self.assertIn("retained until", asset_retention.rationale)
|
||||||
|
self.assertTrue(
|
||||||
|
any(action.action_id == "files:revoke:file_share:share-1" for action in actions)
|
||||||
|
)
|
||||||
|
self.assertFalse(
|
||||||
|
any(action.kind == "delete" and action.executable for action in actions)
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_execution_is_revalidated_and_idempotent(self) -> None:
|
||||||
|
records = self.provider.search_subject(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
subject=self.subject,
|
||||||
|
)
|
||||||
|
actions = self.provider.plan_erasure(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
subject=self.subject,
|
||||||
|
records=records,
|
||||||
|
)
|
||||||
|
executable = tuple(action for action in actions if action.executable)
|
||||||
|
|
||||||
|
first = self.provider.execute_erasure(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
subject=self.subject,
|
||||||
|
actions=executable,
|
||||||
|
request_id="dsar-1",
|
||||||
|
)
|
||||||
|
self.assertEqual({"executed"}, {result.status for result in first})
|
||||||
|
self.assertIsNotNone(self.share.revoked_at)
|
||||||
|
self.assertIsNone(self.asset.created_by_user_id)
|
||||||
|
self.assertIsNone(self.version.created_by_user_id)
|
||||||
|
self.assertIsNone(self.folder.created_by_user_id)
|
||||||
|
self.assertIsNone(self.credential.created_by_user_id)
|
||||||
|
self.assertIsNone(self.credential.updated_by_user_id)
|
||||||
|
|
||||||
|
repeated = self.provider.execute_erasure(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
subject=self.subject,
|
||||||
|
actions=executable,
|
||||||
|
request_id="dsar-1",
|
||||||
|
)
|
||||||
|
self.assertEqual({"unchanged"}, {result.status for result in repeated})
|
||||||
|
|
||||||
|
def test_execution_blocks_when_reference_changed_after_planning(self) -> None:
|
||||||
|
records = self.provider.search_subject(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
subject=self.subject,
|
||||||
|
)
|
||||||
|
actions = self.provider.plan_erasure(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
subject=self.subject,
|
||||||
|
records=records,
|
||||||
|
)
|
||||||
|
detach = next(
|
||||||
|
action
|
||||||
|
for action in actions
|
||||||
|
if action.action_id
|
||||||
|
== "files:detach:file_version:created_by_user_id:version-1"
|
||||||
|
)
|
||||||
|
self.version.created_by_user_id = "replacement-user"
|
||||||
|
self.session.flush()
|
||||||
|
|
||||||
|
result = self.provider.execute_erasure(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
subject=self.subject,
|
||||||
|
actions=(detach,),
|
||||||
|
request_id="dsar-2",
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual("blocked", result[0].status)
|
||||||
|
self.assertEqual("replacement-user", self.version.created_by_user_id)
|
||||||
|
|
||||||
|
def test_core_workflow_discovers_active_provider_and_skips_it_when_disabled(
|
||||||
|
self,
|
||||||
|
) -> None:
|
||||||
|
request = create_data_subject_request(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
reference="DSAR-FILES-1",
|
||||||
|
request_kind="access_and_erasure",
|
||||||
|
subject=self.subject,
|
||||||
|
purpose="Respond to an authorized privacy request.",
|
||||||
|
legal_basis="Article 15 and 17 GDPR",
|
||||||
|
due_at=None,
|
||||||
|
requested_by_account_id="privacy-officer",
|
||||||
|
)
|
||||||
|
self.session.commit()
|
||||||
|
registry = _Registry(self.provider)
|
||||||
|
|
||||||
|
search_data_subject_request(
|
||||||
|
self.session,
|
||||||
|
registry=registry,
|
||||||
|
row=request,
|
||||||
|
expected_revision=1,
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
"searched", request.status, request.search_result["provider_runs"]
|
||||||
|
)
|
||||||
|
self.assertEqual(["files"], request.coverage["covered_modules"])
|
||||||
|
self.assertEqual([], request.coverage["modules_without_provider"])
|
||||||
|
plan_data_subject_erasure(
|
||||||
|
self.session,
|
||||||
|
registry=registry,
|
||||||
|
row=request,
|
||||||
|
expected_revision=2,
|
||||||
|
)
|
||||||
|
executable_ids = [
|
||||||
|
action["action_id"]
|
||||||
|
for action in request.erasure_plan["actions"]
|
||||||
|
if action["executable"]
|
||||||
|
]
|
||||||
|
execute_data_subject_erasure(
|
||||||
|
self.session,
|
||||||
|
registry=registry,
|
||||||
|
row=request,
|
||||||
|
expected_revision=3,
|
||||||
|
action_ids=executable_ids,
|
||||||
|
)
|
||||||
|
self.assertEqual("completed", request.status)
|
||||||
|
|
||||||
|
disabled = create_data_subject_request(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
reference="DSAR-FILES-DISABLED",
|
||||||
|
request_kind="access",
|
||||||
|
subject=self.subject,
|
||||||
|
purpose="Verify disabled-module coverage.",
|
||||||
|
legal_basis="Article 15 GDPR",
|
||||||
|
due_at=None,
|
||||||
|
requested_by_account_id="privacy-officer",
|
||||||
|
)
|
||||||
|
search_data_subject_request(
|
||||||
|
self.session,
|
||||||
|
registry=_Registry(self.provider, files_active=False),
|
||||||
|
row=disabled,
|
||||||
|
expected_revision=1,
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(0, disabled.search_result["record_count"])
|
||||||
|
self.assertEqual(
|
||||||
|
[FILES_DSAR_CAPABILITY],
|
||||||
|
disabled.coverage["inactive_provider_capabilities"],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,164 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import timedelta
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from sqlalchemy import create_engine
|
||||||
|
from sqlalchemy.orm import sessionmaker
|
||||||
|
|
||||||
|
from govoplan_access.backend.db.models import Account, Group, User
|
||||||
|
from govoplan_core.db.base import Base
|
||||||
|
from govoplan_core.core.change_sequence import ChangeSequenceEntry
|
||||||
|
from govoplan_files.backend.db.models import (
|
||||||
|
FileAsset,
|
||||||
|
FileBlob,
|
||||||
|
FileFolder,
|
||||||
|
FileFormEvidenceGrant,
|
||||||
|
FileShare,
|
||||||
|
FileVersion,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.storage.common import FileStorageError, utcnow
|
||||||
|
from govoplan_files.backend.storage.lifecycle import (
|
||||||
|
preview_asset_purge,
|
||||||
|
restore_asset,
|
||||||
|
set_asset_lifecycle,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class FileLifecycleTests(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
self.engine = create_engine("sqlite:///:memory:", future=True)
|
||||||
|
Base.metadata.create_all(
|
||||||
|
bind=self.engine,
|
||||||
|
tables=[
|
||||||
|
Account.__table__,
|
||||||
|
User.__table__,
|
||||||
|
Group.__table__,
|
||||||
|
ChangeSequenceEntry.__table__,
|
||||||
|
FileBlob.__table__,
|
||||||
|
FileAsset.__table__,
|
||||||
|
FileVersion.__table__,
|
||||||
|
FileFolder.__table__,
|
||||||
|
FileFormEvidenceGrant.__table__,
|
||||||
|
FileShare.__table__,
|
||||||
|
],
|
||||||
|
)
|
||||||
|
self.session = sessionmaker(bind=self.engine, future=True)()
|
||||||
|
self.blob = FileBlob(
|
||||||
|
id="blob-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
storage_backend="local",
|
||||||
|
storage_key="tenants/tenant-1/files/blob-1",
|
||||||
|
checksum_sha256="a" * 64,
|
||||||
|
size_bytes=1,
|
||||||
|
ref_count=1,
|
||||||
|
)
|
||||||
|
self.asset = FileAsset(
|
||||||
|
id="file-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
owner_type="user",
|
||||||
|
owner_user_id="user-1",
|
||||||
|
current_version_id="version-1",
|
||||||
|
display_path="records/report.txt",
|
||||||
|
filename="report.txt",
|
||||||
|
deleted_at=utcnow(),
|
||||||
|
)
|
||||||
|
self.version = FileVersion(
|
||||||
|
id="version-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
file_asset_id="file-1",
|
||||||
|
blob_id="blob-1",
|
||||||
|
version_number=1,
|
||||||
|
filename_at_upload="report.txt",
|
||||||
|
display_path_at_upload="records/report.txt",
|
||||||
|
size_bytes=1,
|
||||||
|
checksum_sha256="a" * 64,
|
||||||
|
)
|
||||||
|
self.session.add_all([self.blob, self.asset, self.version])
|
||||||
|
self.session.commit()
|
||||||
|
|
||||||
|
def tearDown(self) -> None:
|
||||||
|
self.session.close()
|
||||||
|
self.engine.dispose()
|
||||||
|
|
||||||
|
def test_preview_hash_changes_with_retention_and_blocks_active_policy(self) -> None:
|
||||||
|
first = preview_asset_purge(
|
||||||
|
self.session, tenant_id="tenant-1", file_ids=[self.asset.id]
|
||||||
|
)
|
||||||
|
self.assertTrue(first.eligible)
|
||||||
|
|
||||||
|
set_asset_lifecycle(
|
||||||
|
self.session,
|
||||||
|
self.asset,
|
||||||
|
retained_until=utcnow() + timedelta(days=7),
|
||||||
|
legal_hold=True,
|
||||||
|
reason="Active proceeding",
|
||||||
|
expected_revision=1,
|
||||||
|
)
|
||||||
|
self.session.commit()
|
||||||
|
second = preview_asset_purge(
|
||||||
|
self.session, tenant_id="tenant-1", file_ids=[self.asset.id]
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertFalse(second.eligible)
|
||||||
|
self.assertNotEqual(first.preview_sha256, second.preview_sha256)
|
||||||
|
self.assertEqual(
|
||||||
|
{"legal_hold", "retention_active"}, set(second.items[0].blockers)
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_restore_preserves_version_and_provenance_and_rejects_collision(self) -> None:
|
||||||
|
self.asset.metadata_ = {"source": {"provider": "s3", "revision": "v1"}}
|
||||||
|
self.session.commit()
|
||||||
|
|
||||||
|
self.assertTrue(restore_asset(self.session, self.asset))
|
||||||
|
self.assertEqual("version-1", self.asset.current_version_id)
|
||||||
|
self.assertEqual("v1", self.asset.metadata_["source"]["revision"])
|
||||||
|
self.session.commit()
|
||||||
|
|
||||||
|
self.asset.deleted_at = utcnow()
|
||||||
|
collision = FileAsset(
|
||||||
|
id="file-2",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
owner_type="user",
|
||||||
|
owner_user_id="user-1",
|
||||||
|
display_path=self.asset.display_path,
|
||||||
|
filename=self.asset.filename,
|
||||||
|
)
|
||||||
|
self.session.add(collision)
|
||||||
|
self.session.commit()
|
||||||
|
with self.assertRaisesRegex(FileStorageError, "path is already in use"):
|
||||||
|
restore_asset(self.session, self.asset)
|
||||||
|
|
||||||
|
def test_active_share_blocks_purge_until_revoked(self) -> None:
|
||||||
|
share = FileShare(
|
||||||
|
id="share-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
file_asset_id=self.asset.id,
|
||||||
|
target_type="tenant",
|
||||||
|
target_id="tenant-1",
|
||||||
|
permission="read",
|
||||||
|
)
|
||||||
|
self.session.add(share)
|
||||||
|
self.session.commit()
|
||||||
|
|
||||||
|
blocked = preview_asset_purge(
|
||||||
|
self.session, tenant_id="tenant-1", file_ids=[self.asset.id]
|
||||||
|
)
|
||||||
|
self.assertIn("active_share", blocked.items[0].blockers)
|
||||||
|
|
||||||
|
share.revoked_at = utcnow()
|
||||||
|
self.session.commit()
|
||||||
|
allowed = preview_asset_purge(
|
||||||
|
self.session, tenant_id="tenant-1", file_ids=[self.asset.id]
|
||||||
|
)
|
||||||
|
self.assertTrue(allowed.eligible)
|
||||||
|
|
||||||
|
def test_missing_campaign_table_is_an_optional_module_safe_path(self) -> None:
|
||||||
|
preview = preview_asset_purge(
|
||||||
|
self.session, tenant_id="tenant-1", file_ids=[self.asset.id]
|
||||||
|
)
|
||||||
|
self.assertNotIn("campaign_evidence", preview.items[0].blockers)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,333 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import UTC, datetime, timedelta
|
||||||
|
from io import BytesIO
|
||||||
|
from types import SimpleNamespace
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from fastapi import UploadFile
|
||||||
|
from sqlalchemy import create_engine
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
from starlette.datastructures import Headers
|
||||||
|
|
||||||
|
from govoplan_access.backend.db.models import Account, User
|
||||||
|
from govoplan_core.core.access import (
|
||||||
|
CAPABILITY_ACCESS_DIRECTORY,
|
||||||
|
AccessSubjectRef,
|
||||||
|
UserRef,
|
||||||
|
)
|
||||||
|
from govoplan_core.core.change_sequence import ChangeSequenceEntry
|
||||||
|
from govoplan_core.core.form_evidence import (
|
||||||
|
FormEvidenceContractError,
|
||||||
|
FormEvidenceGrantRequest,
|
||||||
|
FormEvidenceInspectionRequest,
|
||||||
|
)
|
||||||
|
from govoplan_core.core.institutional import EvidenceReference, InstitutionalReference
|
||||||
|
from govoplan_core.db.base import Base
|
||||||
|
from govoplan_files.backend.db.models import (
|
||||||
|
FileAsset,
|
||||||
|
FileBlob,
|
||||||
|
FileFormEvidenceGrant,
|
||||||
|
FileVersion,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.form_evidence import FilesFormEvidenceProvider
|
||||||
|
from govoplan_files.backend.routes.form_evidence import upload_form_evidence
|
||||||
|
|
||||||
|
|
||||||
|
class _Directory:
|
||||||
|
def __init__(self, user: UserRef) -> None:
|
||||||
|
self._user = user
|
||||||
|
|
||||||
|
def get_account(self, account_id: str):
|
||||||
|
del account_id
|
||||||
|
return None
|
||||||
|
|
||||||
|
def get_user(self, user_id: str):
|
||||||
|
return self._user if user_id == self._user.id else None
|
||||||
|
|
||||||
|
def get_users(self, user_ids):
|
||||||
|
return {user_id: self._user for user_id in user_ids if user_id == self._user.id}
|
||||||
|
|
||||||
|
def users_for_tenant(self, tenant_id: str):
|
||||||
|
return (self._user,) if tenant_id == self._user.tenant_id else ()
|
||||||
|
|
||||||
|
def get_group(self, group_id: str):
|
||||||
|
del group_id
|
||||||
|
return None
|
||||||
|
|
||||||
|
def get_groups(self, group_ids):
|
||||||
|
del group_ids
|
||||||
|
return {}
|
||||||
|
|
||||||
|
def groups_for_tenant(self, tenant_id: str):
|
||||||
|
del tenant_id
|
||||||
|
return ()
|
||||||
|
|
||||||
|
def groups_for_user(self, user_id: str, *, tenant_id: str):
|
||||||
|
del user_id, tenant_id
|
||||||
|
return ()
|
||||||
|
|
||||||
|
def display_label(self, subject: AccessSubjectRef):
|
||||||
|
del subject
|
||||||
|
return self._user.display_name
|
||||||
|
|
||||||
|
|
||||||
|
class _Registry:
|
||||||
|
def __init__(self, directory: _Directory) -> None:
|
||||||
|
self._directory = directory
|
||||||
|
|
||||||
|
def has_capability(self, name: str) -> bool:
|
||||||
|
return name == CAPABILITY_ACCESS_DIRECTORY
|
||||||
|
|
||||||
|
def require_capability(self, name: str):
|
||||||
|
if not self.has_capability(name):
|
||||||
|
raise KeyError(name)
|
||||||
|
return self._directory
|
||||||
|
|
||||||
|
|
||||||
|
class FilesFormEvidenceTests(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
self.engine = create_engine("sqlite+pysqlite:///:memory:")
|
||||||
|
Base.metadata.create_all(
|
||||||
|
self.engine,
|
||||||
|
tables=[
|
||||||
|
Account.__table__,
|
||||||
|
User.__table__,
|
||||||
|
FileBlob.__table__,
|
||||||
|
FileAsset.__table__,
|
||||||
|
FileVersion.__table__,
|
||||||
|
FileFormEvidenceGrant.__table__,
|
||||||
|
ChangeSequenceEntry.__table__,
|
||||||
|
],
|
||||||
|
)
|
||||||
|
self.session = Session(self.engine)
|
||||||
|
user_ref = UserRef(
|
||||||
|
id="user-1",
|
||||||
|
account_id="account-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
email="user@example.test",
|
||||||
|
display_name="Evidence Custodian",
|
||||||
|
)
|
||||||
|
self.provider = FilesFormEvidenceProvider(
|
||||||
|
_Registry(_Directory(user_ref)),
|
||||||
|
SimpleNamespace(file_upload_max_bytes=2_000_000),
|
||||||
|
)
|
||||||
|
self.principal = SimpleNamespace(tenant_id="tenant-1")
|
||||||
|
self.definition_ref = InstitutionalReference(
|
||||||
|
kind="form",
|
||||||
|
owner_module="forms",
|
||||||
|
object_id="form-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
version="3",
|
||||||
|
)
|
||||||
|
|
||||||
|
def tearDown(self) -> None:
|
||||||
|
self.session.close()
|
||||||
|
self.engine.dispose()
|
||||||
|
|
||||||
|
def request(self, **overrides: object) -> FormEvidenceGrantRequest:
|
||||||
|
values: dict[str, object] = {
|
||||||
|
"tenant_id": "tenant-1",
|
||||||
|
"instance_id": "instance-1",
|
||||||
|
"definition_ref": self.definition_ref,
|
||||||
|
"evidence_kind": "document",
|
||||||
|
"purpose": "supporting document",
|
||||||
|
"idempotency_key": "grant-1",
|
||||||
|
"expires_at": datetime.now(UTC) + timedelta(hours=1),
|
||||||
|
"custodian_ref": "user:user-1",
|
||||||
|
"max_size_bytes": 1_000_000,
|
||||||
|
"allowed_content_types": ("application/pdf",),
|
||||||
|
}
|
||||||
|
values.update(overrides)
|
||||||
|
return FormEvidenceGrantRequest(**values) # type: ignore[arg-type]
|
||||||
|
|
||||||
|
def test_grant_persists_only_token_hash_and_replays_without_secret(self) -> None:
|
||||||
|
request = self.request()
|
||||||
|
issued = self.provider.create_upload_grant(
|
||||||
|
self.session,
|
||||||
|
self.principal,
|
||||||
|
request=request,
|
||||||
|
)
|
||||||
|
row = self.session.query(FileFormEvidenceGrant).one()
|
||||||
|
|
||||||
|
self.assertIsNotNone(issued.upload_token)
|
||||||
|
self.assertNotEqual(issued.upload_token, row.token_sha256)
|
||||||
|
self.assertLessEqual(
|
||||||
|
issued.expires_at,
|
||||||
|
datetime.now(UTC) + timedelta(minutes=15, seconds=1),
|
||||||
|
)
|
||||||
|
replay = self.provider.create_upload_grant(
|
||||||
|
self.session,
|
||||||
|
self.principal,
|
||||||
|
request=request,
|
||||||
|
)
|
||||||
|
self.assertTrue(replay.replayed)
|
||||||
|
self.assertIsNone(replay.upload_token)
|
||||||
|
self.assertEqual(issued.grant_id, replay.grant_id)
|
||||||
|
|
||||||
|
def test_grant_capacity_counts_outstanding_uploads_but_allows_replay(self) -> None:
|
||||||
|
request = self.request(metadata={"remaining_attachments": 1})
|
||||||
|
self.provider.create_upload_grant(self.session, self.principal, request=request)
|
||||||
|
|
||||||
|
replay = self.provider.create_upload_grant(
|
||||||
|
self.session,
|
||||||
|
self.principal,
|
||||||
|
request=request,
|
||||||
|
)
|
||||||
|
self.assertTrue(replay.replayed)
|
||||||
|
with self.assertRaisesRegex(
|
||||||
|
FormEvidenceContractError,
|
||||||
|
"maximum number of active attachment uploads",
|
||||||
|
):
|
||||||
|
self.provider.create_upload_grant(
|
||||||
|
self.session,
|
||||||
|
self.principal,
|
||||||
|
request=self.request(
|
||||||
|
idempotency_key="grant-2",
|
||||||
|
metadata={"remaining_attachments": 1},
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_idempotency_key_cannot_be_reused_for_another_request(self) -> None:
|
||||||
|
self.provider.create_upload_grant(
|
||||||
|
self.session,
|
||||||
|
self.principal,
|
||||||
|
request=self.request(),
|
||||||
|
)
|
||||||
|
with self.assertRaisesRegex(
|
||||||
|
FormEvidenceContractError,
|
||||||
|
"idempotency conflict",
|
||||||
|
):
|
||||||
|
self.provider.create_upload_grant(
|
||||||
|
self.session,
|
||||||
|
self.principal,
|
||||||
|
request=self.request(purpose="another purpose"),
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_inspection_accepts_only_exact_verified_managed_version(self) -> None:
|
||||||
|
issued = self.provider.create_upload_grant(
|
||||||
|
self.session,
|
||||||
|
self.principal,
|
||||||
|
request=self.request(),
|
||||||
|
)
|
||||||
|
grant = self.session.get(FileFormEvidenceGrant, issued.grant_id)
|
||||||
|
assert grant is not None
|
||||||
|
blob = FileBlob(
|
||||||
|
id="blob-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
storage_backend="local",
|
||||||
|
storage_key="tenant-1/blob-1",
|
||||||
|
checksum_sha256="a" * 64,
|
||||||
|
size_bytes=100,
|
||||||
|
integrity_status="verified",
|
||||||
|
)
|
||||||
|
asset = FileAsset(
|
||||||
|
id="asset-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
owner_type="user",
|
||||||
|
owner_user_id="user-1",
|
||||||
|
current_version_id="version-1",
|
||||||
|
display_path="Form submissions/instance-1/evidence.pdf",
|
||||||
|
filename="evidence.pdf",
|
||||||
|
)
|
||||||
|
version = FileVersion(
|
||||||
|
id="version-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
file_asset_id="asset-1",
|
||||||
|
blob_id="blob-1",
|
||||||
|
version_number=1,
|
||||||
|
filename_at_upload="evidence.pdf",
|
||||||
|
display_path_at_upload=asset.display_path,
|
||||||
|
content_type="application/pdf",
|
||||||
|
size_bytes=100,
|
||||||
|
checksum_sha256="a" * 64,
|
||||||
|
)
|
||||||
|
self.session.add_all((blob, asset, version))
|
||||||
|
grant.status = "uploaded"
|
||||||
|
grant.file_asset_id = asset.id
|
||||||
|
grant.file_version_id = version.id
|
||||||
|
self.session.flush()
|
||||||
|
reference = EvidenceReference(
|
||||||
|
kind="document",
|
||||||
|
owner_module="files",
|
||||||
|
evidence_id=asset.id,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
version=version.id,
|
||||||
|
checksum="a" * 64,
|
||||||
|
)
|
||||||
|
|
||||||
|
accepted = self.provider.inspect_evidence(
|
||||||
|
self.session,
|
||||||
|
self.principal,
|
||||||
|
request=FormEvidenceInspectionRequest(
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
instance_id="instance-1",
|
||||||
|
definition_ref=self.definition_ref,
|
||||||
|
evidence=reference,
|
||||||
|
purpose="final submission",
|
||||||
|
final=True,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
self.assertTrue(accepted.accepted)
|
||||||
|
|
||||||
|
rejected = self.provider.inspect_evidence(
|
||||||
|
self.session,
|
||||||
|
self.principal,
|
||||||
|
request=FormEvidenceInspectionRequest(
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
instance_id="another-instance",
|
||||||
|
definition_ref=self.definition_ref,
|
||||||
|
evidence=reference,
|
||||||
|
purpose="final submission",
|
||||||
|
final=True,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
self.assertEqual("rejected", rejected.state)
|
||||||
|
|
||||||
|
def test_public_upload_consumes_grant_and_returns_exact_evidence(self) -> None:
|
||||||
|
issued = self.provider.create_upload_grant(
|
||||||
|
self.session,
|
||||||
|
self.principal,
|
||||||
|
request=self.request(),
|
||||||
|
)
|
||||||
|
assert issued.upload_token is not None
|
||||||
|
stored = SimpleNamespace(
|
||||||
|
asset=SimpleNamespace(id="asset-uploaded"),
|
||||||
|
version=SimpleNamespace(
|
||||||
|
id="version-uploaded",
|
||||||
|
checksum_sha256="c" * 64,
|
||||||
|
size_bytes=8,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
upload = UploadFile(
|
||||||
|
filename="evidence.pdf",
|
||||||
|
file=BytesIO(b"evidence"),
|
||||||
|
headers=Headers({"content-type": "application/pdf"}),
|
||||||
|
)
|
||||||
|
|
||||||
|
with (
|
||||||
|
patch(
|
||||||
|
"govoplan_files.backend.routes.form_evidence.create_file_asset",
|
||||||
|
return_value=stored,
|
||||||
|
) as create,
|
||||||
|
patch("govoplan_files.backend.routes.form_evidence.emit_platform_event"),
|
||||||
|
):
|
||||||
|
response = upload_form_evidence(
|
||||||
|
file=upload,
|
||||||
|
x_form_evidence_token=issued.upload_token,
|
||||||
|
session=self.session,
|
||||||
|
)
|
||||||
|
|
||||||
|
grant = self.session.get(FileFormEvidenceGrant, issued.grant_id)
|
||||||
|
assert grant is not None
|
||||||
|
self.assertEqual("uploaded", grant.status)
|
||||||
|
self.assertEqual("asset-uploaded", grant.file_asset_id)
|
||||||
|
self.assertEqual("version-uploaded", grant.file_version_id)
|
||||||
|
self.assertEqual("asset-uploaded", response["evidence"]["evidence_id"])
|
||||||
|
self.assertEqual(b"evidence", create.call_args.kwargs["data"])
|
||||||
|
self.assertEqual("user-1", create.call_args.kwargs["owner_id"])
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,211 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from fastapi import HTTPException
|
||||||
|
from sqlalchemy import create_engine
|
||||||
|
from sqlalchemy.orm import sessionmaker
|
||||||
|
|
||||||
|
from govoplan_access.backend.db.models import Account, Group, User
|
||||||
|
from govoplan_core.db.base import Base
|
||||||
|
from govoplan_files.backend.db.models import (
|
||||||
|
FileBlob,
|
||||||
|
FileIntegrityFinding,
|
||||||
|
FileIntegrityScan,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.routes.integrity import _assert_expected_revision
|
||||||
|
from govoplan_files.backend.storage.backends import LocalFilesystemStorageBackend
|
||||||
|
from govoplan_files.backend.storage.common import FileStorageError
|
||||||
|
from govoplan_files.backend.storage.integrity import (
|
||||||
|
cleanup_orphan_finding,
|
||||||
|
create_integrity_scan,
|
||||||
|
read_verified_blob_bytes,
|
||||||
|
recheck_integrity_finding,
|
||||||
|
run_integrity_scan_batch,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
TENANT_ID = "tenant-1"
|
||||||
|
USER_ID = "user-1"
|
||||||
|
|
||||||
|
|
||||||
|
class IntegrityReconciliationTests(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
self.temporary_directory = tempfile.TemporaryDirectory()
|
||||||
|
self.addCleanup(self.temporary_directory.cleanup)
|
||||||
|
self.backend = LocalFilesystemStorageBackend(
|
||||||
|
Path(self.temporary_directory.name)
|
||||||
|
)
|
||||||
|
self.engine = create_engine("sqlite:///:memory:", future=True)
|
||||||
|
Base.metadata.create_all(
|
||||||
|
bind=self.engine,
|
||||||
|
tables=[
|
||||||
|
Account.__table__,
|
||||||
|
User.__table__,
|
||||||
|
Group.__table__,
|
||||||
|
FileBlob.__table__,
|
||||||
|
FileIntegrityScan.__table__,
|
||||||
|
FileIntegrityFinding.__table__,
|
||||||
|
],
|
||||||
|
)
|
||||||
|
self.session = sessionmaker(bind=self.engine, future=True)()
|
||||||
|
self.addCleanup(self._close)
|
||||||
|
|
||||||
|
def _close(self) -> None:
|
||||||
|
self.session.close()
|
||||||
|
self.engine.dispose()
|
||||||
|
|
||||||
|
def test_scan_is_bounded_resumable_and_reconciles_both_orphan_directions(
|
||||||
|
self,
|
||||||
|
) -> None:
|
||||||
|
valid_data = b"valid"
|
||||||
|
restored_data = b"restore-me"
|
||||||
|
expected_corrupt_data = b"expected"
|
||||||
|
stored_corrupt_data = b"corrupt!"
|
||||||
|
valid = _blob("blob-1", "valid.bin", valid_data)
|
||||||
|
missing = _blob("blob-2", "missing.bin", restored_data)
|
||||||
|
corrupt = _blob("blob-3", "corrupt.bin", expected_corrupt_data)
|
||||||
|
self.session.add_all([valid, missing, corrupt])
|
||||||
|
self.session.commit()
|
||||||
|
self.backend.put_bytes(valid.storage_key, valid_data)
|
||||||
|
self.backend.put_bytes(corrupt.storage_key, stored_corrupt_data)
|
||||||
|
orphan_key = f"tenants/{TENANT_ID}/files/orphan.bin"
|
||||||
|
self.backend.put_bytes(orphan_key, b"orphan")
|
||||||
|
|
||||||
|
scan = create_integrity_scan(
|
||||||
|
self.session,
|
||||||
|
tenant_id=TENANT_ID,
|
||||||
|
user_id=USER_ID,
|
||||||
|
batch_size=1,
|
||||||
|
backend=self.backend,
|
||||||
|
)
|
||||||
|
self.session.commit()
|
||||||
|
self.assertEqual(1, scan.revision)
|
||||||
|
|
||||||
|
invocations = 0
|
||||||
|
while scan.status != "completed":
|
||||||
|
run_integrity_scan_batch(
|
||||||
|
self.session,
|
||||||
|
scan,
|
||||||
|
backend=self.backend,
|
||||||
|
)
|
||||||
|
self.session.commit()
|
||||||
|
invocations += 1
|
||||||
|
scan = self.session.get(FileIntegrityScan, scan.id)
|
||||||
|
self.assertIsNotNone(scan)
|
||||||
|
self.session.expire_all()
|
||||||
|
if invocations > 12:
|
||||||
|
self.fail("Integrity scan did not complete")
|
||||||
|
|
||||||
|
self.assertGreater(invocations, 3)
|
||||||
|
self.assertEqual(1 + invocations, scan.revision)
|
||||||
|
self.assertEqual(3, scan.scanned_blob_count)
|
||||||
|
self.assertEqual(1, scan.verified_blob_count)
|
||||||
|
self.assertEqual(2, scan.quarantined_blob_count)
|
||||||
|
self.assertEqual(3, scan.scanned_object_count)
|
||||||
|
self.assertEqual(1, scan.orphan_object_count)
|
||||||
|
findings = (
|
||||||
|
self.session.query(FileIntegrityFinding)
|
||||||
|
.filter(FileIntegrityFinding.scan_id == scan.id)
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
{"missing", "checksum_mismatch", "orphan_object"},
|
||||||
|
{finding.kind for finding in findings},
|
||||||
|
)
|
||||||
|
|
||||||
|
missing = self.session.get(FileBlob, missing.id)
|
||||||
|
corrupt = self.session.get(FileBlob, corrupt.id)
|
||||||
|
self.assertEqual("missing", missing.integrity_status)
|
||||||
|
self.assertEqual("checksum_mismatch", corrupt.integrity_status)
|
||||||
|
with self.assertRaisesRegex(FileStorageError, "quarantined"):
|
||||||
|
read_verified_blob_bytes(missing, backend=self.backend)
|
||||||
|
|
||||||
|
missing_finding = next(
|
||||||
|
finding for finding in findings if finding.kind == "missing"
|
||||||
|
)
|
||||||
|
self.backend.put_bytes(missing.storage_key, restored_data)
|
||||||
|
preview = recheck_integrity_finding(
|
||||||
|
self.session,
|
||||||
|
missing_finding,
|
||||||
|
user_id=USER_ID,
|
||||||
|
dry_run=True,
|
||||||
|
backend=self.backend,
|
||||||
|
)
|
||||||
|
self.assertTrue(preview.inspection.valid)
|
||||||
|
self.assertEqual("open", missing_finding.state)
|
||||||
|
repaired = recheck_integrity_finding(
|
||||||
|
self.session,
|
||||||
|
missing_finding,
|
||||||
|
user_id=USER_ID,
|
||||||
|
dry_run=False,
|
||||||
|
backend=self.backend,
|
||||||
|
)
|
||||||
|
self.session.commit()
|
||||||
|
self.assertTrue(repaired.changed)
|
||||||
|
self.assertEqual("resolved", missing_finding.state)
|
||||||
|
self.assertEqual(
|
||||||
|
restored_data,
|
||||||
|
read_verified_blob_bytes(missing, backend=self.backend),
|
||||||
|
)
|
||||||
|
|
||||||
|
orphan_finding = next(
|
||||||
|
finding for finding in findings if finding.kind == "orphan_object"
|
||||||
|
)
|
||||||
|
preview_cleanup = cleanup_orphan_finding(
|
||||||
|
self.session,
|
||||||
|
orphan_finding,
|
||||||
|
user_id=USER_ID,
|
||||||
|
dry_run=True,
|
||||||
|
backend=self.backend,
|
||||||
|
)
|
||||||
|
self.assertEqual("would_delete", preview_cleanup.action)
|
||||||
|
self.assertTrue(self.backend.exists(orphan_key))
|
||||||
|
with patch(
|
||||||
|
"govoplan_files.backend.storage.integrity.begin_orphan_cleanup_recovery"
|
||||||
|
):
|
||||||
|
cleanup = cleanup_orphan_finding(
|
||||||
|
self.session,
|
||||||
|
orphan_finding,
|
||||||
|
user_id=USER_ID,
|
||||||
|
dry_run=False,
|
||||||
|
backend=self.backend,
|
||||||
|
)
|
||||||
|
repeated = cleanup_orphan_finding(
|
||||||
|
self.session,
|
||||||
|
orphan_finding,
|
||||||
|
user_id=USER_ID,
|
||||||
|
dry_run=False,
|
||||||
|
backend=self.backend,
|
||||||
|
)
|
||||||
|
self.assertEqual("deleted", cleanup.action)
|
||||||
|
self.assertFalse(self.backend.exists(orphan_key))
|
||||||
|
self.assertEqual("already_deleted", repeated.action)
|
||||||
|
self.assertFalse(repeated.changed)
|
||||||
|
|
||||||
|
def test_stale_integrity_action_revision_is_rejected(self) -> None:
|
||||||
|
with self.assertRaises(HTTPException) as captured:
|
||||||
|
_assert_expected_revision(3, 2)
|
||||||
|
|
||||||
|
self.assertEqual(409, captured.exception.status_code)
|
||||||
|
self.assertIn("reload", str(captured.exception.detail).lower())
|
||||||
|
|
||||||
|
|
||||||
|
def _blob(blob_id: str, filename: str, expected_data: bytes) -> FileBlob:
|
||||||
|
return FileBlob(
|
||||||
|
id=blob_id,
|
||||||
|
tenant_id=TENANT_ID,
|
||||||
|
storage_backend="local",
|
||||||
|
storage_key=f"tenants/{TENANT_ID}/files/{filename}",
|
||||||
|
checksum_sha256=hashlib.sha256(expected_data).hexdigest(),
|
||||||
|
size_bytes=len(expected_data),
|
||||||
|
ref_count=1,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -4,13 +4,24 @@ import unittest
|
|||||||
|
|
||||||
|
|
||||||
STATIC_TOPIC_IDS = {
|
STATIC_TOPIC_IDS = {
|
||||||
|
"files.configuration-package.managed-storage",
|
||||||
|
"files.quick-access-and-product-area",
|
||||||
|
"files.search.managed-content",
|
||||||
"files.workflow.organize-managed-files",
|
"files.workflow.organize-managed-files",
|
||||||
"files.workflow.find-and-download-files",
|
"files.workflow.find-and-download-files",
|
||||||
"files.workflow.share-managed-files",
|
"files.workflow.share-managed-files",
|
||||||
"files.workflow.delete-managed-files",
|
"files.workflow.delete-managed-files",
|
||||||
|
"files.workflow.restore-retain-and-purge",
|
||||||
|
"files.privacy.data-subject-requests",
|
||||||
"files.governed-connectors-and-provenance",
|
"files.governed-connectors-and-provenance",
|
||||||
"files.reference.integrity-recovery-and-fail-closed-transports",
|
"files.reference.integrity-recovery-and-fail-closed-transports",
|
||||||
|
"files.reference.shared-storage-profile",
|
||||||
|
"files.reference.generated-artifact-store",
|
||||||
"files.reference.snapshot-provenance-and-capabilities",
|
"files.reference.snapshot-provenance-and-capabilities",
|
||||||
|
"files.records.exact-version-source",
|
||||||
|
"files.forms-runtime.managed-evidence",
|
||||||
|
"files.postbox.exact-version-references",
|
||||||
|
"files.tabular-content",
|
||||||
"files.assurance.process-and-release-readiness",
|
"files.assurance.process-and-release-readiness",
|
||||||
}
|
}
|
||||||
RUNTIME_TOPIC_IDS = {
|
RUNTIME_TOPIC_IDS = {
|
||||||
@@ -103,17 +114,30 @@ class FilesManifestDocumentationTests(unittest.TestCase):
|
|||||||
def test_share_and_delete_tasks_state_current_boundaries(self) -> None:
|
def test_share_and_delete_tasks_state_current_boundaries(self) -> None:
|
||||||
share = self.topic("files.workflow.share-managed-files")
|
share = self.topic("files.workflow.share-managed-files")
|
||||||
self.assertEqual("available", share.layer)
|
self.assertEqual("available", share.layer)
|
||||||
self.assertIn("does not yet provide a general share editor", share.body)
|
self.assertIn("Expired and revoked grants stop authorizing", share.body)
|
||||||
self.assertIn("no share-revocation route", share.body)
|
self.assertIn("revocation is idempotent", share.body)
|
||||||
self.assertIn(
|
self.assertIn(
|
||||||
"/api/v1/files/{file_id}/shares", {link.href for link in share.links}
|
"/api/v1/files/{file_id}/shares", {link.href for link in share.links}
|
||||||
)
|
)
|
||||||
|
|
||||||
delete = self.topic("files.workflow.delete-managed-files")
|
delete = self.topic("files.workflow.delete-managed-files")
|
||||||
self.assertIn("Soft-delete", delete.summary)
|
self.assertIn("Soft-delete", delete.summary)
|
||||||
self.assertIn("no self-service restore or hard-purge", delete.body)
|
self.assertIn("Authorized restoration", delete.body)
|
||||||
self.assertTrue(
|
self.assertTrue(
|
||||||
any("not a hard purge" in item for item in delete.metadata["limitations"])
|
any("soft deletion" in item.casefold() for item in delete.metadata["limitations"])
|
||||||
|
)
|
||||||
|
|
||||||
|
lifecycle = self.topic("files.workflow.restore-retain-and-purge")
|
||||||
|
self.assertIn("legal hold", lifecycle.body)
|
||||||
|
self.assertIn("preview hash", lifecycle.body)
|
||||||
|
self.assertIn("recovery ledger", lifecycle.body)
|
||||||
|
|
||||||
|
privacy = self.topic("files.privacy.data-subject-requests")
|
||||||
|
self.assertEqual(("admin",), privacy.documentation_types)
|
||||||
|
self.assertIn("without raw file bytes", privacy.body)
|
||||||
|
self.assertIn("separately authorized Files purge", privacy.body)
|
||||||
|
self.assertTrue(
|
||||||
|
any("membership" in item for item in privacy.metadata["limitations"])
|
||||||
)
|
)
|
||||||
|
|
||||||
def test_admin_topic_covers_policy_redaction_and_atomic_credential_deletion(
|
def test_admin_topic_covers_policy_redaction_and_atomic_credential_deletion(
|
||||||
@@ -125,9 +149,28 @@ class FilesManifestDocumentationTests(unittest.TestCase):
|
|||||||
self.assertEqual("reference", topic.metadata["kind"])
|
self.assertEqual("reference", topic.metadata["kind"])
|
||||||
self.assertEqual("File connections", topic.metadata["screen"])
|
self.assertEqual("File connections", topic.metadata["screen"])
|
||||||
self.assertTrue(topic.metadata["section"])
|
self.assertTrue(topic.metadata["section"])
|
||||||
|
self.assertEqual(
|
||||||
|
{
|
||||||
|
"files.connectors",
|
||||||
|
"files.connector.credentials",
|
||||||
|
"files.connector.policy",
|
||||||
|
"files.connector-folder-sync.remote-path",
|
||||||
|
"files.connector-folder-sync.target-folder",
|
||||||
|
"files.connector-folder-sync.conflict-strategy",
|
||||||
|
"files.connector-folder-sync.max-files",
|
||||||
|
"files.connector-folder-sync.recursive",
|
||||||
|
},
|
||||||
|
set(topic.metadata["help_contexts"]),
|
||||||
|
)
|
||||||
self.assertIn("deny rules win", topic.body)
|
self.assertIn("deny rules win", topic.body)
|
||||||
self.assertIn("redact secret values", topic.body)
|
self.assertIn("redact secret values", topic.body)
|
||||||
self.assertIn("same transaction", topic.body)
|
self.assertIn("same transaction", topic.body)
|
||||||
|
self.assertIn("read-only by default", topic.body)
|
||||||
|
self.assertIn("S3", topic.body)
|
||||||
|
self.assertIn("manual connector-folder sync", topic.body)
|
||||||
|
self.assertIn("Per-file savepoints", topic.body)
|
||||||
|
self.assertIn("files.connector.folder_synced", topic.body)
|
||||||
|
self.assertIn("Automatic remote deletion", topic.body)
|
||||||
self.assertTrue(
|
self.assertTrue(
|
||||||
any(
|
any(
|
||||||
"non-owned external references" in item
|
"non-owned external references" in item
|
||||||
@@ -141,8 +184,13 @@ class FilesManifestDocumentationTests(unittest.TestCase):
|
|||||||
self.assertIn(
|
self.assertIn(
|
||||||
"/api/v1/files/connectors/credentials", {link.href for link in topic.links}
|
"/api/v1/files/connectors/credentials", {link.href for link in topic.links}
|
||||||
)
|
)
|
||||||
|
self.assertIn(
|
||||||
|
"/api/v1/files/connector-spaces/{space_id}/sync",
|
||||||
|
{link.href for link in topic.links},
|
||||||
|
)
|
||||||
|
self.assertIn("de", topic.translations)
|
||||||
|
|
||||||
def test_operator_topic_covers_recovery_and_fail_closed_s3_smb(self) -> None:
|
def test_operator_topic_covers_recovery_and_pinned_s3_smb(self) -> None:
|
||||||
topic = self.topic(
|
topic = self.topic(
|
||||||
"files.reference.integrity-recovery-and-fail-closed-transports"
|
"files.reference.integrity-recovery-and-fail-closed-transports"
|
||||||
)
|
)
|
||||||
@@ -153,9 +201,31 @@ class FilesManifestDocumentationTests(unittest.TestCase):
|
|||||||
self.assertIn("SMB", topic.body)
|
self.assertIn("SMB", topic.body)
|
||||||
self.assertIn("fail closed", topic.body)
|
self.assertIn("fail closed", topic.body)
|
||||||
self.assertIn("DFS referrals", topic.body)
|
self.assertIn("DFS referrals", topic.body)
|
||||||
|
self.assertIn("ambient credential discovery", topic.body)
|
||||||
self.assertIn("does not remove backend blob objects", topic.body)
|
self.assertIn("does not remove backend blob objects", topic.body)
|
||||||
|
self.assertIn("bounded resumable integrity scan", topic.body)
|
||||||
|
self.assertIn("quarantined", topic.body)
|
||||||
self.assertIn("MASTER_KEY_B64", topic.metadata["recovery_unit"])
|
self.assertIn("MASTER_KEY_B64", topic.metadata["recovery_unit"])
|
||||||
|
self.assertIn(
|
||||||
|
"Encryption envelope and wrapped-key rows", topic.metadata["recovery_unit"]
|
||||||
|
)
|
||||||
|
self.assertIn("lease-fenced Core recovery", topic.body)
|
||||||
|
self.assertIn("Ops", topic.body)
|
||||||
|
self.assertIn("Hard purge", topic.body)
|
||||||
|
self.assertIn("S3 connector write-back", topic.body)
|
||||||
self.assertTrue(topic.metadata["verification"])
|
self.assertTrue(topic.metadata["verification"])
|
||||||
|
self.assertIn(
|
||||||
|
"/api/v1/files/integrity/scans",
|
||||||
|
{link.href for link in topic.links},
|
||||||
|
)
|
||||||
|
self.assertIn(
|
||||||
|
"/admin?section=tenant-file-integrity",
|
||||||
|
{link.href for link in topic.links},
|
||||||
|
)
|
||||||
|
self.assertIn(
|
||||||
|
"files.admin.tenant-integrity",
|
||||||
|
{surface.id for surface in self.manifest.frontend.view_surfaces},
|
||||||
|
)
|
||||||
self.assertIn(
|
self.assertIn(
|
||||||
"GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS", topic.configuration_keys
|
"GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS", topic.configuration_keys
|
||||||
)
|
)
|
||||||
@@ -166,7 +236,11 @@ class FilesManifestDocumentationTests(unittest.TestCase):
|
|||||||
self.assertEqual(("admin", "user"), topic.documentation_types)
|
self.assertEqual(("admin", "user"), topic.documentation_types)
|
||||||
self.assertEqual("reference", topic.metadata["kind"])
|
self.assertEqual("reference", topic.metadata["kind"])
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
["files.access@0.1.6", "files.campaign_attachments@0.1.6"],
|
[
|
||||||
|
"files.access@0.1.6",
|
||||||
|
"files.campaign_attachments@0.1.6",
|
||||||
|
"forms_runtime.evidence.files@1.0.0",
|
||||||
|
],
|
||||||
topic.metadata["provided_interfaces"],
|
topic.metadata["provided_interfaces"],
|
||||||
)
|
)
|
||||||
self.assertIn("revision", topic.metadata["provenance_fields"])
|
self.assertIn("revision", topic.metadata["provenance_fields"])
|
||||||
@@ -184,8 +258,9 @@ class FilesManifestDocumentationTests(unittest.TestCase):
|
|||||||
self.assertIn("process_owner", topic.audience)
|
self.assertIn("process_owner", topic.audience)
|
||||||
self.assertIn("release_manager", topic.audience)
|
self.assertIn("release_manager", topic.audience)
|
||||||
self.assertIn("versions align", topic.body)
|
self.assertIn("versions align", topic.body)
|
||||||
self.assertIn("no general share-management UI or share revocation", topic.body)
|
self.assertIn("share lifecycle", topic.body)
|
||||||
self.assertIn("no enforced retention or legal hold", topic.body)
|
self.assertIn("legal hold", topic.body)
|
||||||
|
self.assertIn("S3 write-back", topic.body)
|
||||||
for key in ("prerequisites", "steps", "outcome", "verification"):
|
for key in ("prerequisites", "steps", "outcome", "verification"):
|
||||||
self.assertTrue(topic.metadata[key])
|
self.assertTrue(topic.metadata[key])
|
||||||
self.assertTrue(
|
self.assertTrue(
|
||||||
|
|||||||
@@ -0,0 +1,42 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from alembic.runtime.migration import MigrationContext
|
||||||
|
from sqlalchemy import create_engine, inspect
|
||||||
|
|
||||||
|
from govoplan_access.backend.manifest import get_manifest as get_access_manifest
|
||||||
|
from govoplan_core.db.migrations import migrate_database
|
||||||
|
from govoplan_files.backend.manifest import get_manifest as get_files_manifest
|
||||||
|
|
||||||
|
|
||||||
|
class FilesMigrationTests(unittest.TestCase):
|
||||||
|
def test_fresh_migration_creates_form_evidence_grants_and_head(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory(
|
||||||
|
prefix="govoplan-files-migration-"
|
||||||
|
) as directory:
|
||||||
|
url = f"sqlite:///{Path(directory) / 'files.db'}"
|
||||||
|
migrate_database(
|
||||||
|
database_url=url,
|
||||||
|
enabled_modules=("access", "files"),
|
||||||
|
manifest_factories=(get_access_manifest, get_files_manifest),
|
||||||
|
)
|
||||||
|
engine = create_engine(url)
|
||||||
|
try:
|
||||||
|
self.assertIn(
|
||||||
|
"file_form_evidence_grants",
|
||||||
|
inspect(engine).get_table_names(),
|
||||||
|
)
|
||||||
|
with engine.connect() as connection:
|
||||||
|
self.assertIn(
|
||||||
|
"a2b3c4d5e6f9",
|
||||||
|
set(MigrationContext.configure(connection).get_current_heads()),
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
engine.dispose()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,42 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from govoplan_files.backend.operational_checks import managed_storage_roundtrip_check
|
||||||
|
from govoplan_files.backend.storage.backends import LocalFilesystemStorageBackend
|
||||||
|
|
||||||
|
|
||||||
|
def test_managed_storage_roundtrip_removes_probe(tmp_path) -> None:
|
||||||
|
backend = LocalFilesystemStorageBackend(tmp_path)
|
||||||
|
|
||||||
|
with patch(
|
||||||
|
"govoplan_files.backend.operational_checks.get_storage_backend",
|
||||||
|
return_value=backend,
|
||||||
|
):
|
||||||
|
result = managed_storage_roundtrip_check()
|
||||||
|
|
||||||
|
assert result.state == "ok"
|
||||||
|
assert result.metrics["backend"] == "local"
|
||||||
|
assert list(tmp_path.rglob("*.bin")) == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_managed_storage_roundtrip_fails_closed() -> None:
|
||||||
|
class BrokenBackend:
|
||||||
|
name = "broken"
|
||||||
|
|
||||||
|
def put_bytes(self, *_args, **_kwargs):
|
||||||
|
raise OSError("private provider detail")
|
||||||
|
|
||||||
|
def delete(self, *_args, **_kwargs):
|
||||||
|
return None
|
||||||
|
|
||||||
|
with patch(
|
||||||
|
"govoplan_files.backend.operational_checks.get_storage_backend",
|
||||||
|
return_value=BrokenBackend(),
|
||||||
|
):
|
||||||
|
result = managed_storage_roundtrip_check()
|
||||||
|
|
||||||
|
assert result.state == "error"
|
||||||
|
assert result.readiness_critical is True
|
||||||
|
assert "private provider detail" not in result.detail
|
||||||
|
|
||||||
@@ -0,0 +1,160 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from types import SimpleNamespace
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from govoplan_core.core.files import PostboxFileReferenceRequest
|
||||||
|
from govoplan_files.backend.capabilities import FilesPostboxReferenceService
|
||||||
|
from govoplan_files.backend.db.models import FileAsset, FileBlob, FileVersion
|
||||||
|
from govoplan_files.backend.storage.common import FileStorageError
|
||||||
|
|
||||||
|
|
||||||
|
class _Principal:
|
||||||
|
def __init__(self, *, tenant_id: str = "tenant-1", scopes: set[str] | None = None):
|
||||||
|
self.tenant_id = tenant_id
|
||||||
|
self.user = SimpleNamespace(id="user-1")
|
||||||
|
self._scopes = {"files:file:download"} if scopes is None else scopes
|
||||||
|
|
||||||
|
def has(self, scope: str) -> bool:
|
||||||
|
return scope in self._scopes
|
||||||
|
|
||||||
|
|
||||||
|
class _Session:
|
||||||
|
def __init__(self, values: dict[tuple[type[object], str], object]):
|
||||||
|
self.values = values
|
||||||
|
|
||||||
|
def get(self, model: type[object], resource_id: str):
|
||||||
|
return self.values.get((model, resource_id))
|
||||||
|
|
||||||
|
|
||||||
|
def _request(reference_type: str = "file_version", reference_id: str = "version-1"):
|
||||||
|
return PostboxFileReferenceRequest(
|
||||||
|
postbox_id="postbox-1",
|
||||||
|
message_id="message-1",
|
||||||
|
reference_type=reference_type,
|
||||||
|
reference_id=reference_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _objects():
|
||||||
|
asset = SimpleNamespace(
|
||||||
|
id="file-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
current_version_id="version-2",
|
||||||
|
deleted_at=None,
|
||||||
|
filename="current.txt",
|
||||||
|
display_path="Evidence/current.txt",
|
||||||
|
)
|
||||||
|
version = SimpleNamespace(
|
||||||
|
id="version-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
file_asset_id="file-1",
|
||||||
|
blob_id="blob-1",
|
||||||
|
filename_at_upload="evidence.txt",
|
||||||
|
content_type="text/plain",
|
||||||
|
size_bytes=8,
|
||||||
|
checksum_sha256="a" * 64,
|
||||||
|
)
|
||||||
|
blob = SimpleNamespace(
|
||||||
|
id="blob-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
content_type="text/plain",
|
||||||
|
)
|
||||||
|
return asset, version, blob
|
||||||
|
|
||||||
|
|
||||||
|
class FilesPostboxReferenceTests(unittest.TestCase):
|
||||||
|
def test_resolves_the_referenced_version_instead_of_the_current_version(self) -> None:
|
||||||
|
asset, version, blob = _objects()
|
||||||
|
session = _Session(
|
||||||
|
{
|
||||||
|
(FileAsset, asset.id): asset,
|
||||||
|
(FileVersion, version.id): version,
|
||||||
|
(FileBlob, blob.id): blob,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
with patch(
|
||||||
|
"govoplan_files.backend.capabilities.get_asset_for_user",
|
||||||
|
return_value=asset,
|
||||||
|
):
|
||||||
|
result = FilesPostboxReferenceService().resolve_postbox_references(
|
||||||
|
session,
|
||||||
|
_Principal(),
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
requests=(_request(),),
|
||||||
|
)[0]
|
||||||
|
|
||||||
|
self.assertTrue(result.available)
|
||||||
|
self.assertEqual("version-1", result.file_version_id)
|
||||||
|
self.assertEqual(
|
||||||
|
"/api/v1/files/file-1/versions/version-1/download",
|
||||||
|
result.download_path,
|
||||||
|
)
|
||||||
|
self.assertTrue(result.provenance["exact_version"])
|
||||||
|
|
||||||
|
def test_fails_closed_for_tenant_permission_and_file_access_mismatches(self) -> None:
|
||||||
|
service = FilesPostboxReferenceService()
|
||||||
|
empty = _Session({})
|
||||||
|
|
||||||
|
tenant_result = service.resolve_postbox_references(
|
||||||
|
empty,
|
||||||
|
_Principal(tenant_id="tenant-2"),
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
requests=(_request(),),
|
||||||
|
)[0]
|
||||||
|
permission_result = service.resolve_postbox_references(
|
||||||
|
empty,
|
||||||
|
_Principal(scopes=set()),
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
requests=(_request(),),
|
||||||
|
)[0]
|
||||||
|
|
||||||
|
asset, version, blob = _objects()
|
||||||
|
populated = _Session(
|
||||||
|
{
|
||||||
|
(FileAsset, asset.id): asset,
|
||||||
|
(FileVersion, version.id): version,
|
||||||
|
(FileBlob, blob.id): blob,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
with patch(
|
||||||
|
"govoplan_files.backend.capabilities.get_asset_for_user",
|
||||||
|
side_effect=FileStorageError("denied"),
|
||||||
|
):
|
||||||
|
access_result = service.resolve_postbox_references(
|
||||||
|
populated,
|
||||||
|
_Principal(),
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
requests=(_request(),),
|
||||||
|
)[0]
|
||||||
|
|
||||||
|
self.assertEqual("tenant_mismatch", tenant_result.reason_code)
|
||||||
|
self.assertEqual("download_permission_missing", permission_result.reason_code)
|
||||||
|
self.assertEqual("file_access_denied", access_result.reason_code)
|
||||||
|
self.assertFalse(access_result.available)
|
||||||
|
|
||||||
|
def test_asset_reference_does_not_drift_to_a_later_current_version(self) -> None:
|
||||||
|
asset, version, blob = _objects()
|
||||||
|
session = _Session(
|
||||||
|
{
|
||||||
|
(FileAsset, asset.id): asset,
|
||||||
|
(FileVersion, version.id): version,
|
||||||
|
(FileBlob, blob.id): blob,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
result = FilesPostboxReferenceService().resolve_postbox_references(
|
||||||
|
session,
|
||||||
|
_Principal(),
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
requests=(_request("file", asset.id),),
|
||||||
|
)[0]
|
||||||
|
|
||||||
|
self.assertFalse(result.available)
|
||||||
|
self.assertEqual("exact_version_required", result.reason_code)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,97 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from sqlalchemy import create_engine
|
||||||
|
from sqlalchemy.orm import sessionmaker
|
||||||
|
|
||||||
|
from govoplan_access.backend.db import models as access_models # noqa: F401
|
||||||
|
from govoplan_core.core.provider_governance import ExternalProviderStateContext
|
||||||
|
from govoplan_core.db.base import Base
|
||||||
|
from govoplan_files.backend.db.models import FileConnectorProfile, FileConnectorSpace
|
||||||
|
from govoplan_files.backend.manifest import manifest
|
||||||
|
from govoplan_files.backend.provider_state import (
|
||||||
|
REMOTE_STORAGE_PROVIDER_ID,
|
||||||
|
remote_storage_provider_states,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class FilesProviderStateTests(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
self.engine = create_engine("sqlite+pysqlite:///:memory:", future=True)
|
||||||
|
Base.metadata.create_all(
|
||||||
|
self.engine,
|
||||||
|
tables=(FileConnectorProfile.__table__, FileConnectorSpace.__table__),
|
||||||
|
)
|
||||||
|
self.session = sessionmaker(bind=self.engine, expire_on_commit=False)()
|
||||||
|
self.profile = FileConnectorProfile(
|
||||||
|
id="profile-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
scope_type="tenant",
|
||||||
|
scope_id="tenant-1",
|
||||||
|
label="Remote WebDAV",
|
||||||
|
provider="webdav",
|
||||||
|
endpoint_url="https://files.example.test/dav/",
|
||||||
|
enabled=True,
|
||||||
|
)
|
||||||
|
self.session.add_all(
|
||||||
|
(
|
||||||
|
self.profile,
|
||||||
|
FileConnectorSpace(
|
||||||
|
id="space-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
owner_type="tenant",
|
||||||
|
label="Documents",
|
||||||
|
connector_profile_id=self.profile.id,
|
||||||
|
provider="webdav",
|
||||||
|
remote_path="/documents",
|
||||||
|
read_only=True,
|
||||||
|
is_active=True,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
self.session.commit()
|
||||||
|
|
||||||
|
def tearDown(self) -> None:
|
||||||
|
self.session.close()
|
||||||
|
self.engine.dispose()
|
||||||
|
|
||||||
|
def test_state_does_not_overstate_live_health_or_expose_endpoint(self) -> None:
|
||||||
|
state = remote_storage_provider_states(
|
||||||
|
ExternalProviderStateContext(session=self.session, tenant_id="tenant-1")
|
||||||
|
)[0]
|
||||||
|
|
||||||
|
self.assertEqual("external_mirror", state.authority_mode)
|
||||||
|
self.assertEqual("unknown", state.health)
|
||||||
|
self.assertEqual("attention", state.recovery)
|
||||||
|
self.assertEqual(1, state.metrics["active_spaces"])
|
||||||
|
self.assertNotIn("files.example.test", str(state.to_dict()))
|
||||||
|
self.assertEqual(
|
||||||
|
(),
|
||||||
|
remote_storage_provider_states(
|
||||||
|
ExternalProviderStateContext(
|
||||||
|
session=self.session,
|
||||||
|
tenant_id="tenant-2",
|
||||||
|
)
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_unimplemented_profile_is_fail_closed_and_manifest_registered(self) -> None:
|
||||||
|
self.profile.provider = "sharepoint"
|
||||||
|
self.session.flush()
|
||||||
|
|
||||||
|
state = remote_storage_provider_states(
|
||||||
|
ExternalProviderStateContext(session=self.session, tenant_id="tenant-1")
|
||||||
|
)[0]
|
||||||
|
|
||||||
|
self.assertEqual("error", state.health)
|
||||||
|
self.assertEqual("unsupported", state.recovery)
|
||||||
|
self.assertEqual(REMOTE_STORAGE_PROVIDER_ID, manifest.external_providers[0].id)
|
||||||
|
self.assertEqual(
|
||||||
|
REMOTE_STORAGE_PROVIDER_ID,
|
||||||
|
manifest.external_provider_state_providers[0].provider_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,38 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from govoplan_files.backend.manifest import manifest
|
||||||
|
|
||||||
|
|
||||||
|
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
|
||||||
|
|
||||||
|
class FilesQuickAccessContractTests(unittest.TestCase):
|
||||||
|
def test_manifest_declares_exact_version_result_and_fallback(self) -> None:
|
||||||
|
tool = manifest.frontend.quick_access_tools[0]
|
||||||
|
|
||||||
|
self.assertEqual("files.recent", tool.id)
|
||||||
|
self.assertEqual(("files.file-version",), tool.returned_reference_kinds)
|
||||||
|
self.assertEqual("files.quick_access.files", tool.help_context_id)
|
||||||
|
self.assertEqual("/files", tool.full_page_path)
|
||||||
|
|
||||||
|
def test_renderer_and_page_keep_recent_selection_and_upload_bounded(self) -> None:
|
||||||
|
renderer = (
|
||||||
|
REPOSITORY_ROOT / "webui/src/features/files/FileQuickAccess.tsx"
|
||||||
|
).read_text(encoding="utf-8")
|
||||||
|
page = (
|
||||||
|
REPOSITORY_ROOT / "webui/src/features/files/FilesPage.tsx"
|
||||||
|
).read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
self.assertIn('sort: "recent", page_size: 7', renderer)
|
||||||
|
self.assertIn('kind: "file-version"', renderer)
|
||||||
|
self.assertIn("quickAccessLaunchState(launchContext)", renderer)
|
||||||
|
self.assertIn('to="/files?quickAction=upload"', renderer)
|
||||||
|
self.assertIn('parameters.get("quickAction") !== "upload"', page)
|
||||||
|
self.assertIn('openDialog("upload"', page)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,108 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from types import SimpleNamespace
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import MagicMock, patch
|
||||||
|
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from govoplan_core.core.records import RecordContractError, RecordSourceLocator
|
||||||
|
from govoplan_files.backend.db.models import FileAsset, FileBlob, FileVersion
|
||||||
|
from govoplan_files.backend.record_source import FilesRecordSource
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class Principal:
|
||||||
|
tenant_id: str = "tenant-1"
|
||||||
|
membership_id: str = "user-1"
|
||||||
|
user: object = field(default_factory=lambda: SimpleNamespace(id="user-1"))
|
||||||
|
|
||||||
|
def has(self, scope: str) -> bool:
|
||||||
|
return scope in {"files:file:read"}
|
||||||
|
|
||||||
|
|
||||||
|
class FilesRecordSourceTests(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
self.session = MagicMock(spec=Session)
|
||||||
|
self.asset = FileAsset(
|
||||||
|
id="asset-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
owner_type="user",
|
||||||
|
owner_user_id="user-1",
|
||||||
|
current_version_id="version-1",
|
||||||
|
display_path="Evidence/Decision.pdf",
|
||||||
|
filename="Decision.pdf",
|
||||||
|
)
|
||||||
|
self.blob = FileBlob(
|
||||||
|
id="blob-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
storage_backend="local",
|
||||||
|
storage_key="tenant-1/blob-1",
|
||||||
|
checksum_sha256="a" * 64,
|
||||||
|
size_bytes=1024,
|
||||||
|
integrity_status="verified",
|
||||||
|
)
|
||||||
|
self.version = FileVersion(
|
||||||
|
id="version-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
file_asset_id="asset-1",
|
||||||
|
blob_id="blob-1",
|
||||||
|
version_number=1,
|
||||||
|
filename_at_upload="Decision.pdf",
|
||||||
|
display_path_at_upload="Evidence/Decision.pdf",
|
||||||
|
content_type="application/pdf",
|
||||||
|
size_bytes=1024,
|
||||||
|
checksum_sha256="a" * 64,
|
||||||
|
)
|
||||||
|
self.version.created_at = datetime.now(UTC)
|
||||||
|
version_query = self.session.query.return_value.filter.return_value
|
||||||
|
version_query.filter.return_value.one_or_none.return_value = self.version
|
||||||
|
self.session.get.return_value = self.blob
|
||||||
|
|
||||||
|
def locator(self) -> RecordSourceLocator:
|
||||||
|
return RecordSourceLocator(
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
source_module="files",
|
||||||
|
resource_type="file_version",
|
||||||
|
resource_id="asset-1",
|
||||||
|
source_revision="version-1",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_resolves_exact_authorized_version_and_digest(self) -> None:
|
||||||
|
with patch(
|
||||||
|
"govoplan_files.backend.record_source.get_asset_for_user",
|
||||||
|
return_value=self.asset,
|
||||||
|
):
|
||||||
|
result = FilesRecordSource().resolve(
|
||||||
|
self.session,
|
||||||
|
Principal(),
|
||||||
|
locator=self.locator(),
|
||||||
|
purpose="document decision basis",
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual("version-1", result.locator.source_revision)
|
||||||
|
self.assertEqual("a" * 64, result.content_sha256)
|
||||||
|
self.assertEqual("verified", result.metadata["integrity_status"])
|
||||||
|
|
||||||
|
def test_quarantined_version_fails_closed(self) -> None:
|
||||||
|
self.blob.quarantined_at = datetime.now(UTC)
|
||||||
|
self.session.flush()
|
||||||
|
with (
|
||||||
|
patch(
|
||||||
|
"govoplan_files.backend.record_source.get_asset_for_user",
|
||||||
|
return_value=self.asset,
|
||||||
|
),
|
||||||
|
self.assertRaisesRegex(RecordContractError, "integrity gate"),
|
||||||
|
):
|
||||||
|
FilesRecordSource().resolve(
|
||||||
|
self.session,
|
||||||
|
Principal(),
|
||||||
|
locator=self.locator(),
|
||||||
|
purpose="document decision basis",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -1,13 +1,79 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import unittest
|
import unittest
|
||||||
|
from collections import Counter
|
||||||
|
from inspect import signature
|
||||||
|
|
||||||
from govoplan_files.backend.router import router
|
from govoplan_files.backend.router import router
|
||||||
|
from govoplan_files.backend.routes.assets import router as assets_router
|
||||||
|
from govoplan_files.backend.routes.connector_io import router as connector_io_router
|
||||||
|
from govoplan_files.backend.routes.connector_profiles import (
|
||||||
|
router as connector_profiles_router,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.routes.connector_settings import (
|
||||||
|
router as connector_settings_router,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.routes.folders import router as folders_router
|
||||||
|
from govoplan_files.backend.routes.form_evidence import router as form_evidence_router
|
||||||
|
from govoplan_files.backend.routes.integrity import router as integrity_router
|
||||||
|
from govoplan_files.backend.routes.listing import router as listing_router
|
||||||
|
from govoplan_files.backend.routes.lifecycle import router as lifecycle_router
|
||||||
|
from govoplan_files.backend.routes.shares import router as shares_router
|
||||||
|
from govoplan_files.backend.routes.spaces import router as spaces_router
|
||||||
|
from govoplan_files.backend.routes.transfers import router as transfers_router
|
||||||
|
from govoplan_files.backend.routes.uploads import router as uploads_router
|
||||||
|
|
||||||
|
|
||||||
class FilesRouterContractTests(unittest.TestCase):
|
class FilesRouterContractTests(unittest.TestCase):
|
||||||
|
@staticmethod
|
||||||
|
def _operation_keys(candidate_router) -> list[tuple[str, str]]:
|
||||||
|
return [
|
||||||
|
(method, route.path)
|
||||||
|
for route in candidate_router.routes
|
||||||
|
for method in sorted(route.methods or ())
|
||||||
|
]
|
||||||
|
|
||||||
|
def test_composed_router_contains_every_workflow_operation_once(self) -> None:
|
||||||
|
workflow_routers = (
|
||||||
|
spaces_router,
|
||||||
|
folders_router,
|
||||||
|
form_evidence_router,
|
||||||
|
integrity_router,
|
||||||
|
lifecycle_router,
|
||||||
|
listing_router,
|
||||||
|
uploads_router,
|
||||||
|
connector_settings_router,
|
||||||
|
connector_io_router,
|
||||||
|
connector_profiles_router,
|
||||||
|
assets_router,
|
||||||
|
shares_router,
|
||||||
|
transfers_router,
|
||||||
|
)
|
||||||
|
expected = [
|
||||||
|
operation
|
||||||
|
for workflow_router in workflow_routers
|
||||||
|
for operation in self._operation_keys(workflow_router)
|
||||||
|
]
|
||||||
|
actual = self._operation_keys(router)
|
||||||
|
|
||||||
|
self.assertEqual(expected, actual)
|
||||||
|
self.assertEqual(63, len(actual))
|
||||||
|
self.assertFalse(
|
||||||
|
[operation for operation, count in Counter(actual).items() if count > 1]
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_archive_preview_and_confirmation_routes_are_exposed(self) -> None:
|
||||||
|
routes = {
|
||||||
|
(tuple(sorted(route.methods or ())), route.path) for route in router.routes
|
||||||
|
}
|
||||||
|
|
||||||
|
self.assertIn((("POST",), "/files/archive-preview"), routes)
|
||||||
|
self.assertIn((("POST",), "/files/archive-confirm"), routes)
|
||||||
|
|
||||||
def test_connector_routes_keep_existing_api_paths(self) -> None:
|
def test_connector_routes_keep_existing_api_paths(self) -> None:
|
||||||
routes = {(tuple(sorted(route.methods or ())), route.path) for route in router.routes}
|
routes = {
|
||||||
|
(tuple(sorted(route.methods or ())), route.path) for route in router.routes
|
||||||
|
}
|
||||||
|
|
||||||
expected = {
|
expected = {
|
||||||
(("GET",), "/files/connectors/providers"),
|
(("GET",), "/files/connectors/providers"),
|
||||||
@@ -17,20 +83,70 @@ class FilesRouterContractTests(unittest.TestCase):
|
|||||||
(("GET",), "/files/connectors/profiles/{profile_id}/browse"),
|
(("GET",), "/files/connectors/profiles/{profile_id}/browse"),
|
||||||
(("POST",), "/files/connectors/profiles/{profile_id}/import"),
|
(("POST",), "/files/connectors/profiles/{profile_id}/import"),
|
||||||
(("POST",), "/files/connectors/profiles/{profile_id}/sync"),
|
(("POST",), "/files/connectors/profiles/{profile_id}/sync"),
|
||||||
|
(("POST",), "/files/connector-spaces/{space_id}/sync"),
|
||||||
|
(("POST",), "/files/connector-spaces/{space_id}/write-back"),
|
||||||
(("GET",), "/files/connectors/credentials"),
|
(("GET",), "/files/connectors/credentials"),
|
||||||
(("POST",), "/files/connectors/credentials"),
|
(("POST",), "/files/connectors/credentials"),
|
||||||
(("GET",), "/files/connector-spaces"),
|
(("GET",), "/files/connector-spaces"),
|
||||||
(("POST",), "/files/connector-spaces"),
|
(("POST",), "/files/connector-spaces"),
|
||||||
|
(("PATCH",), "/files/connector-spaces/{space_id}"),
|
||||||
|
(("DELETE",), "/files/connector-spaces/{space_id}"),
|
||||||
}
|
}
|
||||||
|
|
||||||
self.assertTrue(expected.issubset(routes))
|
self.assertTrue(expected.issubset(routes))
|
||||||
|
|
||||||
def test_bulk_organize_routes_keep_existing_api_paths(self) -> None:
|
def test_bulk_organize_routes_keep_existing_api_paths(self) -> None:
|
||||||
routes = {(tuple(sorted(route.methods or ())), route.path) for route in router.routes}
|
routes = {
|
||||||
|
(tuple(sorted(route.methods or ())), route.path) for route in router.routes
|
||||||
|
}
|
||||||
|
|
||||||
self.assertIn((("POST",), "/files/bulk-rename"), routes)
|
self.assertIn((("POST",), "/files/bulk-rename"), routes)
|
||||||
self.assertIn((("POST",), "/files/transfer"), routes)
|
self.assertIn((("POST",), "/files/transfer"), routes)
|
||||||
|
|
||||||
|
def test_share_lifecycle_routes_are_exposed(self) -> None:
|
||||||
|
routes = {
|
||||||
|
(tuple(sorted(route.methods or ())), route.path) for route in router.routes
|
||||||
|
}
|
||||||
|
|
||||||
|
self.assertIn((("GET",), "/files/{file_id}/shares"), routes)
|
||||||
|
self.assertIn((("POST",), "/files/{file_id}/shares"), routes)
|
||||||
|
self.assertIn((("DELETE",), "/files/{file_id}/shares/{share_id}"), routes)
|
||||||
|
self.assertIn((("GET",), "/files/{file_id}/share-target-options"), routes)
|
||||||
|
|
||||||
|
def test_exact_version_download_route_is_exposed(self) -> None:
|
||||||
|
routes = {
|
||||||
|
(tuple(sorted(route.methods or ())), route.path) for route in router.routes
|
||||||
|
}
|
||||||
|
|
||||||
|
self.assertIn(
|
||||||
|
(("GET",), "/files/{file_id}/versions/{version_id}/download"),
|
||||||
|
routes,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_governed_lifecycle_routes_are_exposed(self) -> None:
|
||||||
|
routes = {
|
||||||
|
(tuple(sorted(route.methods or ())), route.path) for route in router.routes
|
||||||
|
}
|
||||||
|
|
||||||
|
self.assertIn((("PATCH",), "/files/{file_id}/lifecycle"), routes)
|
||||||
|
self.assertIn((("POST",), "/files/assets/{file_id}/restore"), routes)
|
||||||
|
self.assertIn((("POST",), "/files/purge/preview"), routes)
|
||||||
|
self.assertIn((("POST",), "/files/purge/execute"), routes)
|
||||||
|
self.assertIn((("POST",), "/files/purge/blobs"), routes)
|
||||||
|
|
||||||
|
def test_file_listing_exposes_structured_property_filters(self) -> None:
|
||||||
|
route = next(
|
||||||
|
route
|
||||||
|
for route in listing_router.routes
|
||||||
|
if route.path == "/files" and "GET" in (route.methods or ())
|
||||||
|
)
|
||||||
|
parameters = signature(route.endpoint).parameters
|
||||||
|
|
||||||
|
self.assertIn("campaign_usage", parameters)
|
||||||
|
self.assertIn("audit_relevant", parameters)
|
||||||
|
self.assertIn("sort", parameters)
|
||||||
|
self.assertIn("total", route.response_model.model_fields)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
@@ -0,0 +1,294 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import socket
|
||||||
|
import sys
|
||||||
|
import threading
|
||||||
|
import types
|
||||||
|
import unittest
|
||||||
|
from importlib.util import find_spec
|
||||||
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from govoplan_core.security.outbound_http import (
|
||||||
|
OutboundHttpBlocked,
|
||||||
|
create_outbound_connection as core_create_outbound_connection,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.storage.sdk_peer_pinning import (
|
||||||
|
SdkPeerPinningError,
|
||||||
|
_botocore_transport_types,
|
||||||
|
create_pinned_s3_client,
|
||||||
|
install_pinned_smb_transport,
|
||||||
|
pinned_smb_connection_cache,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class _Socket:
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.connected_to: object | None = None
|
||||||
|
|
||||||
|
def settimeout(self, _value: object) -> None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def setsockopt(self, *_args: object) -> None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def connect(self, value: object) -> None:
|
||||||
|
self.connected_to = value
|
||||||
|
|
||||||
|
def close(self) -> None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
class _S3Handler(BaseHTTPRequestHandler):
|
||||||
|
def do_GET(self) -> None: # noqa: N802 - stdlib handler contract
|
||||||
|
body = (
|
||||||
|
b'<?xml version="1.0" encoding="UTF-8"?>'
|
||||||
|
b'<ListAllMyBucketsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">'
|
||||||
|
b'<Owner><ID>govoplan</ID><DisplayName>GovOPlaN</DisplayName></Owner>'
|
||||||
|
b'<Buckets><Bucket><Name>evidence</Name><CreationDate>2026-08-04T00:00:00Z</CreationDate>'
|
||||||
|
b'</Bucket></Buckets></ListAllMyBucketsResult>'
|
||||||
|
)
|
||||||
|
self.send_response(200)
|
||||||
|
self.send_header("Content-Type", "application/xml")
|
||||||
|
self.send_header("Content-Length", str(len(body)))
|
||||||
|
self.end_headers()
|
||||||
|
self.wfile.write(body)
|
||||||
|
|
||||||
|
def log_message(self, _format: str, *_args: object) -> None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
@unittest.skipUnless(find_spec("botocore") and find_spec("boto3"), "boto3 extra is not installed")
|
||||||
|
class S3PeerPinningTests(unittest.TestCase):
|
||||||
|
def test_client_is_born_with_the_pinned_http_session(self) -> None:
|
||||||
|
client = create_pinned_s3_client(
|
||||||
|
endpoint_url="http://127.0.0.1:9000",
|
||||||
|
region_name="eu-central-1",
|
||||||
|
aws_access_key_id="access",
|
||||||
|
aws_secret_access_key="secret",
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
session_type, _, _ = _botocore_transport_types()
|
||||||
|
self.assertIsInstance(client._endpoint.http_session, session_type)
|
||||||
|
self.assertEqual({}, client._endpoint.http_session._proxy_config._proxies)
|
||||||
|
finally:
|
||||||
|
client.close()
|
||||||
|
|
||||||
|
def test_each_connection_attempt_resolves_and_pins_again(self) -> None:
|
||||||
|
_, http_connection, _ = _botocore_transport_types()
|
||||||
|
connection = http_connection(host="objects.example.test", port=80)
|
||||||
|
sockets = [_Socket(), _Socket()]
|
||||||
|
with patch(
|
||||||
|
"govoplan_files.backend.storage.sdk_peer_pinning.create_outbound_connection",
|
||||||
|
side_effect=sockets,
|
||||||
|
) as connector:
|
||||||
|
self.assertIs(sockets[0], connection._new_conn())
|
||||||
|
self.assertIs(sockets[1], connection._new_conn())
|
||||||
|
|
||||||
|
self.assertEqual(2, connector.call_count)
|
||||||
|
self.assertEqual("objects.example.test", connector.call_args.args[0])
|
||||||
|
|
||||||
|
def test_real_sdk_request_uses_the_pinned_socket_path(self) -> None:
|
||||||
|
server = ThreadingHTTPServer(("127.0.0.1", 0), _S3Handler)
|
||||||
|
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||||
|
thread.start()
|
||||||
|
endpoint = f"http://127.0.0.1:{server.server_port}"
|
||||||
|
try:
|
||||||
|
with patch.dict(
|
||||||
|
"os.environ",
|
||||||
|
{"APP_ENV": "production", "GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS": "true"},
|
||||||
|
), patch(
|
||||||
|
"govoplan_files.backend.storage.sdk_peer_pinning.create_outbound_connection",
|
||||||
|
wraps=core_create_outbound_connection,
|
||||||
|
) as connector:
|
||||||
|
client = create_pinned_s3_client(
|
||||||
|
endpoint_url=endpoint,
|
||||||
|
region_name="eu-central-1",
|
||||||
|
aws_access_key_id="access",
|
||||||
|
aws_secret_access_key="secret",
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
response = client.list_buckets()
|
||||||
|
finally:
|
||||||
|
client.close()
|
||||||
|
finally:
|
||||||
|
server.shutdown()
|
||||||
|
server.server_close()
|
||||||
|
thread.join(timeout=2)
|
||||||
|
|
||||||
|
self.assertEqual("evidence", response["Buckets"][0]["Name"])
|
||||||
|
self.assertEqual("127.0.0.1", connector.call_args.args[0])
|
||||||
|
|
||||||
|
def test_every_sdk_selected_origin_uses_a_pinned_pool(self) -> None:
|
||||||
|
session_type, http_connection, https_connection = _botocore_transport_types()
|
||||||
|
session = session_type(proxies={})
|
||||||
|
try:
|
||||||
|
first = session._manager.connection_from_url("http://one.example.test/root")
|
||||||
|
redirected = session._manager.connection_from_url("https://two.example.test/root")
|
||||||
|
self.assertIs(first.ConnectionCls, http_connection)
|
||||||
|
self.assertIs(redirected.ConnectionCls, https_connection)
|
||||||
|
finally:
|
||||||
|
session.close()
|
||||||
|
|
||||||
|
def test_tls_connection_retains_the_configured_authority_for_sni(self) -> None:
|
||||||
|
_, _, https_connection = _botocore_transport_types()
|
||||||
|
connection = https_connection(host="objects.example.test", port=443)
|
||||||
|
with patch(
|
||||||
|
"govoplan_files.backend.storage.sdk_peer_pinning.create_outbound_connection",
|
||||||
|
return_value=_Socket(),
|
||||||
|
) as connector:
|
||||||
|
connection._new_conn()
|
||||||
|
|
||||||
|
self.assertEqual("objects.example.test", connection.host)
|
||||||
|
self.assertEqual("objects.example.test", connection._dns_host)
|
||||||
|
self.assertEqual("objects.example.test", connector.call_args.args[0])
|
||||||
|
|
||||||
|
def test_mixed_answer_and_peer_change_fail_closed(self) -> None:
|
||||||
|
_, http_connection, _ = _botocore_transport_types()
|
||||||
|
connection = http_connection(host="objects.example.test", port=80)
|
||||||
|
records = [
|
||||||
|
[(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("93.184.216.34", 80))],
|
||||||
|
[
|
||||||
|
(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("93.184.216.34", 80)),
|
||||||
|
(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("10.0.0.5", 80)),
|
||||||
|
],
|
||||||
|
]
|
||||||
|
with patch.dict(
|
||||||
|
"os.environ",
|
||||||
|
{"APP_ENV": "production", "GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS": "false"},
|
||||||
|
), patch(
|
||||||
|
"govoplan_core.security.outbound_http.socket.getaddrinfo",
|
||||||
|
side_effect=records,
|
||||||
|
), patch(
|
||||||
|
"govoplan_core.security.outbound_http.socket.socket",
|
||||||
|
return_value=_Socket(),
|
||||||
|
):
|
||||||
|
connection._new_conn()
|
||||||
|
with self.assertRaisesRegex(Exception, "non-public network"):
|
||||||
|
connection._new_conn()
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeTcp:
|
||||||
|
def __init__(self, server: str, port: int, timeout: float | None = None) -> None:
|
||||||
|
self.server = server
|
||||||
|
self.port = port
|
||||||
|
self.timeout = timeout
|
||||||
|
self.connected = False
|
||||||
|
self._sock = None
|
||||||
|
self._sock_lock = threading.Lock()
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeConnection:
|
||||||
|
def __init__(self, _guid: object, server: str, port: int, *, require_signing: bool) -> None:
|
||||||
|
self.server = server
|
||||||
|
self.port = port
|
||||||
|
self.require_signing = require_signing
|
||||||
|
self.transport = _FakeTcp(server, port)
|
||||||
|
self.session_table: dict[str, object] = {}
|
||||||
|
|
||||||
|
def connect(self, *, timeout: float) -> None:
|
||||||
|
self.transport.timeout = timeout
|
||||||
|
self.transport.connect()
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeSession:
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
connection: _FakeConnection,
|
||||||
|
*,
|
||||||
|
username: str | None,
|
||||||
|
password: str | None,
|
||||||
|
require_encryption: bool,
|
||||||
|
auth_protocol: str,
|
||||||
|
) -> None:
|
||||||
|
self.connection = connection
|
||||||
|
self.username = username
|
||||||
|
self.password = password
|
||||||
|
self.encrypt_data = require_encryption
|
||||||
|
self.auth_protocol = auth_protocol
|
||||||
|
self.encrypt = require_encryption
|
||||||
|
|
||||||
|
def connect(self) -> None:
|
||||||
|
self.connection.session_table[self.username or "anonymous"] = self
|
||||||
|
|
||||||
|
|
||||||
|
class SmbPeerPinningTests(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
pinned_smb_connection_cache.cache_clear()
|
||||||
|
|
||||||
|
def _modules(self) -> tuple[types.ModuleType, dict[str, types.ModuleType]]:
|
||||||
|
smbclient = types.ModuleType("smbclient")
|
||||||
|
pool = types.ModuleType("smbclient._pool")
|
||||||
|
|
||||||
|
def original_register_session(
|
||||||
|
server: str,
|
||||||
|
username: str | None = None,
|
||||||
|
password: str | None = None,
|
||||||
|
port: int = 445,
|
||||||
|
encrypt: bool | None = None,
|
||||||
|
connection_timeout: float = 60,
|
||||||
|
connection_cache: dict[str, object] | None = None,
|
||||||
|
auth_protocol: str = "negotiate",
|
||||||
|
require_signing: bool = True,
|
||||||
|
) -> None:
|
||||||
|
del server, username, password, port, encrypt, connection_timeout
|
||||||
|
del connection_cache, auth_protocol, require_signing
|
||||||
|
|
||||||
|
pool.register_session = original_register_session
|
||||||
|
pool.ClientConfig = lambda: types.SimpleNamespace(client_guid="guid")
|
||||||
|
connection = types.ModuleType("smbprotocol.connection")
|
||||||
|
connection.Connection = _FakeConnection
|
||||||
|
connection.Tcp = _FakeTcp
|
||||||
|
session = types.ModuleType("smbprotocol.session")
|
||||||
|
session.Session = _FakeSession
|
||||||
|
transport = types.ModuleType("smbprotocol.transport")
|
||||||
|
transport.Tcp = _FakeTcp
|
||||||
|
return smbclient, {
|
||||||
|
"smbclient._pool": pool,
|
||||||
|
"smbprotocol.connection": connection,
|
||||||
|
"smbprotocol.session": session,
|
||||||
|
"smbprotocol.transport": transport,
|
||||||
|
}
|
||||||
|
|
||||||
|
def test_initial_reconnect_and_referral_hosts_are_each_pinned(self) -> None:
|
||||||
|
smbclient, modules = self._modules()
|
||||||
|
sockets = [_Socket(), _Socket(), _Socket()]
|
||||||
|
with patch.dict(sys.modules, modules), patch(
|
||||||
|
"govoplan_files.backend.storage.sdk_peer_pinning.create_outbound_connection",
|
||||||
|
side_effect=sockets,
|
||||||
|
) as connector:
|
||||||
|
install_pinned_smb_transport(smbclient)
|
||||||
|
register = modules["smbclient._pool"].register_session
|
||||||
|
cache: dict[str, object] = {}
|
||||||
|
first = register("files.example.test", connection_cache=cache)
|
||||||
|
first.connection.transport.connected = False
|
||||||
|
register("files.example.test", connection_cache=cache)
|
||||||
|
register("dfs-target.example.test", connection_cache=cache)
|
||||||
|
|
||||||
|
self.assertEqual(
|
||||||
|
["files.example.test", "files.example.test", "dfs-target.example.test"],
|
||||||
|
[call.args[0] for call in connector.call_args_list],
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_referral_to_disallowed_peer_fails_before_session_creation(self) -> None:
|
||||||
|
smbclient, modules = self._modules()
|
||||||
|
with patch.dict(sys.modules, modules), patch(
|
||||||
|
"govoplan_files.backend.storage.sdk_peer_pinning.create_outbound_connection",
|
||||||
|
side_effect=OutboundHttpBlocked("non-public network"),
|
||||||
|
):
|
||||||
|
install_pinned_smb_transport(smbclient)
|
||||||
|
register = modules["smbclient._pool"].register_session
|
||||||
|
with self.assertRaisesRegex(ValueError, "non-public network"):
|
||||||
|
register("private-referral.example.test", connection_cache={})
|
||||||
|
|
||||||
|
def test_transport_tampering_after_installation_fails_closed(self) -> None:
|
||||||
|
smbclient, modules = self._modules()
|
||||||
|
with patch.dict(sys.modules, modules):
|
||||||
|
install_pinned_smb_transport(smbclient)
|
||||||
|
modules["smbprotocol.connection"].Tcp = _FakeTcp
|
||||||
|
with self.assertRaisesRegex(SdkPeerPinningError, "changed after peer pinning"):
|
||||||
|
install_pinned_smb_transport(smbclient)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,145 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from types import SimpleNamespace
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from sqlalchemy import create_engine
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from govoplan_access.backend.db.models import Account, Group, User
|
||||||
|
from govoplan_core.auth import ApiPrincipal
|
||||||
|
from govoplan_core.core.access import PrincipalRef
|
||||||
|
from govoplan_core.core.change_sequence import ChangeSequenceEntry
|
||||||
|
from govoplan_core.core.events import EventObjectRef, EventTenantRef, PlatformEvent
|
||||||
|
from govoplan_core.core.search import (
|
||||||
|
SearchAuthorizationRequest,
|
||||||
|
SearchBackfillRequest,
|
||||||
|
SearchResourceReference,
|
||||||
|
)
|
||||||
|
from govoplan_core.db.base import Base
|
||||||
|
from govoplan_files.backend.db.models import FileAsset, FileFolder, FileShare
|
||||||
|
from govoplan_files.backend.search_source import (
|
||||||
|
FilesSearchSource,
|
||||||
|
PROVIDER_ID,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class FilesSearchSourceTests(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
self.engine = create_engine("sqlite://")
|
||||||
|
Base.metadata.create_all(
|
||||||
|
self.engine,
|
||||||
|
tables=(
|
||||||
|
Account.__table__,
|
||||||
|
User.__table__,
|
||||||
|
Group.__table__,
|
||||||
|
FileAsset.__table__,
|
||||||
|
FileFolder.__table__,
|
||||||
|
FileShare.__table__,
|
||||||
|
ChangeSequenceEntry.__table__,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
self.session = Session(self.engine)
|
||||||
|
self.session.add_all(
|
||||||
|
(
|
||||||
|
Account(
|
||||||
|
id="account-1",
|
||||||
|
email="one@example.test",
|
||||||
|
normalized_email="one@example.test",
|
||||||
|
),
|
||||||
|
User(
|
||||||
|
id="user-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
account_id="account-1",
|
||||||
|
email="one@example.test",
|
||||||
|
),
|
||||||
|
FileAsset(
|
||||||
|
id="file-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
owner_type="user",
|
||||||
|
owner_user_id="user-1",
|
||||||
|
display_path="records/permit.pdf",
|
||||||
|
filename="permit.pdf",
|
||||||
|
description="Monthly permit evidence",
|
||||||
|
),
|
||||||
|
FileAsset(
|
||||||
|
id="file-other",
|
||||||
|
tenant_id="tenant-2",
|
||||||
|
owner_type="user",
|
||||||
|
display_path="other.pdf",
|
||||||
|
filename="other.pdf",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
self.session.commit()
|
||||||
|
self.source = FilesSearchSource()
|
||||||
|
|
||||||
|
def tearDown(self) -> None:
|
||||||
|
self.session.close()
|
||||||
|
self.engine.dispose()
|
||||||
|
|
||||||
|
def test_backfill_and_authorization_are_tenant_bounded(self) -> None:
|
||||||
|
page = self.source.backfill(
|
||||||
|
self.session,
|
||||||
|
request=SearchBackfillRequest(
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
provider_id=PROVIDER_ID,
|
||||||
|
resource_type="file",
|
||||||
|
rebuild_id="rebuild-1",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
self.assertEqual(("file-1",), tuple(doc.resource_id for doc in page.documents))
|
||||||
|
reference = SearchResourceReference(
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
module_id="files",
|
||||||
|
resource_type="file",
|
||||||
|
resource_id="file-1",
|
||||||
|
)
|
||||||
|
request = SearchAuthorizationRequest(reference=reference, source_revision="1")
|
||||||
|
self.assertTrue(
|
||||||
|
self.source.authorize(
|
||||||
|
self.session,
|
||||||
|
_principal({"files:file:read"}),
|
||||||
|
requests=(request,),
|
||||||
|
)[reference.key]
|
||||||
|
)
|
||||||
|
self.assertFalse(
|
||||||
|
self.source.authorize(
|
||||||
|
self.session,
|
||||||
|
_principal(set()),
|
||||||
|
requests=(request,),
|
||||||
|
)[reference.key]
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_committed_file_event_produces_authoritative_upsert(self) -> None:
|
||||||
|
event = PlatformEvent(
|
||||||
|
type="files.file.updated",
|
||||||
|
module_id="files",
|
||||||
|
tenant=EventTenantRef(id="tenant-1"),
|
||||||
|
resource=EventObjectRef(type="file", id="file-1"),
|
||||||
|
)
|
||||||
|
changes = self.source.index_changes_for_event(
|
||||||
|
self.session,
|
||||||
|
event=event,
|
||||||
|
delivery_key="delivery-1",
|
||||||
|
)
|
||||||
|
self.assertEqual(1, len(changes))
|
||||||
|
self.assertEqual("upsert", changes[0].kind)
|
||||||
|
self.assertEqual(event.event_id, changes[0].cursor)
|
||||||
|
|
||||||
|
|
||||||
|
def _principal(scopes: set[str]) -> ApiPrincipal:
|
||||||
|
return ApiPrincipal(
|
||||||
|
principal=PrincipalRef(
|
||||||
|
account_id="account-1",
|
||||||
|
membership_id="user-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
scopes=frozenset(scopes),
|
||||||
|
),
|
||||||
|
account=SimpleNamespace(id="account-1"),
|
||||||
|
user=SimpleNamespace(id="user-1"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,237 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import unittest
|
||||||
|
from datetime import timedelta
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from sqlalchemy import create_engine
|
||||||
|
from sqlalchemy.dialects import postgresql
|
||||||
|
from sqlalchemy.orm import sessionmaker
|
||||||
|
|
||||||
|
from govoplan_access.backend.db.models import Account, Group, User
|
||||||
|
from govoplan_core.core.change_sequence import ChangeSequenceEntry
|
||||||
|
from govoplan_core.db.base import Base
|
||||||
|
from govoplan_files.backend.db.models import FileAsset, FileShare
|
||||||
|
from govoplan_files.backend.storage.common import FileStorageError, utcnow
|
||||||
|
from govoplan_files.backend.storage.files import (
|
||||||
|
_asset_visibility_query_for_user,
|
||||||
|
get_asset_for_user,
|
||||||
|
list_file_shares,
|
||||||
|
revoke_file_share,
|
||||||
|
share_file,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
TENANT_ID = "tenant-1"
|
||||||
|
OWNER_ID = "owner-1"
|
||||||
|
RECIPIENT_ID = "recipient-1"
|
||||||
|
GROUP_ID = "group-1"
|
||||||
|
|
||||||
|
|
||||||
|
class FileShareLifecycleTests(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
self.engine = create_engine("sqlite:///:memory:", future=True)
|
||||||
|
Base.metadata.create_all(
|
||||||
|
bind=self.engine,
|
||||||
|
tables=[
|
||||||
|
Account.__table__,
|
||||||
|
User.__table__,
|
||||||
|
Group.__table__,
|
||||||
|
ChangeSequenceEntry.__table__,
|
||||||
|
FileAsset.__table__,
|
||||||
|
FileShare.__table__,
|
||||||
|
],
|
||||||
|
)
|
||||||
|
self.session = sessionmaker(bind=self.engine, future=True)()
|
||||||
|
self.asset = FileAsset(
|
||||||
|
id="file-1",
|
||||||
|
tenant_id=TENANT_ID,
|
||||||
|
owner_type="user",
|
||||||
|
owner_user_id=OWNER_ID,
|
||||||
|
display_path="shared.pdf",
|
||||||
|
filename="shared.pdf",
|
||||||
|
)
|
||||||
|
self.session.add(self.asset)
|
||||||
|
self.session.commit()
|
||||||
|
|
||||||
|
def tearDown(self) -> None:
|
||||||
|
self.session.close()
|
||||||
|
self.engine.dispose()
|
||||||
|
|
||||||
|
@patch(
|
||||||
|
"govoplan_files.backend.storage.files.user_group_ids",
|
||||||
|
return_value=[],
|
||||||
|
)
|
||||||
|
def test_expired_share_stops_access_immediately(self, _groups) -> None:
|
||||||
|
self.session.add(
|
||||||
|
FileShare(
|
||||||
|
id="expired-share",
|
||||||
|
tenant_id=TENANT_ID,
|
||||||
|
file_asset_id=self.asset.id,
|
||||||
|
target_type="user",
|
||||||
|
target_id=RECIPIENT_ID,
|
||||||
|
permission="read",
|
||||||
|
expires_at=utcnow() - timedelta(seconds=1),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
self.session.commit()
|
||||||
|
|
||||||
|
with self.assertRaisesRegex(FileStorageError, "No access"):
|
||||||
|
get_asset_for_user(
|
||||||
|
self.session,
|
||||||
|
tenant_id=TENANT_ID,
|
||||||
|
user_id=RECIPIENT_ID,
|
||||||
|
asset_id=self.asset.id,
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
[],
|
||||||
|
list_file_shares(
|
||||||
|
self.session,
|
||||||
|
tenant_id=TENANT_ID,
|
||||||
|
asset_id=self.asset.id,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
@patch(
|
||||||
|
"govoplan_files.backend.storage.files.user_group_ids",
|
||||||
|
return_value=[GROUP_ID],
|
||||||
|
)
|
||||||
|
def test_independent_active_grant_survives_other_expiry(self, _groups) -> None:
|
||||||
|
self.session.add_all(
|
||||||
|
[
|
||||||
|
FileShare(
|
||||||
|
id="expired-user-share",
|
||||||
|
tenant_id=TENANT_ID,
|
||||||
|
file_asset_id=self.asset.id,
|
||||||
|
target_type="user",
|
||||||
|
target_id=RECIPIENT_ID,
|
||||||
|
permission="read",
|
||||||
|
expires_at=utcnow() - timedelta(seconds=1),
|
||||||
|
),
|
||||||
|
FileShare(
|
||||||
|
id="active-group-share",
|
||||||
|
tenant_id=TENANT_ID,
|
||||||
|
file_asset_id=self.asset.id,
|
||||||
|
target_type="group",
|
||||||
|
target_id=GROUP_ID,
|
||||||
|
permission="read",
|
||||||
|
expires_at=utcnow() + timedelta(hours=1),
|
||||||
|
),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
self.session.commit()
|
||||||
|
|
||||||
|
result = get_asset_for_user(
|
||||||
|
self.session,
|
||||||
|
tenant_id=TENANT_ID,
|
||||||
|
user_id=RECIPIENT_ID,
|
||||||
|
asset_id=self.asset.id,
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(self.asset.id, result.id)
|
||||||
|
self.assertEqual(
|
||||||
|
["active-group-share"],
|
||||||
|
[
|
||||||
|
share.id
|
||||||
|
for share in list_file_shares(
|
||||||
|
self.session,
|
||||||
|
tenant_id=TENANT_ID,
|
||||||
|
asset_id=self.asset.id,
|
||||||
|
)
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
@patch(
|
||||||
|
"govoplan_files.backend.storage.files.ensure_share_target_exists",
|
||||||
|
return_value=None,
|
||||||
|
)
|
||||||
|
def test_revoke_is_idempotent_and_future_expiry_is_persisted(
|
||||||
|
self, _target_exists
|
||||||
|
) -> None:
|
||||||
|
expiry = utcnow() + timedelta(days=1)
|
||||||
|
share = share_file(
|
||||||
|
self.session,
|
||||||
|
tenant_id=TENANT_ID,
|
||||||
|
asset=self.asset,
|
||||||
|
target_type="user",
|
||||||
|
target_id=RECIPIENT_ID,
|
||||||
|
permission="read",
|
||||||
|
user_id=OWNER_ID,
|
||||||
|
expires_at=expiry,
|
||||||
|
)
|
||||||
|
self.session.commit()
|
||||||
|
|
||||||
|
revoked, first_changed = revoke_file_share(
|
||||||
|
self.session,
|
||||||
|
tenant_id=TENANT_ID,
|
||||||
|
asset_id=self.asset.id,
|
||||||
|
share_id=share.id,
|
||||||
|
user_id=OWNER_ID,
|
||||||
|
)
|
||||||
|
self.session.commit()
|
||||||
|
repeated, second_changed = revoke_file_share(
|
||||||
|
self.session,
|
||||||
|
tenant_id=TENANT_ID,
|
||||||
|
asset_id=self.asset.id,
|
||||||
|
share_id=share.id,
|
||||||
|
user_id=OWNER_ID,
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertTrue(first_changed)
|
||||||
|
self.assertFalse(second_changed)
|
||||||
|
self.assertEqual(OWNER_ID, revoked.revoked_by_user_id)
|
||||||
|
self.assertEqual(revoked.revoked_at, repeated.revoked_at)
|
||||||
|
self.assertEqual([], list_file_shares(
|
||||||
|
self.session,
|
||||||
|
tenant_id=TENANT_ID,
|
||||||
|
asset_id=self.asset.id,
|
||||||
|
))
|
||||||
|
self.assertEqual(
|
||||||
|
[share.id],
|
||||||
|
[
|
||||||
|
item.id
|
||||||
|
for item in list_file_shares(
|
||||||
|
self.session,
|
||||||
|
tenant_id=TENANT_ID,
|
||||||
|
asset_id=self.asset.id,
|
||||||
|
include_inactive=True,
|
||||||
|
)
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
@patch(
|
||||||
|
"govoplan_files.backend.storage.files.ensure_share_target_exists",
|
||||||
|
return_value=None,
|
||||||
|
)
|
||||||
|
def test_past_expiry_is_rejected(self, _target_exists) -> None:
|
||||||
|
with self.assertRaisesRegex(FileStorageError, "future"):
|
||||||
|
share_file(
|
||||||
|
self.session,
|
||||||
|
tenant_id=TENANT_ID,
|
||||||
|
asset=self.asset,
|
||||||
|
target_type="user",
|
||||||
|
target_id=RECIPIENT_ID,
|
||||||
|
permission="read",
|
||||||
|
user_id=OWNER_ID,
|
||||||
|
expires_at=utcnow() - timedelta(seconds=1),
|
||||||
|
)
|
||||||
|
|
||||||
|
@patch(
|
||||||
|
"govoplan_files.backend.storage.files.user_group_ids",
|
||||||
|
return_value=[],
|
||||||
|
)
|
||||||
|
def test_visibility_query_is_postgresql_json_safe(self, _groups) -> None:
|
||||||
|
query = _asset_visibility_query_for_user(
|
||||||
|
self.session,
|
||||||
|
tenant_id=TENANT_ID,
|
||||||
|
user_id=RECIPIENT_ID,
|
||||||
|
)
|
||||||
|
|
||||||
|
compiled = str(query.statement.compile(dialect=postgresql.dialect()))
|
||||||
|
|
||||||
|
self.assertNotIn("SELECT DISTINCT", compiled.upper())
|
||||||
|
self.assertIn("EXISTS", compiled.upper())
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -1,19 +1,26 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import io
|
import io
|
||||||
|
from types import ModuleType
|
||||||
|
import sys
|
||||||
import unittest
|
import unittest
|
||||||
from unittest.mock import MagicMock, PropertyMock, patch
|
from unittest.mock import MagicMock, PropertyMock, patch
|
||||||
|
|
||||||
from govoplan_files.backend.storage.backends import S3StorageBackend, StorageBackendError
|
from govoplan_files.backend.storage.backends import S3StorageBackend, StorageBackendError
|
||||||
|
|
||||||
|
|
||||||
def _backend() -> S3StorageBackend:
|
def _backend(
|
||||||
|
*,
|
||||||
|
endpoint_url: str = "https://objects.example.test",
|
||||||
|
deployment_managed: bool = False,
|
||||||
|
) -> S3StorageBackend:
|
||||||
return S3StorageBackend(
|
return S3StorageBackend(
|
||||||
bucket="files",
|
bucket="files",
|
||||||
endpoint_url="https://objects.example.test",
|
endpoint_url=endpoint_url,
|
||||||
region_name="test",
|
region_name="test",
|
||||||
access_key_id="access",
|
access_key_id="access",
|
||||||
secret_access_key="secret",
|
secret_access_key="secret",
|
||||||
|
deployment_managed=deployment_managed,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -32,6 +39,48 @@ class S3StorageBackendTests(unittest.TestCase):
|
|||||||
), self.assertRaisesRegex(StorageBackendError, "until that transport supports.*DNS/IP pinning"):
|
), self.assertRaisesRegex(StorageBackendError, "until that transport supports.*DNS/IP pinning"):
|
||||||
_backend().client
|
_backend().client
|
||||||
|
|
||||||
|
def test_installer_managed_garage_uses_exact_endpoint_and_path_style(self) -> None:
|
||||||
|
boto3 = ModuleType("boto3")
|
||||||
|
boto3.client = MagicMock(return_value=object())
|
||||||
|
botocore = ModuleType("botocore")
|
||||||
|
botocore.__path__ = []
|
||||||
|
botocore_config = ModuleType("botocore.config")
|
||||||
|
|
||||||
|
class Config:
|
||||||
|
def __init__(self, **values):
|
||||||
|
self.values = values
|
||||||
|
|
||||||
|
botocore_config.Config = Config
|
||||||
|
with patch.dict(
|
||||||
|
sys.modules,
|
||||||
|
{
|
||||||
|
"boto3": boto3,
|
||||||
|
"botocore": botocore,
|
||||||
|
"botocore.config": botocore_config,
|
||||||
|
},
|
||||||
|
):
|
||||||
|
_backend(
|
||||||
|
endpoint_url="http://garage:3900",
|
||||||
|
deployment_managed=True,
|
||||||
|
).client
|
||||||
|
|
||||||
|
_args, kwargs = boto3.client.call_args
|
||||||
|
self.assertEqual("http://garage:3900", kwargs["endpoint_url"])
|
||||||
|
self.assertEqual(
|
||||||
|
{"s3": {"addressing_style": "path"}},
|
||||||
|
kwargs["config"].values,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_installer_managed_garage_rejects_any_other_endpoint(self) -> None:
|
||||||
|
with self.assertRaisesRegex(
|
||||||
|
StorageBackendError,
|
||||||
|
"restricted to http://garage:3900",
|
||||||
|
):
|
||||||
|
_backend(
|
||||||
|
endpoint_url="http://other-s3:3900",
|
||||||
|
deployment_managed=True,
|
||||||
|
).client
|
||||||
|
|
||||||
def test_get_bytes_rejects_declared_oversize_object_without_reading(self) -> None:
|
def test_get_bytes_rejects_declared_oversize_object_without_reading(self) -> None:
|
||||||
body = MagicMock()
|
body = MagicMock()
|
||||||
client = MagicMock()
|
client = MagicMock()
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user