Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a8c3192c3a | ||
|
|
ce4eaefbfd | ||
|
|
6c3cf1c55e | ||
|
|
95aef18955 | ||
|
|
a9c7a7a40c | ||
|
|
e1361427b8 | ||
|
|
ad55d47645 | ||
|
|
21122e058e | ||
|
|
25140fb0b6 | ||
|
|
e9561a87e1 | ||
|
|
9d7d3f96ec | ||
|
|
44ba0ec9c1 | ||
|
|
1e511e4c15 | ||
|
|
cb37079465 | ||
|
|
04e2bae190 | ||
|
|
aa9f90c71e | ||
|
|
60c3d14f08 | ||
|
|
3a3360650f | ||
|
|
c9f8e9262e | ||
|
|
edee29af51 | ||
|
|
0293e49def | ||
|
|
aaa364b8d4 | ||
|
|
4bb17c0f4c | ||
|
|
92e649477f | ||
|
|
0c68e904cf | ||
|
|
8ec31b16ee | ||
|
|
7e6be4b017 | ||
|
|
04882f1628 | ||
|
|
d8ae506ff8 | ||
|
|
6baf2a421b | ||
|
|
b993d8e31a | ||
|
|
359c4e9570 | ||
|
|
233ce40983 | ||
|
|
f084a0f3a9 | ||
|
|
b752dea610 | ||
|
|
159a012833 | ||
|
|
835eacfc5d | ||
|
|
85606d5580 | ||
|
|
632cc6cf7d | ||
|
|
86a905a3a7 | ||
|
|
5b868272b9 | ||
|
|
93eea839c8 | ||
|
|
b667e7ff0e | ||
|
|
0fd8f6e02a |
@@ -0,0 +1,270 @@
|
||||
name: Module Package Release
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- "v*"
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
release_tag:
|
||||
description: Existing protected version tag to publish
|
||||
required: true
|
||||
type: string
|
||||
|
||||
jobs:
|
||||
publish-packages:
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
GITEA_REPOSITORY: ${{ gitea.repository }}
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065
|
||||
with:
|
||||
python-version: "3.12"
|
||||
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020
|
||||
with:
|
||||
node-version: "22"
|
||||
- name: Select and validate protected release tag
|
||||
shell: bash
|
||||
env:
|
||||
REQUESTED_TAG: ${{ inputs.release_tag }}
|
||||
TRIGGER_TAG: ${{ gitea.ref_name }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
tag="${REQUESTED_TAG:-$TRIGGER_TAG}"
|
||||
case "$tag" in
|
||||
v[0-9]*.[0-9]*.[0-9]*) ;;
|
||||
*) echo "Release tag must start with a SemVer-shaped vX.Y.Z value" >&2; exit 1 ;;
|
||||
esac
|
||||
git fetch --force origin "refs/tags/$tag:refs/tags/$tag" refs/heads/main:refs/remotes/origin/main
|
||||
tag_commit="$(git rev-list -n 1 "$tag")"
|
||||
git merge-base --is-ancestor "$tag_commit" refs/remotes/origin/main || {
|
||||
echo "Release tag is not contained in main" >&2
|
||||
exit 1
|
||||
}
|
||||
git checkout --detach "$tag"
|
||||
printf 'RELEASE_TAG=%s\n' "$tag" >> "$GITEA_ENV"
|
||||
printf 'SOURCE_DATE_EPOCH=%s\n' "$(git show -s --format=%ct HEAD)" >> "$GITEA_ENV"
|
||||
- name: Validate package versions
|
||||
run: |
|
||||
python - <<'PY'
|
||||
import json
|
||||
from pathlib import Path
|
||||
import os
|
||||
import re
|
||||
import tomllib
|
||||
|
||||
tag = os.environ["RELEASE_TAG"]
|
||||
expected = tag.removeprefix("v")
|
||||
project = tomllib.loads(Path("pyproject.toml").read_text(encoding="utf-8"))["project"]
|
||||
if project.get("version") != expected:
|
||||
raise SystemExit(f"pyproject version {project.get('version')!r} does not match {tag}")
|
||||
if re.fullmatch(r"govoplan-[a-z0-9-]+", str(project.get("name", ""))) is None:
|
||||
raise SystemExit("Python distribution name must use the govoplan-* namespace")
|
||||
webui = Path("webui/package.json")
|
||||
if webui.is_file():
|
||||
package = json.loads(webui.read_text(encoding="utf-8"))
|
||||
if package.get("version") != expected:
|
||||
raise SystemExit(f"WebUI version {package.get('version')!r} does not match {tag}")
|
||||
if re.fullmatch(r"@govoplan/[a-z0-9-]+-webui", str(package.get("name", ""))) is None:
|
||||
raise SystemExit("WebUI package name must use the @govoplan/*-webui namespace")
|
||||
release = Path("webui/package.release.json")
|
||||
if release.is_file():
|
||||
release_package = json.loads(release.read_text(encoding="utf-8"))
|
||||
if (
|
||||
release_package.get("name") != package.get("name")
|
||||
or release_package.get("version") != expected
|
||||
):
|
||||
raise SystemExit("WebUI release package identity does not match package.json and the release tag")
|
||||
PY
|
||||
- name: Build immutable package artifacts
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python -m pip install --disable-pip-version-check build==1.5.0 twine==7.0.0
|
||||
rm -rf dist .package-webui
|
||||
python -m build --wheel --outdir dist
|
||||
python -m twine check dist/*.whl
|
||||
if [[ -f webui/package.json ]]; then
|
||||
mkdir .package-webui
|
||||
cp -a webui/. .package-webui/
|
||||
rm -rf .package-webui/node_modules .package-webui/dist
|
||||
if [[ -f .package-webui/package.release.json ]]; then
|
||||
cp .package-webui/package.release.json .package-webui/package.json
|
||||
fi
|
||||
node <<'NODE'
|
||||
const fs = require("node:fs");
|
||||
const path = ".package-webui/package.json";
|
||||
const packageJson = JSON.parse(fs.readFileSync(path, "utf8"));
|
||||
const groups = ["dependencies", "optionalDependencies", "peerDependencies"];
|
||||
for (const group of groups) {
|
||||
for (const [name, specifier] of Object.entries(packageJson[group] || {})) {
|
||||
if (!name.startsWith("@govoplan/")) continue;
|
||||
if (typeof specifier !== "string") {
|
||||
throw new Error(`${group}.${name} must use a string version`);
|
||||
}
|
||||
const packageSlug = name.slice("@govoplan/".length);
|
||||
if (!packageSlug.endsWith("-webui")) {
|
||||
throw new Error(`${group}.${name} is outside the WebUI package namespace`);
|
||||
}
|
||||
const repository = `govoplan-${packageSlug.slice(0, -"-webui".length)}`;
|
||||
const escapedRepository = repository.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
const gitTag = specifier.match(
|
||||
new RegExp(
|
||||
`^git\\+(?:ssh://git@|https://)git\\.add-ideas\\.de/(?:GovOPlaN|add-ideas)/${escapedRepository}\\.git#v([0-9]+\\.[0-9]+\\.[0-9]+)$`,
|
||||
),
|
||||
);
|
||||
if (gitTag) {
|
||||
packageJson[group][name] = gitTag[1];
|
||||
continue;
|
||||
}
|
||||
if (specifier.startsWith("file:") || specifier.startsWith("git+")) {
|
||||
throw new Error(
|
||||
`${group}.${name} must resolve to an exact registry version for publication`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
delete packageJson.private;
|
||||
fs.writeFileSync(path, `${JSON.stringify(packageJson, null, 2)}\n`);
|
||||
NODE
|
||||
npm pkg delete private --prefix .package-webui
|
||||
(cd .package-webui && npm pack --ignore-scripts --pack-destination ../dist)
|
||||
fi
|
||||
python - <<'PY'
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import os
|
||||
import subprocess
|
||||
|
||||
artifacts = []
|
||||
for path in sorted(Path("dist").iterdir()):
|
||||
if path.suffix not in {".whl", ".tgz"}:
|
||||
continue
|
||||
digest = hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
artifacts.append({"filename": path.name, "sha256": digest, "size": path.stat().st_size})
|
||||
payload = {
|
||||
"schema_version": "1",
|
||||
"repository": os.environ["GITEA_REPOSITORY"],
|
||||
"tag": os.environ["RELEASE_TAG"],
|
||||
"commit": subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip(),
|
||||
"artifacts": artifacts,
|
||||
}
|
||||
Path("dist/package-artifacts.json").write_text(
|
||||
json.dumps(payload, indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
PY
|
||||
- name: Retain package hash evidence
|
||||
uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32
|
||||
with:
|
||||
name: module-packages-${{ gitea.ref_name }}
|
||||
path: dist/package-artifacts.json
|
||||
- name: Check immutable registry state
|
||||
shell: bash
|
||||
env:
|
||||
PACKAGE_TOKEN: ${{ secrets.GOVOPLAN_PACKAGE_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
test -n "$PACKAGE_TOKEN"
|
||||
python - <<'PY'
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import tomllib
|
||||
from urllib.error import HTTPError
|
||||
from urllib.parse import quote
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
api_root = "https://git.add-ideas.de/api/v1/packages/GovOPlaN"
|
||||
token = os.environ["PACKAGE_TOKEN"]
|
||||
|
||||
def should_publish(kind, name, version, path):
|
||||
package_url = "/".join(
|
||||
(api_root, kind, quote(name, safe=""), quote(version, safe=""), "files")
|
||||
)
|
||||
request = Request(
|
||||
package_url,
|
||||
headers={"Accept": "application/json", "Authorization": f"token {token}"},
|
||||
)
|
||||
try:
|
||||
with urlopen(request, timeout=30) as response:
|
||||
files = json.load(response)
|
||||
except HTTPError as exc:
|
||||
if exc.code == 404:
|
||||
print(f"{kind} package {name}=={version} is not published yet")
|
||||
return True
|
||||
raise
|
||||
if not isinstance(files, list) or len(files) != 1:
|
||||
raise SystemExit(
|
||||
f"immutable {kind} package {name}=={version} has an unexpected file set"
|
||||
)
|
||||
expected_sha256 = hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
if files[0].get("sha256") != expected_sha256:
|
||||
raise SystemExit(
|
||||
f"immutable {kind} package {name}=={version} already exists with a different SHA-256"
|
||||
)
|
||||
print(f"verified existing {kind} package {name}=={version} ({expected_sha256})")
|
||||
return False
|
||||
|
||||
project = tomllib.loads(Path("pyproject.toml").read_text(encoding="utf-8"))["project"]
|
||||
wheels = tuple(Path("dist").glob("*.whl"))
|
||||
if len(wheels) != 1:
|
||||
raise SystemExit("release build must contain exactly one wheel")
|
||||
publish_pypi = should_publish(
|
||||
"pypi", str(project["name"]), str(project["version"]), wheels[0]
|
||||
)
|
||||
|
||||
tarballs = tuple(Path("dist").glob("*.tgz"))
|
||||
if len(tarballs) > 1:
|
||||
raise SystemExit("release build must contain at most one npm package")
|
||||
publish_npm = False
|
||||
if tarballs:
|
||||
webui = json.loads(
|
||||
Path(".package-webui/package.json").read_text(encoding="utf-8")
|
||||
)
|
||||
publish_npm = should_publish(
|
||||
"npm", str(webui["name"]), str(webui["version"]), tarballs[0]
|
||||
)
|
||||
|
||||
with Path(os.environ["GITEA_ENV"]).open("a", encoding="utf-8") as env_file:
|
||||
env_file.write(f"PUBLISH_PYPI={int(publish_pypi)}\n")
|
||||
env_file.write(f"PUBLISH_NPM={int(publish_npm)}\n")
|
||||
PY
|
||||
- name: Publish wheel and WebUI package
|
||||
shell: bash
|
||||
env:
|
||||
PACKAGE_USERNAME: ${{ secrets.GOVOPLAN_PACKAGE_USERNAME }}
|
||||
PACKAGE_TOKEN: ${{ secrets.GOVOPLAN_PACKAGE_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
test -n "$PACKAGE_USERNAME"
|
||||
test -n "$PACKAGE_TOKEN"
|
||||
if [[ "$PUBLISH_PYPI" == 1 ]]; then
|
||||
TWINE_USERNAME="$PACKAGE_USERNAME" TWINE_PASSWORD="$PACKAGE_TOKEN" \
|
||||
python -m twine upload --non-interactive \
|
||||
--repository-url https://git.add-ideas.de/api/packages/GovOPlaN/pypi \
|
||||
dist/*.whl
|
||||
else
|
||||
echo "Exact wheel is already present; skipping immutable retry."
|
||||
fi
|
||||
shopt -s nullglob
|
||||
webui_packages=(dist/*.tgz)
|
||||
if (( ${#webui_packages[@]} )) && [[ "$PUBLISH_NPM" == 1 ]]; then
|
||||
npmrc="$(mktemp)"
|
||||
trap 'rm -f "$npmrc"' EXIT
|
||||
chmod 600 "$npmrc"
|
||||
printf '%s\n' \
|
||||
'@govoplan:registry=https://git.add-ideas.de/api/packages/GovOPlaN/npm/' \
|
||||
"//git.add-ideas.de/api/packages/GovOPlaN/npm/:_authToken=$PACKAGE_TOKEN" \
|
||||
> "$npmrc"
|
||||
NPM_CONFIG_USERCONFIG="$npmrc" npm publish "./${webui_packages[0]}" \
|
||||
--ignore-scripts --access public \
|
||||
--registry https://git.add-ideas.de/api/packages/GovOPlaN/npm/
|
||||
elif (( ${#webui_packages[@]} )); then
|
||||
echo "Exact WebUI package is already present; skipping immutable retry."
|
||||
fi
|
||||
@@ -1,5 +1,11 @@
|
||||
# GovOPlaN Files Codex Guide
|
||||
|
||||
## Documentation Contract
|
||||
|
||||
- Treat documentation as part of every behavior change. Update this module's manifest-driven `DocumentationTopic` contributions for affected user and administrator behavior.
|
||||
- Keep feature content here; `govoplan-docs` projects it without importing Files internals.
|
||||
- Maintain a static user/admin baseline and run `/mnt/DATA/git/govoplan/tools/checks/check-manifest-shapes.py` after behavior or manifest changes.
|
||||
|
||||
## Scope
|
||||
|
||||
This repository owns the `files` module: managed file storage APIs, file metadata, shares, uploads/downloads, folder and pattern helpers, backend module manifest, and `@govoplan/files-webui`.
|
||||
|
||||
@@ -55,6 +55,15 @@ for example for managed attachment selection and campaign file sharing. Files
|
||||
does not import campaign internals; campaign share/existence checks use the core
|
||||
`campaigns.access` capability registered by the campaign module.
|
||||
|
||||
Files also publishes the optional `privacy.dsar.files` capability. The Core
|
||||
data-subject workflow can use it to collect bounded, tenant-scoped file,
|
||||
version, share, folder, evidence, and non-secret configuration metadata for a
|
||||
direct membership subject. The provider may revoke a subject-targeted share or
|
||||
detach mutable actor references idempotently, but it never exports credential
|
||||
material or raw bytes and never bypasses Files retention, legal hold, evidence,
|
||||
purge approval, audit, or recovery controls. See the handbook's data-subject
|
||||
request coverage section for the review and erasure boundary.
|
||||
|
||||
Platform RBAC and governance rules are documented in `govoplan-core/docs/`.
|
||||
|
||||
Managed files can carry source provenance for connector and import workflows.
|
||||
@@ -105,7 +114,8 @@ run profile policy with the `browse` operation, and support Seafile,
|
||||
WebDAV/Nextcloud, and SMB when the optional `smb` extra is installed. Seafile
|
||||
profiles browse libraries and directories via Seafile's read-only API; profiles
|
||||
can still opt into WebDAV browsing by setting `metadata.webdav_endpoint_url` or
|
||||
`metadata.browse_protocol` to `webdav`.
|
||||
`metadata.browse_protocol` to `webdav`. S3-compatible profiles support bucket
|
||||
and prefix browsing when the optional `s3` extra is installed.
|
||||
`POST /api/v1/files/connectors/profiles/{profile_id}/import` imports a Seafile
|
||||
or WebDAV/Nextcloud/SMB file into managed storage through the same governance,
|
||||
conflict handling, source provenance, and connector audit path as direct
|
||||
@@ -113,17 +123,38 @@ uploads. The Seafile provider uses account-token auth and the native file
|
||||
download-link API; Nextcloud and generic WebDAV profiles use authenticated `GET`
|
||||
requests against the configured WebDAV endpoint. SMB profiles use
|
||||
`smb://server[:port]/share[/path]` endpoints and deployment-owned or encrypted
|
||||
stored credentials through `smbprotocol`. Because that SDK cannot accept a
|
||||
preconnected socket and may follow DFS referrals to additional hosts, live SMB
|
||||
access fails closed in both public-only and private-network deployments. It will
|
||||
remain disabled until every initial connection and referral target can be
|
||||
policy-validated and pinned.
|
||||
stored credentials through `smbprotocol`. Files installs a pinned transport for
|
||||
every initial session, reconnect, alias, and DFS referral target. The deployment
|
||||
private-network policy is re-evaluated immediately before each socket opens.
|
||||
|
||||
The S3 browse/import implementation and S3 managed-storage backend currently
|
||||
fail closed before creating a `boto3` client. Botocore does not yet use the
|
||||
GovOPlaN pinned HTTP transport and may manage redirects itself, so neither an
|
||||
explicit endpoint nor SDK endpoint discovery is allowed until both peer pinning
|
||||
and redirect revalidation are enforced.
|
||||
S3 connector browse/import binds botocore HTTP and HTTPS pools to the same pinned
|
||||
socket policy. Retries, redirects, endpoint discovery, bucket aliases, and new
|
||||
connections are therefore revalidated while TLS keeps the configured hostname
|
||||
for SNI and certificate checks. Connector clients do not use outbound proxies or
|
||||
ambient AWS credential discovery; configure credentials on the governed profile,
|
||||
or explicitly use an anonymous profile for public objects. Keep connector spaces
|
||||
read-only unless a remote write is explicitly needed.
|
||||
An S3 profile with the `write` capability can back a two-way connector space.
|
||||
`POST /api/v1/files/connector-spaces/{space_id}/write-back` then conditionally
|
||||
creates or replaces one remote object from a managed file, records durable
|
||||
recovery intent before the provider effect, and verifies request/content markers
|
||||
afterward. Automatic remote delete, rename, move, and ACL propagation remain
|
||||
disabled; other connector providers remain read-only.
|
||||
An incompatible SDK upgrade fails closed before a usable client/session is
|
||||
returned.
|
||||
|
||||
Durable platform storage has a separate deployment boundary: installer-owned
|
||||
Garage is accepted only at its exact generated endpoint, while an
|
||||
operator-selected external backend requires a clean HTTPS origin and
|
||||
`FILE_STORAGE_S3_ENDPOINT_TRUSTED=true`. That flag is deployment configuration,
|
||||
cannot be supplied through a Files connector profile, and does not replace
|
||||
operator responsibility for DNS, certificates, egress, bucket policy,
|
||||
versioning, and recovery.
|
||||
|
||||
The actual local/S3 backend implementation is owned by Core so Files, Campaign,
|
||||
and workers resolve the same object namespace. Files owns file metadata and key
|
||||
layout. Node-local storage is supported only for `local` or one-host
|
||||
`host-shared` profiles; multi-host `shared` deployments require S3.
|
||||
|
||||
Destructive Files-module retirement applies the same credential lifecycle before
|
||||
dropping tables. Every remaining Files-owned encrypted connector secret is
|
||||
@@ -139,17 +170,46 @@ Connector and collaboration ownership boundaries are documented in
|
||||
`docs/CONNECTOR_BOUNDARY.md` and `docs/DOCUMENT_COLLABORATION_BOUNDARY.md`.
|
||||
The role-adaptive user, administration, integration, and operator guide is the
|
||||
[Files handbook](docs/FILES_HANDBOOK.md).
|
||||
The Files route, connector surfaces, consequence classes, and pattern-language
|
||||
verification are recorded in
|
||||
[Files interface pattern migration](docs/INTERFACE_PATTERN_MIGRATION.md).
|
||||
|
||||
ZIP uploads are processed without buffering the whole archive in memory. The API
|
||||
spools incoming ZIP request bodies to a bounded temporary file, then extracts
|
||||
members with per-file and total extracted-size limits before storing managed
|
||||
files.
|
||||
Archive imports use a two-phase preview and confirmation flow for ZIP, TAR,
|
||||
TAR.GZ, TAR.BZ2, and TAR.XZ. Requests are spooled to bounded temporary files;
|
||||
the server validates paths, entry count, expanded size, and expansion ratio,
|
||||
then returns a 30-minute tenant/user-bound preview token. Confirmation reuploads
|
||||
the original archive and stores only the selected members. Password-protected
|
||||
ZIP passwords remain request-only and are never included in the preview token.
|
||||
|
||||
Managed blob writes and applied orphan cleanup use Core's durable recovery
|
||||
ledger. On PostgreSQL, intent, request digests, recovery mode, and a distributed
|
||||
lease are committed before physical storage effects; the Files session commit
|
||||
verifies database and streamed object evidence, while rollback compensates only
|
||||
a newly reserved unreferenced key. Development SQLite records blob intent in
|
||||
the caller transaction to avoid its second-writer deadlock, then verifies on
|
||||
commit or reconstructs compensation evidence after handled rollback. Because a
|
||||
hard loss before that commit can leave an unrecorded object, SQLite requires a
|
||||
complete integrity scan after a crash and is not a production recovery profile.
|
||||
New object keys are opaque and do not retain the uploaded filename. Uncertain
|
||||
or mismatched effects remain visible through Ops.
|
||||
|
||||
Operators with `files:file:admin` can run bounded, resumable integrity scans in
|
||||
Administration. Scan batches and finding actions carry monotonic revisions;
|
||||
stale resume, recheck, or cleanup requests fail before touching object storage.
|
||||
Orphan cleanup always requires a dry-run preview followed by separate
|
||||
confirmation and records recovery-ledger evidence.
|
||||
|
||||
Bulk rename and transfer APIs are owner-scoped: callers must provide the active
|
||||
user or group file space with `owner_type` and `owner_id`. The storage layer
|
||||
keeps a named legacy file-only helper for historical callers that lack owner
|
||||
context, and regression tests cover its write-access checks.
|
||||
|
||||
Optional producer modules can persist generated output through the provider-
|
||||
neutral `files.artifact_store` capability. Files remains responsible for upload
|
||||
authorization, ownership, path normalization, versioning, and blob storage;
|
||||
producers receive stable file/version references without importing Files
|
||||
internals. See [Generated Artifact Store](docs/GENERATED_ARTIFACT_STORE.md).
|
||||
|
||||
## Release packaging
|
||||
|
||||
The repository root includes a `package.json` for git-based WebUI installs. It exports the package `@govoplan/files-webui` from `webui/src` so release builds can depend on tagged git refs instead of local `file:` paths.
|
||||
|
||||
@@ -19,6 +19,7 @@ normal product workflows and can share the same governance model:
|
||||
- Nextcloud through WebDAV
|
||||
- generic WebDAV
|
||||
- SMB through the optional `smb` extra
|
||||
- S3-compatible stores through the optional `s3` extra
|
||||
|
||||
These providers are surfaced through connector descriptors at
|
||||
`GET /api/v1/files/connectors/providers`. Provider descriptors declare whether
|
||||
@@ -60,11 +61,14 @@ Every provider must:
|
||||
- use a transport that pins every connection to a policy-validated DNS/IP answer
|
||||
and revalidates redirects; SDK transports without that guarantee fail closed
|
||||
|
||||
Live SMB access is disabled in all modes until `smbprotocol` initial connections
|
||||
and DFS referral targets can be pinned and policy-validated. An explicit IP is
|
||||
not sufficient because the server may still issue a referral to another peer.
|
||||
Live S3 access is likewise disabled until `boto3`/botocore can be bound to the
|
||||
pinned transport, including SDK-managed redirects and endpoint discovery.
|
||||
SMB initial connections, reconnects, aliases, and DFS referral targets are
|
||||
created through a Files-owned pinned `smbprotocol` transport. S3 HTTP/HTTPS pools
|
||||
use the equivalent botocore adapter for every connection selected by retries,
|
||||
redirects, endpoint discovery, and virtual-host addressing. Both adapters apply
|
||||
the deployment-wide private-network policy at socket creation. S3 keeps the
|
||||
configured hostname for TLS SNI and certificate verification, but does not use
|
||||
outbound proxies or ambient AWS credential discovery. An incompatible optional
|
||||
SDK release fails closed before a usable session or client is returned.
|
||||
|
||||
## Non-Goals For Files
|
||||
|
||||
|
||||
+416
-94
@@ -1,6 +1,6 @@
|
||||
# GovOPlaN Files Handbook
|
||||
|
||||
This handbook describes the Files module as implemented in version `0.1.9`.
|
||||
This handbook describes the Files module as implemented in version `0.1.18`.
|
||||
It is the operational source of truth for users, process owners, administrators,
|
||||
operators, auditors, and module integrators. Statements about future behavior
|
||||
are marked **planned**; an unmarked statement describes the current code.
|
||||
@@ -8,7 +8,8 @@ are marked **planned**; an unmarked statement describes the current code.
|
||||
Files is a governed snapshot store. It owns managed file content, versions,
|
||||
logical folders, shares, source provenance, and the evidence that another
|
||||
GovOPlaN module used a particular file version. It can browse selected external
|
||||
stores read-only and import a frozen copy. It is not a general remote filesystem,
|
||||
stores and import a frozen copy. Explicit, conditional S3 write-back is available
|
||||
only through an administrator-enabled two-way connector space. Files is not a general remote filesystem,
|
||||
a document collaboration engine, or a records-management system.
|
||||
|
||||
## Choose a reading path
|
||||
@@ -24,32 +25,65 @@ a document collaboration engine, or a records-management system.
|
||||
| Verify a release or scenario | [Acceptance scenarios](#acceptance-scenarios) |
|
||||
| Check whether an idea exists today | [Implemented and planned boundary](#implemented-and-planned-boundary) |
|
||||
|
||||
The Files-owned interface archetypes, consequence classes, disabled-state
|
||||
wording, and verification evidence are recorded in
|
||||
[Files Interface Pattern Migration](INTERFACE_PATTERN_MIGRATION.md).
|
||||
|
||||
## The service contract
|
||||
|
||||
A managed file is a tenant-scoped logical asset with exactly one user or group
|
||||
owner, a normalized path, and a current version. The current version points to a
|
||||
blob record containing the storage location, SHA-256 checksum, byte size, and
|
||||
content type. Current service paths append a version when connector sync finds
|
||||
content type. A protected blob also records its Encryption envelope, protection
|
||||
discriminator, stored-ciphertext checksum, and stored-ciphertext size. Current
|
||||
service paths append a version when connector sync finds
|
||||
changed content; they do not mutate the previous version record.
|
||||
|
||||
Content with the same tenant, SHA-256 checksum, and size can reuse one blob.
|
||||
This is storage deduplication and integrity evidence, not proof of authorship or
|
||||
source authenticity.
|
||||
Unprotected content with the same tenant, plaintext SHA-256 checksum, size, and
|
||||
protection discriminator can reuse one blob. Protected content is deduplicated
|
||||
only inside the same vault/profile discriminator; ciphertext is never silently
|
||||
reused across protection boundaries. Plaintext checksums remain semantic
|
||||
version evidence, while download and integrity scans verify stored ciphertext
|
||||
before asking Encryption to open it. Neither digest proves authorship or source
|
||||
authenticity.
|
||||
|
||||
The main domain objects are:
|
||||
|
||||
| Object | Meaning | Lifecycle today |
|
||||
| --- | --- | --- |
|
||||
| File asset | The user-facing file identity, owner, logical path, description, metadata, and current version | Created, organized, shared, and soft-deleted |
|
||||
| File version | A numbered snapshot of one asset and its blob | Appended by connector sync when bytes change; retained |
|
||||
| File blob | Stored bytes plus checksum, size, backend, key, reference count, and optional retention timestamp | Reused within a tenant; no automated garbage collection |
|
||||
| Folder | An explicit logical path in a user or group space | Created, moved/renamed through organize operations, and soft-deleted |
|
||||
| File asset | The user-facing file identity, owner, logical path, description, metadata, current version, retention deadline, legal hold, and lifecycle revision | Created, organized, shared, soft-deleted, restored, or governed-purged |
|
||||
| File version | A numbered snapshot of one asset and its blob | Appended by connector sync when bytes change; retained until its asset is eligible for purge |
|
||||
| File blob | Stored plaintext semantics plus stored-byte integrity, backend key, optional Encryption envelope, reference count, and retention timestamp | Reused only within a tenant and matching protection boundary; collected only after a fenced reference recheck |
|
||||
| Folder | An explicit logical path in a user or group space | Created, moved/renamed, soft-deleted, and restored when its original path is free |
|
||||
| Share | A grant from an asset to a user, group, tenant, or campaign with `read`, `write`, or `manage` permission | Created or updated; no public revocation endpoint yet |
|
||||
| Connector profile | A governed external endpoint, scope, optional credential link, policy, and descriptive capabilities | Created, updated, disabled, or credential-scrubbed on deletion |
|
||||
| Connector credential | Reusable authentication material, optionally limited to a provider and scope | Encrypted when database-managed; immediately scrubbed on deletion |
|
||||
| Connector policy | Allow and deny rules inherited from system through tenant to one leaf scope | Evaluated before configuration and connector use |
|
||||
| Connector space | A read-only, manually synchronized remote folder/library linked to a user or group space | Created, updated, disabled, and soft-deleted |
|
||||
| Connector space | A manually synchronized remote folder/library linked to a user or group space; read-only by default, with explicit S3 two-way mode available | Created, updated, disabled, soft-deleted, and restored |
|
||||
| Campaign attachment use | Evidence connecting a campaign job or entry to an exact asset, version, blob, checksum, and stage | Retained for campaign execution evidence |
|
||||
| Form evidence upload grant | A one-use, hash-only bearer grant tied to an exact Form instance/revision, purpose, custodian, size, and media-type policy | Issued for at most 15 minutes, consumed by one managed upload, then retained as evidence provenance |
|
||||
|
||||
## Deployment configuration packages
|
||||
|
||||
Files registers the `files.configuration` capability for `managed_storage`
|
||||
fragments. Managed content storage is deployment-owned: the installer selects
|
||||
local persistent storage, managed Garage, or an external S3-compatible service
|
||||
and mounts a validated non-secret infrastructure capability receipt. Files
|
||||
does not copy that endpoint or its credentials into module-owned tables.
|
||||
|
||||
Preflight compares the receipt's `files.storage` capability with the effective
|
||||
runtime backend. It validates backend kind, sanitized S3 endpoint, bucket,
|
||||
Garage management marker or external trust marker, an absolute persistent path
|
||||
for local storage, and the presence of Files-owned `env:` secret references.
|
||||
Secret values are never read into a plan, diagnostic, export, or fragment.
|
||||
|
||||
When runtime and receipt agree, the plan reports `skip`: the desired binding is
|
||||
already effective, and repeated apply is a no-op. A mismatch blocks import and
|
||||
explains which deployment setting must be reconciled. The provider deliberately
|
||||
does not mutate process environment, migrate stored objects, probe remote
|
||||
storage, or reinterpret an infrastructure replacement as safe. Use the Files
|
||||
integrity and Ops checks after deployment and complete migration/recovery review
|
||||
before changing an active backend.
|
||||
|
||||
## User tasks
|
||||
|
||||
@@ -72,9 +106,12 @@ Files administrator; governed API operations can administer other tenant-owned
|
||||
Files resources when their owner is specified. Linked connector spaces appear
|
||||
beside managed spaces when they are active and visible to the user.
|
||||
|
||||
A connector space is intentionally read-only. It is a view of an approved
|
||||
remote location and a starting point for importing or synchronizing selected
|
||||
files into managed storage; it is not a mounted write-through filesystem.
|
||||
A connector space is read-only by default. It is a view of an approved remote
|
||||
location and a starting point for importing or synchronizing selected files
|
||||
into managed storage. An administrator may opt an S3 space into two-way mode
|
||||
only after enabling the profile's `write` capability. Even then, Files exposes
|
||||
explicit, conditional file write-back—not a mounted filesystem. Automatic
|
||||
remote delete, rename, move, and ACL propagation stay disabled.
|
||||
|
||||
### Upload files
|
||||
|
||||
@@ -94,21 +131,29 @@ An ordinary upload does not append a new version to an existing asset. The
|
||||
`overwrite` strategy retires the old asset at that path. Version-preserving
|
||||
updates currently belong to connector sync.
|
||||
|
||||
### Upload and unpack ZIP files
|
||||
### Preview and unpack archives
|
||||
|
||||
The UI can unpack a ZIP upload. The request body is spooled to a bounded
|
||||
temporary file rather than buffered wholly in memory. The defaults are:
|
||||
The UI previews ZIP, TAR, TAR.GZ, TAR.BZ2, and TAR.XZ before writing managed
|
||||
files. Users may select individual files or complete folders. The request is
|
||||
spooled to a bounded temporary file rather than buffered wholly in memory. The
|
||||
defaults are:
|
||||
|
||||
- 250 MiB compressed request and total extracted data;
|
||||
- 250 MiB compressed request;
|
||||
- 50 MiB per extracted member;
|
||||
- 1,000 non-directory members;
|
||||
- encrypted archives are rejected;
|
||||
- member paths are normalized and `..` traversal is rejected;
|
||||
- the actual bytes read are counted, not only the ZIP header declarations.
|
||||
- 2 GiB total expanded data;
|
||||
- 10,000 declared entries;
|
||||
- 100:1 maximum expansion ratio;
|
||||
- a 30-minute preview token bound to the tenant, user, archive digest, and
|
||||
destination;
|
||||
- password-protected ZIP support with request-only password handling;
|
||||
- traversal, duplicate paths, links, devices, and other special entries are
|
||||
rejected;
|
||||
- actual bytes read are counted, not only archive header declarations.
|
||||
|
||||
The byte limits use `FILE_UPLOAD_ZIP_MAX_BYTES` and
|
||||
`FILE_UPLOAD_MAX_BYTES`. The 1,000-member limit is currently fixed in the
|
||||
extraction service.
|
||||
The browser retains the selected archive and password until confirmation.
|
||||
Confirmation reuploads the archive, verifies the token and digest, repeats all
|
||||
safety checks, and commits only the selected files. No preview archive or
|
||||
password is retained server-side.
|
||||
|
||||
### Organize files and folders
|
||||
|
||||
@@ -129,6 +174,73 @@ Recursive folder deletion is the default. It soft-deletes the selected folder,
|
||||
its child folders, and files below it. A non-recursive delete fails when the
|
||||
folder is not empty.
|
||||
|
||||
### Restore, retain, hold, and purge
|
||||
|
||||
Soft deletion is reversible for callers with `files:file:restore`. Restoring a
|
||||
file preserves its asset identity, complete version history, blob references,
|
||||
and source provenance. Recursive folder restore reactivates the folder tree and
|
||||
its deleted files. Connector-space restore reactivates only the local link.
|
||||
Each restore fails if an active resource already occupies the original path or
|
||||
label.
|
||||
|
||||
Retention and legal hold are independent lifecycle controls. A caller with
|
||||
`files:file:retention` supplies the current lifecycle revision, a reason, an
|
||||
optional retained-until time, and legal-hold state. Stale revisions fail rather
|
||||
than overwriting a concurrent decision.
|
||||
|
||||
Hard purge is deliberately separate from ordinary delete:
|
||||
|
||||
1. A caller with `files:file:purge` previews 1–100 soft-deleted assets.
|
||||
2. The preview reports retention, legal-hold, active-share, Campaign-evidence,
|
||||
and Form-evidence blockers and returns a SHA-256 over current lifecycle and
|
||||
blob-reference state.
|
||||
3. Execution requires that exact hash, a stable idempotency key, an approval
|
||||
reference, and the literal `PURGE` confirmation.
|
||||
4. The recovery-ledger operation and tenant purge fence are durable before the
|
||||
irreversible database transaction. A stale preview or blocker rejects the
|
||||
operation without erasure.
|
||||
5. Purge removes eligible asset, version, and inactive-share rows and
|
||||
recalculates blob reference counts. It does not delete bytes inline.
|
||||
6. A separately authorized bounded blob-GC call takes the same per-blob lease
|
||||
used by uploads, rechecks all `FileVersion` references, deletes the exact
|
||||
object, verifies absence, and only then deletes `FileBlob` metadata.
|
||||
|
||||
Automatic time-based purge scheduling is not implemented. Operators initiate
|
||||
preview, execute, and garbage collection under their local retention process.
|
||||
|
||||
### Data-subject request coverage
|
||||
|
||||
Files registers `privacy.dsar.files` when the module is active. The provider
|
||||
requires a corroborated tenant membership identifier (or a namespaced Files
|
||||
user reference), searches only that tenant, and fails explicitly if its bounded
|
||||
result limit would be exceeded. It reports managed assets, exact versions,
|
||||
folders, user-targeted shares, Form and Campaign evidence, connector
|
||||
configuration actor references, and integrity-operation evidence.
|
||||
|
||||
The export includes only governed metadata. It never embeds file bytes, blob
|
||||
storage keys, passwords, tokens, environment-variable names, secret-provider
|
||||
references, or encrypted connector values. A reviewer follows the authorized
|
||||
version download route when the file itself must be inspected.
|
||||
|
||||
The erasure plan deliberately separates four outcomes:
|
||||
|
||||
- an active share aimed at the subject can be revoked idempotently;
|
||||
- mutable creator/updater references can be detached after tenant, subject, and
|
||||
current-value revalidation;
|
||||
- legal hold, active retention, submitted Form evidence, Campaign delivery
|
||||
evidence, connector configuration history, and integrity evidence are
|
||||
retained with a reason; and
|
||||
- unstructured file content, ownership, filenames, and paths require manual
|
||||
review.
|
||||
|
||||
DSAR execution never invokes physical byte deletion. If the privacy decision
|
||||
authorizes erasure, the operator must use the separate Files soft-delete, purge
|
||||
preview, approval, execution, and blob-GC sequence. This preserves its distinct
|
||||
authority, evidence blockers, audit trail, distributed fencing, and recovery
|
||||
ledger semantics. Email, account, or identity selectors alone are insufficient
|
||||
because Files does not import the Access directory; the Access search supplies
|
||||
the corroborated membership reference for Files coverage.
|
||||
|
||||
### Find and download files
|
||||
|
||||
Files can list by owner and path, use cursor pagination, and consume incremental
|
||||
@@ -169,7 +281,22 @@ inside the chosen owner space:
|
||||
- identical checksum and size updates provenance and returns `unchanged`;
|
||||
- changed bytes append a version and return `updated`.
|
||||
|
||||
Browse, import, and sync never write, rename, or delete the remote source.
|
||||
Browse, import, and inbound sync never mutate the remote source. An S3 space in
|
||||
explicit two-way mode can write a selected managed file to one remote object
|
||||
path through `POST /api/v1/files/connector-spaces/{space_id}/write-back`.
|
||||
Creating a path uses `If-None-Match`; overwriting requires the currently
|
||||
observed ETag or version and uses a conditional request. The provider object is
|
||||
then re-read and must contain both the expected content digest and the recovery
|
||||
operation marker. Uncertain or mismatching outcomes remain fenced and visible
|
||||
in Ops. Remote deletion, rename, move, and permission propagation are disabled.
|
||||
|
||||
Connector administration separates endpoint profiles, reusable credentials,
|
||||
and inherited policy. Ordinary setup uses typed fields and provider discovery;
|
||||
provider metadata JSON is available only under advanced compatibility options.
|
||||
Read-only deployment entries explain where they must be changed, and disabled
|
||||
actions identify the missing permission, target, input, or running operation.
|
||||
The contextual help icon opens the configured Help Center topic when Docs is
|
||||
enabled and the hosted GovOPlaN documentation otherwise.
|
||||
|
||||
## Process perspective
|
||||
|
||||
@@ -187,9 +314,10 @@ The managed-file flow is:
|
||||
8. Connector-originated operations also emit their connector audit evidence.
|
||||
|
||||
Blob storage is not part of the database transaction. An object may therefore
|
||||
be left without committed metadata after a process or database failure. No
|
||||
automatic orphan reconciliation exists yet; operators must account for this in
|
||||
integrity checks and retention plans.
|
||||
be left without committed metadata after a process or database failure. The
|
||||
operator integrity API scans database blobs and the tenant storage prefix in
|
||||
bounded, resumable phases. It reports orphan objects before any cleanup and
|
||||
never deletes them as part of a scan.
|
||||
|
||||
### Governed connector import
|
||||
|
||||
@@ -270,6 +398,10 @@ those relationships first.
|
||||
| `files:file:organize` | Create folders, rename, move/copy, and manage linked connector spaces |
|
||||
| `files:file:share` | Create or update file shares |
|
||||
| `files:file:delete` | Soft-delete accessible writable files and folders |
|
||||
| `files:file:restore` | Restore owned or administered soft-deleted files, folders, and connector-space links |
|
||||
| `files:file:retention` | Set retention deadlines and legal holds with optimistic revision checks |
|
||||
| `files:file:purge` | Preview and execute irreversible metadata purge and collect unreferenced blobs |
|
||||
| `files:connector:write` | Write an accessible managed file to an explicitly writable connector space |
|
||||
| `files:file:admin` | Administer all Files spaces and connector settings in the active tenant |
|
||||
|
||||
The `file_manager` role template grants all normal file operations except
|
||||
@@ -289,12 +421,14 @@ Keep these definitions separate:
|
||||
local policy, and descriptive operation capabilities;
|
||||
- a **policy** restricts what a scope may configure or use;
|
||||
- a **connector space** links one approved profile/library/path to one user or
|
||||
group and always uses manual, read-only synchronization today.
|
||||
group, uses manual synchronization, and is read-only unless explicitly
|
||||
configured for the supported two-way S3 write boundary.
|
||||
|
||||
Profile capability values such as `browse`, `import`, and `sync` are stored and
|
||||
returned, but they are descriptive today. Provider implementation and policy
|
||||
checks enforce actual availability; do not use the capability list as the sole
|
||||
security control.
|
||||
Profile capability values such as `browse`, `import`, `sync`, and `write` are
|
||||
stored and returned. `write` is additionally enforced for S3 write-back, but
|
||||
provider implementation, connector-space mode, operation permission, resource
|
||||
access, and inherited policy all remain mandatory; never use the capability
|
||||
list as the sole security control.
|
||||
|
||||
Profiles, credentials, and policies support `system`, `tenant`, `user`,
|
||||
`group`, and `campaign` scopes. A normal user sees system and active-tenant
|
||||
@@ -378,15 +512,15 @@ in a development/test runtime.
|
||||
| Seafile | Read-only native API browse/download-link import and manual sync implemented using the pinned HTTP transport; WebDAV opt-in supported |
|
||||
| Nextcloud | Read-only WebDAV browse/import/manual sync implemented using the pinned HTTP transport |
|
||||
| Generic WebDAV | Read-only browse/import/manual sync implemented using the pinned HTTP transport |
|
||||
| SMB | Browse/import code and descriptor implemented, but every live connection fails closed until initial connections and DFS referrals can be policy-validated and pinned |
|
||||
| S3 connector | Browse/import code and descriptor implemented, but every live SDK connection fails closed until botocore connections, redirects, and endpoint discovery can be validated and pinned |
|
||||
| SMB | Read-only browse/import/manual sync implemented through a pinned smbprotocol transport for initial peers, reconnects, aliases, and DFS referral targets |
|
||||
| S3 connector | Bucket/prefix browse, import, manual inbound sync, and explicit conditional write-back implemented through pinned botocore pools covering retries, redirects, endpoint discovery, and provider aliases; automatic remote delete/rename/move/ACL propagation disabled |
|
||||
| SharePoint and OneDrive | Provider keys/descriptors reserved; live Microsoft Graph browse/import is planned |
|
||||
| NFS and local connector | Described as optional future providers; the local managed-storage backend is a different feature |
|
||||
|
||||
Provider descriptors are available from
|
||||
`GET /api/v1/files/connectors/providers`. Use their `implemented`, `installed`,
|
||||
and support fields for display, but expect a fail-closed transport error where
|
||||
the table above says live access is disabled.
|
||||
and support fields for display. An incompatible optional SDK release fails closed
|
||||
before it can return a usable client or session.
|
||||
|
||||
## Operator runbook
|
||||
|
||||
@@ -398,8 +532,15 @@ the table above says live access is disabled.
|
||||
| `FILE_STORAGE_LOCAL_ROOT` | `runtime/files` | Primary local write/read root |
|
||||
| `FILE_STORAGE_LOCAL_FALLBACK_ROOTS` | empty | Comma-separated older read-only roots checked after the primary root |
|
||||
| `FILE_STORAGE_S3_ENDPOINT_URL` and related `FILE_STORAGE_S3_*` values | deployment-specific | S3-compatible endpoint, region, credentials, and bucket |
|
||||
| `FILE_UPLOAD_MAX_BYTES` | 50 MiB | Direct-upload and ZIP-member maximum |
|
||||
| `FILE_UPLOAD_ZIP_MAX_BYTES` | 250 MiB | ZIP request and extracted-total maximum |
|
||||
| `FILE_STORAGE_S3_DEPLOYMENT_MANAGED` | `false` | Installer-only trust marker for the exact `http://garage:3900` service; never use it for another endpoint |
|
||||
| `FILE_STORAGE_S3_ENDPOINT_TRUSTED` | `false` | Deployment-owner acknowledgement for one clean HTTPS external S3 origin; never expose this through connector configuration |
|
||||
| `GOVOPLAN_STATE_PROFILE` | `local` | Selects `local`, one-host `host-shared`, or multi-host `shared` state validation |
|
||||
| `FILE_UPLOAD_MAX_BYTES` | 50 MiB | Direct-upload and extracted archive-member maximum |
|
||||
| `FILE_UPLOAD_ZIP_MAX_BYTES` | 250 MiB | Compressed archive request maximum (legacy name retained for compatibility) |
|
||||
| `FILE_ARCHIVE_MAX_ENTRIES` | 10,000 | Maximum declared archive entries |
|
||||
| `FILE_ARCHIVE_MAX_EXPANDED_BYTES` | 2 GiB | Maximum expanded archive bytes |
|
||||
| `FILE_ARCHIVE_MAX_EXPANSION_RATIO` | 100 | Maximum expanded-to-compressed ratio |
|
||||
| `FILE_ARCHIVE_PREVIEW_TTL_SECONDS` | 1,800 | Lifetime of the sealed archive preview token |
|
||||
| `MASTER_KEY_B64` | development fallback only | Encrypts database-managed connector secrets |
|
||||
|
||||
The local backend is the operational baseline. It resolves every storage key
|
||||
@@ -407,16 +548,19 @@ under the configured root and rejects escape attempts. Fallback roots support a
|
||||
controlled storage-root migration: new writes go to the primary root while
|
||||
reads can still find older objects.
|
||||
|
||||
The S3 managed-storage adapter currently fails closed before creating a boto3
|
||||
client because the SDK cannot yet guarantee connection-time DNS/IP pinning and
|
||||
redirect revalidation. Do not select `FILE_STORAGE_BACKEND=s3` for a live Files
|
||||
deployment until that boundary is implemented and the status above changes.
|
||||
The supported installer may provision a deployment-owned Garage service at the
|
||||
exact `http://garage:3900` endpoint and set
|
||||
`FILE_STORAGE_S3_DEPLOYMENT_MANAGED=true`. An operator-selected external S3
|
||||
backend instead requires a clean HTTPS origin and
|
||||
`FILE_STORAGE_S3_ENDPOINT_TRUSTED=true`. Both are deployment authority, not a
|
||||
general connector or private-network bypass. Core owns the shared backend
|
||||
implementation; Files owns metadata and the Files key namespace.
|
||||
|
||||
Multiple API replicas require the same durable blob namespace. Separate local
|
||||
container filesystems will produce incomplete reads. Until a pinned shared
|
||||
object-storage transport is available, use one durable shared mount or constrain
|
||||
Files traffic to a deployment topology that preserves one consistent local
|
||||
root.
|
||||
container filesystems will produce incomplete reads. Use `host-shared` with one
|
||||
durable shared mount only for same-host replicas. Independent hosts require the
|
||||
`shared` profile with external S3, PostgreSQL, Redis, a stable installation id,
|
||||
and one immutable module composition.
|
||||
|
||||
### Connector egress
|
||||
|
||||
@@ -441,15 +585,29 @@ The built-in HTTP transport:
|
||||
- bounds structured responses to 16 MiB and file transfers to 512 MiB by
|
||||
default.
|
||||
|
||||
The S3 SDK adapter applies the same socket rule to every botocore pool selected
|
||||
for a retry, redirect, discovered endpoint, or virtual-host bucket alias. The
|
||||
original authority remains in the request and TLS SNI/certificate check. S3
|
||||
connector clients use no outbound proxy and never discover ambient AWS
|
||||
credentials: configure both access and secret keys on the governed profile, or
|
||||
use an anonymous profile for a public source.
|
||||
|
||||
The SMB adapter owns a separate connection cache and replaces smbprotocol's TCP
|
||||
factory process-wide with the stricter pinned socket. Initial peers, reconnects,
|
||||
server aliases, domain-controller connections, and DFS referral targets therefore
|
||||
pass the same policy at connection time. Signing is required by default; enable
|
||||
SMB encryption on the profile where the server supports it.
|
||||
|
||||
Override the connector response limits with
|
||||
`GOVOPLAN_CONNECTOR_MAX_STRUCTURED_RESPONSE_BYTES` and
|
||||
`GOVOPLAN_CONNECTOR_MAX_FILE_TRANSFER_BYTES`. The smaller applicable limit wins
|
||||
when an import is also subject to `FILE_UPLOAD_MAX_BYTES`.
|
||||
|
||||
Never work around a pinning failure by adding a raw IP, disabling TLS, or
|
||||
enabling private networks. SMB may redirect through DFS, and S3 SDKs may perform
|
||||
their own redirects or endpoint discovery; both remain disabled even for an IP
|
||||
literal until every connection peer can be governed.
|
||||
Never work around a connector pinning failure by adding a raw IP, disabling TLS,
|
||||
or enabling private networks. A failure means the peer policy rejected an actual
|
||||
connection destination or the installed SDK no longer exposes the verified
|
||||
transport seam. The separately configured platform S3 backend is trusted only by
|
||||
the deployment owner and is not selectable by a user or connector profile.
|
||||
|
||||
### Backup and restore
|
||||
|
||||
@@ -459,13 +617,113 @@ include:
|
||||
- Files database rows, including asset/version/blob relationships, shares,
|
||||
connector settings, and campaign attachment-use evidence;
|
||||
- every object below `FILE_STORAGE_LOCAL_ROOT` and any still-used fallback root;
|
||||
or the complete S3 bucket/prefix and version/lifecycle evidence for an S3
|
||||
backend;
|
||||
- the exact `MASTER_KEY_B64` needed to decrypt retained connector credentials;
|
||||
- deployment-owned connector profile files, referenced CA bundles, and secret
|
||||
environment configuration where those definitions are in use.
|
||||
|
||||
There is no Files backup/restore API and no automated blob integrity or orphan
|
||||
reconciliation job. Use a write quiesce or coordinated snapshots so database
|
||||
references and objects represent the same recovery point.
|
||||
There is no Files backup/restore API. Use a write quiesce or coordinated
|
||||
snapshots so database references and objects represent the same recovery point.
|
||||
The integrity API verifies a restored set, but it does not replace a coordinated
|
||||
backup.
|
||||
|
||||
Operators normally use **Administration > File integrity**. The equivalent API
|
||||
creates a scan with `POST /api/v1/files/integrity/scans`, then calls
|
||||
`POST /api/v1/files/integrity/scans/{scan_id}/run` with the scan's current
|
||||
`expected_revision` until it reports `completed`. Each call advances at most
|
||||
the persisted batch size, so a stopped operator or worker can resume from the
|
||||
committed blob/object cursors. Concurrent or stale actions receive `409` before
|
||||
the storage backend is invoked; reload the scan and inspect the newer state.
|
||||
|
||||
Findings distinguish:
|
||||
|
||||
- `missing`: metadata references an absent object;
|
||||
- `size_mismatch` or `checksum_mismatch`: bytes do not match immutable blob
|
||||
metadata and the blob is quarantined;
|
||||
- `orphan_object`: an object exists in the tenant Files prefix without a
|
||||
corresponding blob row.
|
||||
|
||||
Missing or corrupt blobs fail closed for ordinary downloads and Campaign
|
||||
attachment materialization. After restoring the expected bytes, use the finding
|
||||
`recheck` action with its current `expected_revision`. Orphan cleanup starts
|
||||
with a dry-run preview and requires separate destructive confirmation. The
|
||||
confirmation reuses the finding revision from that preview, rechecks that no
|
||||
database reference exists, remains scoped to the scanned tenant prefix, and is
|
||||
idempotent. Both applied and dry-run actions emit audit evidence. A shared
|
||||
reference blocks orphan cleanup. Managed-asset purge separately enforces asset
|
||||
retention, legal hold, active shares, and retained Campaign/Form evidence before
|
||||
releasing a blob for fenced garbage collection.
|
||||
|
||||
### Recovery ledger for object effects
|
||||
|
||||
On PostgreSQL, every managed blob creation or integrity repair starts a Core
|
||||
recovery operation in an independent committed transaction before Files
|
||||
protects or writes bytes. The operation records tenant/blob identifiers, an opaque object
|
||||
locator or locator digest, semantic SHA-256/size evidence, the recovery mode,
|
||||
and a distributed lease fence. It never records file contents, ZIP passwords,
|
||||
connector credentials, or a newly uploaded filename. New object keys are opaque;
|
||||
legacy filename-bearing keys remain readable but repair operations record only
|
||||
their digest and recover through the blob ID.
|
||||
|
||||
SQLite is a supported local-development database but permits only one writer.
|
||||
Files therefore uses an explicit reduced-durability mode there: recovery intent
|
||||
and its lease are written in the caller transaction, while a process-local
|
||||
fence prevents competing effects in the same runtime. Commit makes the intent
|
||||
durable before independent verification. A handled rollback reconstructs a
|
||||
durable recovery operation and verifies compensation or forward completion.
|
||||
A hard process loss before commit can leave an object without a surviving
|
||||
ledger row, so SQLite is not a production recovery profile; after such a loss,
|
||||
run a complete Files integrity scan and reconcile every reported orphan before
|
||||
resuming writes. PostgreSQL retains the independent pre-effect durability
|
||||
guarantee.
|
||||
|
||||
The Files business transaction then creates or updates the blob, version, and
|
||||
asset rows. Its actual SQLAlchemy commit or rollback settles every pending
|
||||
operation:
|
||||
|
||||
- commit reloads the blob through an independent session and streams the object
|
||||
to verify its stored-byte SHA-256 and size before recording success;
|
||||
- rollback deletes only a newly reserved object after independently proving
|
||||
that no `FileBlob` references it, then records verified compensation;
|
||||
- a repaired existing object is forward-completed only when its identity,
|
||||
envelope, semantic evidence, and stored bytes all match;
|
||||
- missing or mismatched bytes quarantine a committed blob and leave the
|
||||
operation `recovery_required`; an unavailable probe remains
|
||||
`outcome_unknown` rather than becoming an ordinary upload failure.
|
||||
|
||||
Applied orphan cleanup has its own forward-recovery operation. The database
|
||||
reference check and tenant-prefix check happen before deletion; object absence
|
||||
and the durable finding state are verified afterward. If the caller transaction
|
||||
rolls back after deletion, Files may forward-complete only that existing
|
||||
finding after rechecking that the key is still unreferenced.
|
||||
|
||||
Hard purge uses an irreversible Core recovery plan with an approval reference
|
||||
and a tenant-wide purge lease. The preview hash binds target IDs, deletion
|
||||
state, lifecycle revisions, holds, deadlines, blockers, and blob IDs. Asset and
|
||||
version removal plus audit evidence commit together. Released objects are not
|
||||
deleted by that transaction. Blob GC later takes `files:blob:<tenant>:<blob>`,
|
||||
the same distributed resource fence as upload/repair, locks and rechecks the
|
||||
blob, and verifies object absence before removing metadata. Provider or database
|
||||
ambiguity remains recovery-required or outcome-unknown in Ops.
|
||||
|
||||
Explicit S3 connector writes use forward recovery because the provider cannot
|
||||
join the database transaction. The durable request contains only tenant,
|
||||
profile, opaque target digest, content SHA-256/size, and expected revision. A
|
||||
conditional create or overwrite is followed by a metadata probe for the exact
|
||||
content and operation markers. Never retry an unresolved target with another
|
||||
request key; reconcile the owning Files operation from provider evidence first.
|
||||
|
||||
Archive preview and confirmation use bounded process-local temporary staging.
|
||||
Staging is not authoritative and is removed on every handled exit; extracted
|
||||
members enter the same per-blob recovery boundary as direct uploads. A hard
|
||||
process loss may leave a temporary OS file for normal host temporary-file
|
||||
cleanup, but cannot make that staging path a managed Files object.
|
||||
|
||||
Use the Ops recovery-operation view to inspect `files` operations. Do not retry
|
||||
a busy or unresolved blob or connector path blindly: first verify the FileBlob
|
||||
row and object hash, or the remote request/content markers and revision, plus
|
||||
any Encryption envelope named by a managed blob.
|
||||
|
||||
After restore:
|
||||
|
||||
@@ -493,9 +751,10 @@ Legacy external secret references are detached and identified as non-owned;
|
||||
Files never calls a provider delete operation for them.
|
||||
|
||||
The retirement executor drops database tables but does not delete corresponding
|
||||
objects from the configured blob backend. Operators must include those orphaned
|
||||
objects in the approved retention/destruction plan. Validate the installer
|
||||
snapshot and independent blob backup before retirement.
|
||||
objects from the configured blob backend. Operators must include those objects
|
||||
in the approved retention/destruction plan, report them with an integrity scan,
|
||||
and explicitly approve cleanup. Validate the installer snapshot and independent
|
||||
blob backup before retirement.
|
||||
|
||||
### Operational signals
|
||||
|
||||
@@ -506,7 +765,7 @@ Use these symptoms as routing hints:
|
||||
| `Stored object does not exist` | Database/blob restore mismatch, wrong root, or missing shared storage |
|
||||
| `Stored secret cannot be decrypted` | Wrong or rotated `MASTER_KEY_B64` |
|
||||
| Private/non-public endpoint blocked | Deployment-wide egress policy is public-only or DNS returned a forbidden answer |
|
||||
| SDK cannot pin redirects/referrals | Expected fail-closed S3/SMB boundary, not a transient connector outage |
|
||||
| SDK peer-pinning seam is unavailable | Optional S3/SMB SDK is incompatible; keep access fail-closed and validate the supported dependency range before upgrade |
|
||||
| Connector response exceeds limit | Remote payload exceeds connector or upload limit |
|
||||
| Profile is not visible | Scope, disabled state, campaign access, or policy mismatch |
|
||||
| Group removal is vetoed | The group still owns a file/folder/connector space or is a share target |
|
||||
@@ -522,11 +781,26 @@ public HTTP API. They must not import Files ORM models or storage helpers.
|
||||
| --- | --- |
|
||||
| `files.access` (`0.1.6`) | Explain resource access provenance for managed files, explicit folders, and virtual folders |
|
||||
| `files.campaign_attachments` (`0.1.6`) | Resolve managed attachment matches, prepare frozen campaign snapshots, annotate built messages, share assets with a campaign, and record/mark exact attachment use |
|
||||
| `records.source.files` (`1.0.0`) | Recheck current Files access and resolve one exact, integrity-approved managed file version for Records filing |
|
||||
| `forms_runtime.evidence.files` (`1.0.0`) | Issue a one-time managed attachment grant and re-verify the exact file/version/checksum at Form submission |
|
||||
|
||||
Files requires Core principal resolution and permission evaluation. Campaign is
|
||||
an optional dependency; when installed, Files consumes the optional
|
||||
`campaigns.access` interface to verify campaign existence and access. Missing
|
||||
optional Campaign support fails explicitly rather than bypassing the check.
|
||||
Records is also optional. When enabled, the source capability returns the
|
||||
requested `FileVersion` identity, path snapshot, content metadata, SHA-256,
|
||||
integrity/protection state, and launch link. It rejects mutable aliases,
|
||||
cross-tenant requests, missing access, and quarantined or failed blobs. Records
|
||||
stores the filing decision; Files continues to own the version and bytes.
|
||||
Forms Runtime is optional as well. Its public or authenticated participant never
|
||||
receives general Files access through this integration. Forms Runtime asks for a
|
||||
purpose-bound grant, Files verifies an active same-tenant user custodian, stores
|
||||
only the token digest, accepts one bounded upload, and returns an immutable
|
||||
`EvidenceReference`. Draft save and final submit independently recheck the exact
|
||||
Form instance/revision, grant, asset, version, checksum, deletion state, and
|
||||
integrity state. An idempotent replay returns the existing grant without
|
||||
reproducing its bearer secret.
|
||||
|
||||
### API families
|
||||
|
||||
@@ -535,17 +809,32 @@ All routes below are under `/api/v1/files`.
|
||||
| Area | Routes |
|
||||
| --- | --- |
|
||||
| Spaces and content | `GET /spaces`, `GET /`, `GET /folders`, `GET /delta` |
|
||||
| Upload and folders | `POST /upload`, `POST /upload-zip`, `POST /folders`, `POST /folders/delete` |
|
||||
| Upload and folders | `POST /upload`, `POST /upload-zip` (compatibility), `POST /archive-preview`, `POST /archive-confirm`, `POST /folders`, `POST /folders/delete` |
|
||||
| File access | `GET /{file_id}`, `GET /{file_id}/download`, `DELETE /{file_id}`, `POST /bulk-delete` |
|
||||
| Organization | `POST /bulk-rename`, `POST /transfer`, `POST /archive.zip`, `POST /resolve-patterns` |
|
||||
| Sharing | `POST /{file_id}/shares`, `POST /bulk-shares` |
|
||||
| Connector spaces | `GET/POST /connector-spaces`, `PATCH/DELETE /connector-spaces/{space_id}` |
|
||||
| Connector spaces | `GET/POST /connector-spaces`, `PATCH/DELETE /connector-spaces/{space_id}`, `POST /connector-spaces/{space_id}/restore`, `POST /connector-spaces/{space_id}/write-back` |
|
||||
| Connector catalog/discovery | `GET /connectors/providers`, `POST /connectors/discover` |
|
||||
| Connector profiles | `GET/POST /connectors/profiles`, `GET/PATCH/DELETE /connectors/profiles/{profile_id}` |
|
||||
| Browse/import/sync | `GET /connectors/profiles/{profile_id}/browse`, `POST /connectors/profiles/{profile_id}/import`, `POST /connectors/profiles/{profile_id}/sync` |
|
||||
| Restore and lifecycle | `POST /assets/{file_id}/restore`, `POST /folders/restore`, `PATCH /{file_id}/lifecycle` |
|
||||
| Governed erasure | `POST /purge/preview`, `POST /purge/execute`, `POST /purge/blobs` |
|
||||
| Credentials | `GET/POST /connectors/credentials`, `GET/PATCH/DELETE /connectors/credentials/{credential_id}` |
|
||||
| Policy | `GET/PUT /connectors/policies/{scope_type}`, `POST /connector-policy/evaluate` |
|
||||
| Incremental connector settings | `GET /connectors/settings/delta` |
|
||||
| Form evidence | `POST /form-evidence/upload` with a short-lived `X-Form-Evidence-Token` issued by Forms Runtime |
|
||||
|
||||
The Files workspace exposes **Remove space** only for connector spaces and only
|
||||
to actors with file-organization authority over the owning user
|
||||
or group space. Confirmation explains the exact boundary: removal soft-deletes
|
||||
the local connector-space definition and makes that virtual view disappear. It
|
||||
does not mutate or delete remote provider content, previously imported managed
|
||||
files, their metadata or shares, the connector profile, credentials, or remote
|
||||
object references. Those retained objects therefore do not block removal and
|
||||
the connector location can be linked again later. User and group managed spaces
|
||||
are intrinsic ownership scopes rather than removable records, so they never
|
||||
offer this action. A missing, already removed, cross-tenant, or inaccessible
|
||||
connector space fails through the backend lookup and owner-access checks.
|
||||
|
||||
Consumers should use cursor/watermark contracts instead of assuming an
|
||||
unbounded complete list. The default full-list page size is 500 and public page
|
||||
@@ -580,21 +869,32 @@ Files baseline indiscriminately.
|
||||
- Owner and share checks protect individual resources after operation-scope
|
||||
checks.
|
||||
- Logical paths reject traversal and storage keys cannot escape the local root.
|
||||
- Upload, ZIP extraction, connector response, and S3 stream code use bounded
|
||||
reads; encrypted ZIP archives are rejected.
|
||||
- Upload, archive extraction, connector response, and S3 stream code use bounded
|
||||
reads. Archive previews are sealed and short-lived; ZIP passwords are
|
||||
request-only.
|
||||
- Public Form evidence uploads require a custom-header bearer grant that is
|
||||
stored only as SHA-256, expires after at most 15 minutes, is bound to one
|
||||
exact submission and user custodian, and can create only one managed file.
|
||||
- Connector HTTP sockets use connection-time DNS/IP validation and pinning,
|
||||
redirects are refused, and unsafe SDK transports fail before client creation.
|
||||
- Database-managed connector passwords/tokens are encrypted; responses redact
|
||||
secrets and deployment references.
|
||||
- API metadata is recursively checked for secret-like values.
|
||||
- Downloads use sanitized attachment filenames.
|
||||
- SHA-256 and byte size are recorded for every blob/version.
|
||||
- Plaintext semantic and stored-byte SHA-256/size evidence are recorded for
|
||||
every protected blob; they are identical for unprotected blobs.
|
||||
- Upload and archive-confirm APIs can select an Encryption vault. Protected
|
||||
writes and reads fail closed if the optional Encryption capability is absent.
|
||||
- Managed-object writes and applied orphan cleanup start lease-fenced Core
|
||||
recovery operations before their physical effects; terminal success and
|
||||
compensation require independent database and object checks.
|
||||
|
||||
The module does **not** currently provide malware scanning, content disarm and
|
||||
reconstruction, a file-type allowlist, per-user quota, at-rest encryption for
|
||||
local blob bytes, or a dedicated preview sandbox. Deployments that require
|
||||
these controls must supply them outside Files until explicit module contracts
|
||||
exist.
|
||||
reconstruction, a file-type allowlist, per-user quota, automatic encryption
|
||||
policy assignment, client E2EE, or a dedicated preview sandbox. The optional
|
||||
server-envelope profile protects selected managed blob bytes at rest but remains
|
||||
server-decryptable. Deployments that require the other controls must supply
|
||||
them outside Files until explicit module contracts exist.
|
||||
|
||||
### Provenance
|
||||
|
||||
@@ -647,6 +947,9 @@ The word "delete" has different meanings by object type:
|
||||
| Connector profile | Immediately disables the tombstone and clears credential links/material/references and private metadata |
|
||||
| Legacy `secret_ref` | Detached and audited as an unowned external reference; no provider deletion is attempted or claimed |
|
||||
| Module retirement | Scrubs/audits credential material, then drops Files database tables; blob-backend cleanup is an operator responsibility |
|
||||
| Governed hard purge | Removes only preview-matched, soft-deleted asset/version rows without active retention, legal hold, active shares, Campaign evidence, or Form evidence |
|
||||
| Blob garbage collection | Deletes an exact managed object only after a fresh zero-reference check under the shared blob fence; metadata follows verified object absence |
|
||||
| Connector write-back | Explicit conditional S3 create/overwrite only; automatic provider delete, rename, move, and ACL propagation remain disabled |
|
||||
|
||||
Credential/profile scrubbing and its audit event use the same database
|
||||
transaction. If audit creation fails, the deletion rolls back. Repeating a
|
||||
@@ -654,15 +957,19 @@ delete against an already scrubbed tombstone does not recreate secret evidence.
|
||||
|
||||
### Retention boundary
|
||||
|
||||
File versions and blobs are effectively retained indefinitely today. Although a
|
||||
blob has `ref_count` and `retained_until` fields, no complete retention-policy,
|
||||
legal-hold, hard-purge, or garbage-collection service enforces them. There is
|
||||
also no supported user restore endpoint for soft-deleted assets/folders and no
|
||||
automated orphan-object reconciliation.
|
||||
Each asset has an enforceable retained-until value, legal-hold flag, reason, and
|
||||
optimistic lifecycle revision. These controls block hard purge; they do not
|
||||
automatically schedule it. Restore, purge preview/execute, and blob GC are
|
||||
explicit authorized operations. Blob `retained_until` remains an additional
|
||||
storage-level safeguard and must also have expired before automated collection
|
||||
is introduced. Campaign and Form evidence are hard blockers rather than
|
||||
silently cascaded references.
|
||||
|
||||
Do not promise erasure, timed retention, legal hold, or self-service recovery
|
||||
from the current soft-delete behavior. Those require an explicit, auditable
|
||||
retention/purge design that preserves campaign and other evidence references.
|
||||
Do not equate soft deletion with erasure. Erasure is complete only after the
|
||||
approved purge removes asset/version metadata, bounded GC verifies that no
|
||||
version references the blob and removes its bytes/metadata, and any owning
|
||||
Encryption retention/key-custody consequence has been handled under that
|
||||
module's policy.
|
||||
|
||||
## Acceptance scenarios
|
||||
|
||||
@@ -675,12 +982,14 @@ her personal or Finance space, then the asset has the selected owner and tenant.
|
||||
Given Bob is outside Finance and has no share or Files admin permission, the
|
||||
same asset ID must not make the asset readable to Bob.
|
||||
|
||||
### Safe ZIP import
|
||||
### Safe archive import
|
||||
|
||||
Given a ZIP member contains `../../secret.txt`, is encrypted, exceeds 50 MiB,
|
||||
or pushes actual extracted bytes above 250 MiB, unpacking must fail without a
|
||||
committed managed asset. A valid archive must preserve normalized relative paths
|
||||
below the chosen logical folder.
|
||||
Given an archive member contains `../../secret.txt`, is a link or special
|
||||
filesystem object, exceeds 50 MiB, pushes actual expanded bytes above 2 GiB, or
|
||||
exceeds the 100:1 expansion ratio, preview or confirmation must fail without a
|
||||
committed managed asset. A password-protected ZIP requires the correct
|
||||
request-only password. A valid confirmation must match its unexpired preview
|
||||
token and preserve normalized relative paths below the chosen logical folder.
|
||||
|
||||
### Explicit conflicts
|
||||
|
||||
@@ -699,9 +1008,11 @@ bypass the same rule.
|
||||
### Pinned connector transport
|
||||
|
||||
Given public-only mode and DNS returns any private address, the connection must
|
||||
be rejected before a socket opens. Given private mode, the HTTP connection must
|
||||
still use the validated address and reject redirects. SMB and S3 must fail
|
||||
before their SDK clients connect until all SDK-managed peers can be pinned.
|
||||
be rejected before a socket opens. Given private mode, every HTTP, S3, and SMB
|
||||
connection must still use an address from the answer validated for that exact
|
||||
attempt. Botocore retries, redirects, endpoint discovery, and aliases, plus SMB
|
||||
reconnects and DFS referrals, must pass through the pinned factories. A changed
|
||||
or unsupported SDK seam must fail before a usable client/session is returned.
|
||||
|
||||
### Imported evidence and sync
|
||||
|
||||
@@ -735,20 +1046,22 @@ returning different content or credentials.
|
||||
|
||||
| Area | Implemented now | Planned or explicitly outside the current boundary |
|
||||
| --- | --- | --- |
|
||||
| Managed storage | Local durable-root backend, fallback read roots, tenant blob deduplication, checksums | Operational S3 after pinned SDK transport ([#34](https://git.add-ideas.de/add-ideas/govoplan-files/issues/34)); automated integrity/orphan reconciliation ([#36](https://git.add-ideas.de/add-ideas/govoplan-files/issues/36)) |
|
||||
| Upload | Bounded direct upload, drag-and-drop UI, ZIP spool/extract limits, explicit conflicts | Malware scanning, quotas, type policy, resumable/chunked upload |
|
||||
| Organization | Folders, bulk rename preview/apply, move/copy, drag-and-drop, ZIP download, pattern resolution | General file-history UI and user-driven append-version/restore |
|
||||
| Sharing | User/group/tenant/campaign grant or permission update through API; campaign linkage display | Share revocation/expiry and complete general share-management UI ([#37](https://git.add-ideas.de/add-ideas/govoplan-files/issues/37)) |
|
||||
| Deletion/retention | Soft-delete assets/folders/spaces; immediate audited connector-secret scrubbing | File restore API, hard purge, retention policy, legal hold, and blob GC ([#38](https://git.add-ideas.de/add-ideas/govoplan-files/issues/38)) |
|
||||
| Managed storage | Core local/S3 backend, exact managed-Garage or explicitly trusted HTTPS external S3, state-profile validation, fallback local read roots, tenant blob deduplication, checksums, bounded resumable integrity scans, quarantine, dry-run-first orphan cleanup, and Core-ledger verification/forward recovery | Scheduled scan execution and deployment-specific S3 HA/backup automation |
|
||||
| Upload | Bounded direct upload, drag-and-drop UI, archive preview/selective extraction, password-protected ZIP support, explicit conflicts, opaque new object keys, and rollback compensation | Malware scanning, quotas, type policy, resumable/chunked upload |
|
||||
| Organization | Folders, bulk rename preview/apply, move/copy, drag-and-drop, ZIP download, pattern resolution, and API restoration preserving versions/provenance | General file-history UI and user-driven append-version UI |
|
||||
| Sharing | User/group/tenant/campaign grants, expiry, idempotent revocation, searchable share-management UI, and campaign linkage display | Richer policy-driven share lifecycles |
|
||||
| Deletion/retention | Soft-delete and restore assets/folders/spaces; optimistic retention and legal-hold controls; preview-bound, approval-referenced hard purge; reference-checked blob GC; immediate audited connector-secret scrubbing | Automatic time-based purge scheduling and richer lifecycle administration UI |
|
||||
| Privacy requests | Tenant-scoped bounded DSAR metadata search; retained/manual/revoke/detach planning; idempotent share revocation and mutable actor-reference detachment; explicit separation from byte purge | Content-specific automated redaction and policy-specific approval remain manual or belong to the owning process |
|
||||
| Connector governance | Scoped profiles/credentials/policies, effective source explanation, separate credentials, linked user/group spaces | Provider-owned external secret lifecycle; API `secret_ref` remains rejected |
|
||||
| HTTP connectors | Pinned, bounded, no-redirect Seafile and WebDAV/Nextcloud browse/import/manual sync | Background/folder sync, remote mutation, long-running transfer workers |
|
||||
| SMB and S3 connectors | Provider descriptors and browse/import logic | Live access only after SDK connections plus DFS referrals/redirects are validated and pinned ([SMB #35](https://git.add-ideas.de/add-ideas/govoplan-files/issues/35), [S3 #34](https://git.add-ideas.de/add-ideas/govoplan-files/issues/34)) |
|
||||
| SMB and S3 connectors | Provider descriptors, browse/import/manual sync, pinned SDK transports, redirect/retry/referral transport-contract tests, and explicit conditional S3 write-back with Core-ledger recovery | Live topology smoke evidence, provider-specific OAuth, additional provider writes, and background indexing remain separate deployment or connector-module concerns |
|
||||
| Other providers | Reserved SharePoint/OneDrive keys and NFS/local descriptors | Graph/OAuth/provider paging, NFS deployment integration, DMS connectors |
|
||||
| Connector spaces | User/group link, browse, manual selected-file sync, edit/disable/delete | Background sync, remote writes/deletes, full conflict-reporting jobs |
|
||||
| Profile capabilities | Stored and displayed | Enforce capability flags as an independent operation gate |
|
||||
| Connector spaces | User/group link, browse, manual selected-file sync, edit/disable/delete/restore, read-only default, and opt-in S3 two-way mode | Background sync and automatic remote delete/rename/move/ACL propagation |
|
||||
| Profile capabilities | Stored, displayed, and enforced for explicit connector write-back | Broader provider-specific capability negotiation |
|
||||
| Audit | Connector discovery/import/sync/access and connector deletion; campaign exact-use evidence | Dedicated canonical audit events for every ordinary Files mutation |
|
||||
| Preview | File metadata and attachment download | Dedicated safe content-preview service |
|
||||
| Campaign | Stable capability-based frozen attachments and sent-use evidence | Campaign-specific process state remains in Campaign |
|
||||
| Forms Runtime | One-time managed attachment grants plus exact-version, checksum, deletion, tenant, submission, and integrity verification | Malware scanning and advanced/qualified signature providers remain separate assurance depth |
|
||||
| Collaboration | Governed input/output snapshots | Co-editing, comments, review, presence, locks, and semantic document versions belong to Documents/workflow/provider modules |
|
||||
|
||||
## Release and change checklist
|
||||
@@ -764,11 +1077,20 @@ Before releasing Files:
|
||||
5. Exercise an allowed and denied owner/share path.
|
||||
6. Exercise upload, ZIP bounds, conflict handling, download, and soft deletion.
|
||||
7. Exercise connector policy explanation and one pinned HTTP provider where
|
||||
configured; verify S3/SMB still fail closed.
|
||||
configured; verify the deployment-managed or trusted external S3 backend if
|
||||
selected, and verify one configured S3 and SMB connector while recording the
|
||||
actual target topology and private-network policy.
|
||||
8. Verify credential deletion scrubs dependents and produces audit evidence.
|
||||
9. Verify a campaign attachment snapshot still identifies its exact version and
|
||||
checksum after the current file changes.
|
||||
10. Update the implemented/planned table whenever a boundary changes.
|
||||
10. Exercise a committed upload, a rolled-back upload, object tamper detection,
|
||||
and applied orphan cleanup; inspect their `files` operations in Ops.
|
||||
11. Exercise a Form evidence grant, token replay, wrong-submission reference,
|
||||
expired token, unsupported media type, and quarantined-file rejection.
|
||||
12. Exercise a Files DSAR search/plan, repeat an approved reversible action, and
|
||||
confirm retained file content can only be erased through the separate purge
|
||||
authority and recovery path.
|
||||
13. Update the implemented/planned table whenever a boundary changes.
|
||||
|
||||
## Related documents
|
||||
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
# Generated Artifact Store
|
||||
|
||||
Files implements Core's optional `files.artifact_store` capability for modules
|
||||
that generate deterministic output without owning file storage.
|
||||
|
||||
The producer supplies bytes, filename, content type, destination folder,
|
||||
optional idempotency key, and bounded non-secret provenance. Files applies the
|
||||
actor's `files:file:upload` permission, tenant/user ownership, path rules,
|
||||
versioning, configured blob backend, and conflict behavior. An idempotency key
|
||||
is represented as source provenance so an unchanged retry does not create an
|
||||
unrelated file version.
|
||||
|
||||
The shared Files session owns finalization. Before a new managed object is
|
||||
written, Files commits a lease-fenced Core recovery operation containing only
|
||||
identifiers and digests. The caller's eventual session commit independently
|
||||
verifies both `FileBlob` metadata and stored bytes; rollback compensates only an
|
||||
unreferenced key. Producers must therefore complete the supplied transaction
|
||||
normally and must not bypass or replace Files session lifecycle handling.
|
||||
|
||||
The response contains only file/version identifiers, display path, media type,
|
||||
size, digest, and storage provenance. Producers must not put credentials,
|
||||
tokens, or rendered plaintext into metadata. Storing an artifact proves Files
|
||||
accepted it; it does not prove printing, mailing, or any other external effect.
|
||||
@@ -0,0 +1,55 @@
|
||||
# Files Interface Pattern Migration
|
||||
|
||||
This inventory records the Files-owned part of the GovOPlaN interface pattern
|
||||
language. Core owns the shell and shared components; Files owns the composition
|
||||
and consequences described here.
|
||||
|
||||
## Surface inventory
|
||||
|
||||
| Surface | Primary task | Archetype | Consequence | Pattern evidence |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| `/files` space and folder panes | Browse managed and connected content without losing location | Directory/explorer | Low for navigation; medium for exposing filenames and provenance | Full-height two-pane workspace, bounded panes, stable selection and contextual Help Center link |
|
||||
| `/files` toolbar and property filters | Find and act on the current selection | Explorer actions and local filtering | Medium for upload, move, copy, share and synchronization; high for delete | Actions remain beside the affected list, disabled controls explain permission/state/selection blockers, destructive work uses `ConfirmDialog` |
|
||||
| Upload/archive, transfer, rename and connector-import dialogs | Supply and review one bounded change | Adaptive create/edit or guided import | Medium to high because files, paths and external bytes change | Shared `Dialog`, `FileDropZone`, validation, conflict review, unsaved inputs and explicit confirmation |
|
||||
| File share dialog | Inspect and change access | Review/decision | High because another actor gains access | Shared dialog, access explanation, stable row actions and destructive confirmation |
|
||||
| System/tenant/group/user connector surfaces | Compare connections, credentials and effective policy | Administration/configuration | High because endpoints, secrets and inherited policy control external access | Shared `ConnectionTree`, adaptive forms, `ActionBlockerHint`, policy provenance and contextual admin help |
|
||||
| Connection and credential editors | Create or edit one governed endpoint or secret | Adaptive create/edit | High because a saved change may enable remote access | Relevant fields only, typed credential controls, discovery/test, advanced compatibility section, unsaved-change guard and disabled-save reasons |
|
||||
| Connector policy card | Narrow inherited connector access | Effective-policy editor | High because deny/allow changes affect lower scopes | Typed reference selectors, deny precedence warning, effective source evidence and permission blocker |
|
||||
| `files.widget.spaces` | See available managed/connected spaces and open Files | Dashboard widget | Low; names and provider state may still be sensitive | Shared loading/alert/status components, bounded item count, permission-filtered contribution |
|
||||
| Files chooser capability used by another module | Select a managed snapshot without importing Files internals | Directory chooser | Medium because the exact selected version becomes another module's input | Shared dialog/confirmation, capability boundary and exact file/version evidence |
|
||||
|
||||
## State and consequence contract
|
||||
|
||||
- Loading, errors, success, empty results, access explanations and confirmation
|
||||
use Core components. Files does not reproduce the application shell.
|
||||
- A connector that comes from deployment settings remains visible but read-only;
|
||||
its action explains that bootstrap configuration and a restart are required.
|
||||
- Missing permission, target, selection, endpoint, or compatible provider is an
|
||||
explained disabled state. It is not represented only by color or absence.
|
||||
- Provider metadata JSON is an expert compatibility escape hatch inside the
|
||||
collapsed shared advanced-options component. Ordinary connector setup uses
|
||||
typed provider, endpoint, credential, capability and policy controls.
|
||||
- Endpoint discovery and credential tests are explicit and report their result;
|
||||
they do not save the draft. Save remains the only committing action.
|
||||
- Connector/profile disable and managed-file delete remain confirmed actions and
|
||||
state their immediate effect. Soft deletion must not be described as purge.
|
||||
- External connector data, paths and credential references are rendered only in
|
||||
already-authorized administration or explorer contexts. Secret values are
|
||||
never returned for rendering.
|
||||
|
||||
## Accessibility and responsive evidence
|
||||
|
||||
Shared `Dialog` owns focus entry, Escape handling and focus return. Form and
|
||||
toolbar DOM order is the keyboard order; disabled-action tooltips are themselves
|
||||
focusable and expose the reason. The connector form uses semantic sections and
|
||||
labels, status is textual as well as colored, and result alerts are announced by
|
||||
the shared alert component. The explorer collapses to one column below 1050 px;
|
||||
connector forms and action rows collapse below 760 px while preserving source
|
||||
order. Long provider choices scroll inside the segmented control rather than
|
||||
expanding the page.
|
||||
|
||||
The focused structural test guards these contracts, optional-module boundaries,
|
||||
confirmation, contextual help, advanced-only JSON and responsive rules. Core's
|
||||
TypeScript build, structural localization audit, module-permutation suite and
|
||||
full-product bundle check provide the integration gates.
|
||||
|
||||
+8
-8
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@govoplan/files-webui",
|
||||
"version": "0.1.9",
|
||||
"version": "0.1.19",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "webui/src/index.ts",
|
||||
@@ -19,14 +19,14 @@
|
||||
"LICENSE"
|
||||
],
|
||||
"peerDependencies": {
|
||||
"@govoplan/core-webui": "^0.1.9",
|
||||
"lucide-react": "^1.23.0",
|
||||
"react": "^19.0.0",
|
||||
"react-dom": "^19.0.0",
|
||||
"react-router-dom": "^7.1.1",
|
||||
"@vitejs/plugin-react": "^4.3.4",
|
||||
"@vitejs/plugin-react": "^5.2.0",
|
||||
"vite": "^7.3.6",
|
||||
"typescript": "^5.7.2",
|
||||
"vite": "^6.0.6"
|
||||
"react": ">=19.2.7 <20",
|
||||
"react-dom": ">=19.2.7 <20",
|
||||
"react-router": ">=8.3.0 <9",
|
||||
"lucide-react": "^1.23.0",
|
||||
"@govoplan/core-webui": "^0.1.18"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"@govoplan/core-webui": {
|
||||
|
||||
+3
-2
@@ -4,15 +4,16 @@ build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "govoplan-files"
|
||||
version = "0.1.9"
|
||||
version = "0.1.19"
|
||||
description = "GovOPlaN files module with backend and WebUI integration."
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.12"
|
||||
license = { file = "LICENSE" }
|
||||
authors = [{ name = "GovOPlaN" }]
|
||||
dependencies = [
|
||||
"govoplan-core>=0.1.9",
|
||||
"govoplan-core>=0.1.19",
|
||||
"defusedxml>=0.7,<1",
|
||||
"pyzipper>=0.3.6,<1",
|
||||
"python-multipart>=0.0.31,<1",
|
||||
]
|
||||
|
||||
|
||||
@@ -5,13 +5,32 @@ import binascii
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import or_
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from govoplan_core.audit.logging import audit_from_principal
|
||||
from govoplan_core.auth import ApiPrincipal, has_scope
|
||||
from govoplan_core.core.access import AccessDecisionProvenance, PrincipalRef
|
||||
from govoplan_core.core.files import FileAccessProvider
|
||||
from govoplan_core.core.files import (
|
||||
ManagedTabularFile,
|
||||
ManagedTabularFileAccessError,
|
||||
ManagedTabularFileContent,
|
||||
ManagedTabularFileNotFoundError,
|
||||
ManagedTabularFileProvider,
|
||||
ManagedTabularFileUnavailableError,
|
||||
ManagedTabularFileValidationError,
|
||||
PostboxFileReferenceRef,
|
||||
PostboxFileReferenceRequest,
|
||||
PostboxFileReferenceProvider,
|
||||
ManagedArtifactRef,
|
||||
ManagedArtifactStore,
|
||||
ManagedArtifactWriteRequest,
|
||||
)
|
||||
from govoplan_core.core.modules import ModuleContext
|
||||
from govoplan_core.security.module_permissions import scopes_grant_compatible
|
||||
from govoplan_files.backend.db.models import FileAsset, FileFolder, FileShare
|
||||
from govoplan_files.backend.db.models import FileAsset, FileBlob, FileFolder, FileShare, FileVersion
|
||||
from govoplan_files.backend.runtime import configure_runtime
|
||||
from govoplan_files.backend.storage.common import FileStorageError
|
||||
from govoplan_files.backend.storage.campaign_attachments import (
|
||||
annotate_built_messages_with_managed_files,
|
||||
managed_match_payloads,
|
||||
@@ -20,8 +39,16 @@ from govoplan_files.backend.storage.campaign_attachments import (
|
||||
share_assets_with_campaign,
|
||||
)
|
||||
from govoplan_files.backend.storage.campaign_usage import record_campaign_attachment_uses_for_jobs
|
||||
from govoplan_files.backend.storage.files import current_version_and_blob
|
||||
from govoplan_files.backend.storage.files import (
|
||||
create_file_asset,
|
||||
current_version_and_blob,
|
||||
get_asset_for_user,
|
||||
list_recent_assets_for_user,
|
||||
read_asset_version_bytes,
|
||||
sync_file_asset_from_source,
|
||||
)
|
||||
from govoplan_files.backend.storage.paths import normalize_folder
|
||||
from govoplan_files.backend.storage.share_state import effective_file_share_clause
|
||||
|
||||
|
||||
VIRTUAL_FOLDER_RESOURCE_PREFIX = "virtual-folder:v1"
|
||||
@@ -59,6 +86,214 @@ def campaign_capability(context: ModuleContext) -> FilesCampaignCapability:
|
||||
return FilesCampaignCapability()
|
||||
|
||||
|
||||
class FilesArtifactStore(ManagedArtifactStore):
|
||||
def store_artifact(
|
||||
self,
|
||||
session: object,
|
||||
principal: object,
|
||||
*,
|
||||
request: ManagedArtifactWriteRequest,
|
||||
) -> ManagedArtifactRef:
|
||||
if not hasattr(session, "query") or not hasattr(session, "flush"):
|
||||
raise TypeError("Files artifact storage requires a SQLAlchemy session.")
|
||||
if not hasattr(principal, "has") or not principal.has("files:file:upload"):
|
||||
raise PermissionError("Managed artifact storage requires files:file:upload.")
|
||||
user = getattr(principal, "user", None)
|
||||
user_id = str(getattr(user, "id", "") or "")
|
||||
tenant_id = str(getattr(principal, "tenant_id", "") or "")
|
||||
if not user_id or not tenant_id:
|
||||
raise PermissionError("Managed artifact storage requires a tenant user principal.")
|
||||
metadata = dict(request.metadata)
|
||||
if request.idempotency_key:
|
||||
metadata["source_provenance"] = {
|
||||
"source_type": "generated_artifact",
|
||||
"connector_id": "files.artifact_store",
|
||||
"provider": str(metadata.get("producer_module") or "platform"),
|
||||
"external_id": request.idempotency_key,
|
||||
"revision": str(metadata.get("output_sha256") or "") or None,
|
||||
}
|
||||
stored, _action, _previous_version_id = sync_file_asset_from_source(
|
||||
session,
|
||||
tenant_id=tenant_id,
|
||||
owner_type="user",
|
||||
owner_id=user_id,
|
||||
user_id=user_id,
|
||||
filename=request.filename,
|
||||
data=request.payload,
|
||||
metadata=metadata,
|
||||
folder=request.folder,
|
||||
content_type=request.content_type,
|
||||
conflict_strategy="rename",
|
||||
is_admin=principal.has("files:file:admin"),
|
||||
)
|
||||
else:
|
||||
stored = create_file_asset(
|
||||
session,
|
||||
tenant_id=tenant_id,
|
||||
owner_type="user",
|
||||
owner_id=user_id,
|
||||
user_id=user_id,
|
||||
filename=request.filename,
|
||||
data=request.payload,
|
||||
folder=request.folder,
|
||||
content_type=request.content_type,
|
||||
description=request.description,
|
||||
metadata=metadata,
|
||||
conflict_strategy="rename",
|
||||
is_admin=principal.has("files:file:admin"),
|
||||
)
|
||||
return ManagedArtifactRef(
|
||||
file_asset_id=stored.asset.id,
|
||||
file_version_id=stored.version.id,
|
||||
filename=stored.version.filename_at_upload,
|
||||
display_path=stored.asset.display_path,
|
||||
content_type=stored.version.content_type or request.content_type,
|
||||
size_bytes=stored.version.size_bytes,
|
||||
sha256=stored.version.checksum_sha256,
|
||||
provenance={
|
||||
"module": "files",
|
||||
"owner_type": stored.asset.owner_type,
|
||||
"managed": True,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def artifact_store_capability(context: ModuleContext) -> FilesArtifactStore:
|
||||
configure_runtime(registry=context.registry, settings=context.settings)
|
||||
return FilesArtifactStore()
|
||||
|
||||
|
||||
class FilesPostboxReferenceService(PostboxFileReferenceProvider):
|
||||
"""Resolve Postbox evidence without turning message access into file access."""
|
||||
|
||||
def resolve_postbox_references(
|
||||
self,
|
||||
session: object,
|
||||
principal: object,
|
||||
*,
|
||||
tenant_id: str,
|
||||
requests: tuple[PostboxFileReferenceRequest, ...],
|
||||
) -> tuple[PostboxFileReferenceRef, ...]:
|
||||
if not hasattr(session, "get"):
|
||||
raise TypeError("Postbox file resolution requires a SQLAlchemy session.")
|
||||
principal_tenant = str(getattr(principal, "tenant_id", "") or "")
|
||||
can_download = bool(
|
||||
hasattr(principal, "has") and principal.has("files:file:download")
|
||||
)
|
||||
user = getattr(principal, "user", None)
|
||||
user_id = str(getattr(user, "id", "") or "")
|
||||
is_admin = bool(
|
||||
hasattr(principal, "has") and principal.has("files:file:admin")
|
||||
)
|
||||
results: list[PostboxFileReferenceRef] = []
|
||||
for request in requests:
|
||||
if principal_tenant != tenant_id:
|
||||
results.append(_unavailable_postbox_reference(request, "tenant_mismatch"))
|
||||
continue
|
||||
if not can_download or not user_id:
|
||||
results.append(
|
||||
_unavailable_postbox_reference(
|
||||
request,
|
||||
"download_permission_missing",
|
||||
)
|
||||
)
|
||||
continue
|
||||
reference_type = request.reference_type.strip().casefold()
|
||||
version: FileVersion | None = None
|
||||
if reference_type in {"file", "file_asset", "files:file"}:
|
||||
results.append(
|
||||
_unavailable_postbox_reference(request, "exact_version_required")
|
||||
)
|
||||
continue
|
||||
elif reference_type in {"file_version", "files:file_version"}:
|
||||
version = session.get(FileVersion, request.reference_id) # type: ignore[attr-defined]
|
||||
asset = (
|
||||
session.get(FileAsset, version.file_asset_id) # type: ignore[attr-defined]
|
||||
if version is not None
|
||||
else None
|
||||
)
|
||||
else:
|
||||
results.append(
|
||||
_unavailable_postbox_reference(
|
||||
request,
|
||||
"unsupported_reference_type",
|
||||
)
|
||||
)
|
||||
continue
|
||||
if (
|
||||
asset is None
|
||||
or version is None
|
||||
or asset.tenant_id != tenant_id
|
||||
or version.tenant_id != tenant_id
|
||||
or asset.deleted_at is not None
|
||||
):
|
||||
results.append(_unavailable_postbox_reference(request, "file_not_found"))
|
||||
continue
|
||||
try:
|
||||
get_asset_for_user(
|
||||
session, # type: ignore[arg-type]
|
||||
tenant_id=tenant_id,
|
||||
user_id=user_id,
|
||||
asset_id=asset.id,
|
||||
is_admin=is_admin,
|
||||
)
|
||||
except FileStorageError:
|
||||
results.append(
|
||||
_unavailable_postbox_reference(request, "file_access_denied")
|
||||
)
|
||||
continue
|
||||
blob = session.get(FileBlob, version.blob_id) # type: ignore[attr-defined]
|
||||
if blob is None or blob.tenant_id != tenant_id:
|
||||
results.append(
|
||||
_unavailable_postbox_reference(request, "file_payload_missing")
|
||||
)
|
||||
continue
|
||||
results.append(
|
||||
PostboxFileReferenceRef(
|
||||
reference_type=request.reference_type,
|
||||
reference_id=request.reference_id,
|
||||
available=True,
|
||||
reason_code="available",
|
||||
file_asset_id=asset.id,
|
||||
file_version_id=version.id,
|
||||
filename=version.filename_at_upload or asset.filename,
|
||||
content_type=version.content_type or blob.content_type,
|
||||
size_bytes=version.size_bytes,
|
||||
sha256=version.checksum_sha256,
|
||||
download_path=(
|
||||
f"/api/v1/files/{asset.id}/versions/{version.id}/download"
|
||||
),
|
||||
provenance={
|
||||
"module": "files",
|
||||
"postbox_id": request.postbox_id,
|
||||
"message_id": request.message_id,
|
||||
"display_path": asset.display_path,
|
||||
"exact_version": True,
|
||||
},
|
||||
)
|
||||
)
|
||||
return tuple(results)
|
||||
|
||||
|
||||
def _unavailable_postbox_reference(
|
||||
request: PostboxFileReferenceRequest,
|
||||
reason_code: str,
|
||||
) -> PostboxFileReferenceRef:
|
||||
return PostboxFileReferenceRef(
|
||||
reference_type=request.reference_type,
|
||||
reference_id=request.reference_id,
|
||||
available=False,
|
||||
reason_code=reason_code,
|
||||
)
|
||||
|
||||
|
||||
def postbox_reference_capability(
|
||||
context: ModuleContext,
|
||||
) -> FilesPostboxReferenceService:
|
||||
configure_runtime(registry=context.registry, settings=context.settings)
|
||||
return FilesPostboxReferenceService()
|
||||
|
||||
|
||||
class FilesAccessService(FileAccessProvider):
|
||||
def explain_resource_provenance(
|
||||
self,
|
||||
@@ -96,7 +331,7 @@ class FilesAccessService(FileAccessProvider):
|
||||
.filter(
|
||||
FileShare.tenant_id == asset.tenant_id,
|
||||
FileShare.file_asset_id == asset.id,
|
||||
FileShare.revoked_at.is_(None),
|
||||
effective_file_share_clause(),
|
||||
FileShare.permission.in_(sorted(permission_values)),
|
||||
or_(
|
||||
(FileShare.target_type == "user") & (FileShare.target_id == principal.membership_id),
|
||||
@@ -197,6 +432,228 @@ def access_capability(context: ModuleContext) -> FilesAccessService:
|
||||
return FilesAccessService()
|
||||
|
||||
|
||||
class FilesManagedTabularFileService(ManagedTabularFileProvider):
|
||||
"""Expose authorized immutable CSV/XLSX versions to optional consumers."""
|
||||
|
||||
def list_tabular_files(
|
||||
self,
|
||||
session: object,
|
||||
principal: object,
|
||||
*,
|
||||
query: str = "",
|
||||
limit: int = 100,
|
||||
) -> tuple[ManagedTabularFile, ...]:
|
||||
db, api_principal, user_id = _tabular_context(
|
||||
session,
|
||||
principal,
|
||||
required_scope="files:file:read",
|
||||
)
|
||||
normalized_query = str(query or "").strip().casefold()
|
||||
requested_limit = max(1, min(int(limit), 100))
|
||||
assets = list_recent_assets_for_user(
|
||||
db,
|
||||
tenant_id=api_principal.tenant_id,
|
||||
user_id=user_id,
|
||||
limit=min(500, requested_limit * 5),
|
||||
is_admin=has_scope(api_principal, "files:file:admin"),
|
||||
)
|
||||
results: list[ManagedTabularFile] = []
|
||||
for asset in assets:
|
||||
if normalized_query and normalized_query not in (
|
||||
f"{asset.filename} {asset.display_path} {asset.description or ''}"
|
||||
).casefold():
|
||||
continue
|
||||
try:
|
||||
version, blob = current_version_and_blob(db, asset)
|
||||
except FileStorageError:
|
||||
continue
|
||||
if not _is_tabular_file(version.filename_at_upload, version.content_type):
|
||||
continue
|
||||
results.append(_managed_tabular_file(asset, version, blob))
|
||||
if len(results) >= requested_limit:
|
||||
break
|
||||
return tuple(results)
|
||||
|
||||
def get_tabular_file(
|
||||
self,
|
||||
session: object,
|
||||
principal: object,
|
||||
*,
|
||||
file_asset_id: str,
|
||||
file_version_id: str | None = None,
|
||||
) -> ManagedTabularFile | None:
|
||||
db, api_principal, user_id = _tabular_context(
|
||||
session,
|
||||
principal,
|
||||
required_scope="files:file:read",
|
||||
)
|
||||
try:
|
||||
asset = get_asset_for_user(
|
||||
db,
|
||||
tenant_id=api_principal.tenant_id,
|
||||
user_id=user_id,
|
||||
asset_id=file_asset_id,
|
||||
is_admin=has_scope(api_principal, "files:file:admin"),
|
||||
)
|
||||
except FileStorageError:
|
||||
return None
|
||||
if file_version_id:
|
||||
version = db.get(FileVersion, file_version_id)
|
||||
if (
|
||||
version is None
|
||||
or version.file_asset_id != asset.id
|
||||
or version.tenant_id != api_principal.tenant_id
|
||||
):
|
||||
return None
|
||||
blob = db.get(FileBlob, version.blob_id)
|
||||
if blob is None or blob.tenant_id != api_principal.tenant_id:
|
||||
return None
|
||||
else:
|
||||
try:
|
||||
version, blob = current_version_and_blob(db, asset)
|
||||
except FileStorageError as exc:
|
||||
raise ManagedTabularFileUnavailableError(
|
||||
"Managed file version metadata is unavailable."
|
||||
) from exc
|
||||
if not _is_tabular_file(version.filename_at_upload, version.content_type):
|
||||
return None
|
||||
return _managed_tabular_file(asset, version, blob)
|
||||
|
||||
def read_tabular_file(
|
||||
self,
|
||||
session: object,
|
||||
principal: object,
|
||||
*,
|
||||
file_asset_id: str,
|
||||
file_version_id: str,
|
||||
max_bytes: int,
|
||||
) -> ManagedTabularFileContent:
|
||||
db, api_principal, user_id = _tabular_context(
|
||||
session,
|
||||
principal,
|
||||
required_scope="files:file:download",
|
||||
)
|
||||
try:
|
||||
asset = get_asset_for_user(
|
||||
db,
|
||||
tenant_id=api_principal.tenant_id,
|
||||
user_id=user_id,
|
||||
asset_id=file_asset_id,
|
||||
is_admin=has_scope(api_principal, "files:file:admin"),
|
||||
)
|
||||
except FileStorageError as exc:
|
||||
raise ManagedTabularFileNotFoundError(
|
||||
"Managed tabular file not found."
|
||||
) from exc
|
||||
metadata = self.get_tabular_file(
|
||||
db,
|
||||
api_principal,
|
||||
file_asset_id=asset.id,
|
||||
file_version_id=file_version_id,
|
||||
)
|
||||
if metadata is None:
|
||||
raise ManagedTabularFileNotFoundError(
|
||||
"Managed tabular file version not found."
|
||||
)
|
||||
effective_max = max(1, int(max_bytes))
|
||||
if metadata.size_bytes > effective_max:
|
||||
raise ManagedTabularFileValidationError(
|
||||
f"Managed tabular files are limited to {effective_max:,} bytes for this operation."
|
||||
)
|
||||
try:
|
||||
payload, version, blob = read_asset_version_bytes(
|
||||
db,
|
||||
asset,
|
||||
file_version_id,
|
||||
)
|
||||
except FileStorageError as exc:
|
||||
raise ManagedTabularFileUnavailableError(
|
||||
"Managed tabular file content is unavailable or failed integrity verification."
|
||||
) from exc
|
||||
if len(payload) > effective_max:
|
||||
raise ManagedTabularFileValidationError(
|
||||
f"Managed tabular files are limited to {effective_max:,} bytes for this operation."
|
||||
)
|
||||
result = _managed_tabular_file(asset, version, blob)
|
||||
audit_from_principal(
|
||||
db,
|
||||
api_principal,
|
||||
action="files.tabular_content.read",
|
||||
object_type="file_version",
|
||||
object_id=version.id,
|
||||
details={
|
||||
"file_asset_id": asset.id,
|
||||
"size_bytes": version.size_bytes,
|
||||
"checksum_sha256": version.checksum_sha256,
|
||||
"consumer": "tabular_content",
|
||||
},
|
||||
)
|
||||
return ManagedTabularFileContent(file=result, payload=payload)
|
||||
|
||||
|
||||
def managed_tabular_file_capability(
|
||||
context: ModuleContext,
|
||||
) -> FilesManagedTabularFileService:
|
||||
configure_runtime(registry=context.registry, settings=context.settings)
|
||||
return FilesManagedTabularFileService()
|
||||
|
||||
|
||||
def _tabular_context(
|
||||
session: object,
|
||||
principal: object,
|
||||
*,
|
||||
required_scope: str,
|
||||
) -> tuple[Session, ApiPrincipal, str]:
|
||||
if not isinstance(session, Session):
|
||||
raise TypeError("Managed tabular file access requires a SQLAlchemy session.")
|
||||
if not isinstance(principal, ApiPrincipal):
|
||||
raise ManagedTabularFileAccessError(
|
||||
"Managed tabular file access requires a tenant API principal."
|
||||
)
|
||||
if not (
|
||||
has_scope(principal, required_scope)
|
||||
or has_scope(principal, "files:file:admin")
|
||||
):
|
||||
raise ManagedTabularFileAccessError(
|
||||
f"Managed tabular file access requires {required_scope}."
|
||||
)
|
||||
user_id = str(getattr(principal.user, "id", "") or "").strip()
|
||||
if not principal.tenant_id or not user_id:
|
||||
raise ManagedTabularFileAccessError(
|
||||
"Managed tabular file access requires a tenant user principal."
|
||||
)
|
||||
return session, principal, user_id
|
||||
|
||||
|
||||
def _managed_tabular_file(
|
||||
asset: FileAsset,
|
||||
version: FileVersion,
|
||||
blob: FileBlob,
|
||||
) -> ManagedTabularFile:
|
||||
return ManagedTabularFile(
|
||||
file_asset_id=asset.id,
|
||||
file_version_id=version.id,
|
||||
filename=version.filename_at_upload or asset.filename,
|
||||
display_path=version.display_path_at_upload or asset.display_path,
|
||||
content_type=version.content_type or blob.content_type,
|
||||
size_bytes=version.size_bytes,
|
||||
sha256=version.checksum_sha256,
|
||||
updated_at=version.created_at,
|
||||
current_version=asset.current_version_id == version.id,
|
||||
)
|
||||
|
||||
|
||||
def _is_tabular_file(filename: str, content_type: str | None) -> bool:
|
||||
normalized_name = str(filename or "").strip().casefold()
|
||||
normalized_type = str(content_type or "").split(";", 1)[0].strip().casefold()
|
||||
return normalized_name.endswith((".csv", ".xlsx")) or normalized_type in {
|
||||
"text/csv",
|
||||
"application/csv",
|
||||
"application/vnd.ms-excel",
|
||||
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
}
|
||||
|
||||
|
||||
def virtual_folder_resource_id(*, tenant_id: str, owner_type: str, owner_id: str, path: str) -> str:
|
||||
normalized_path = normalize_folder(path)
|
||||
encoded_path = base64.urlsafe_b64encode(normalized_path.encode("utf-8")).decode("ascii").rstrip("=")
|
||||
|
||||
@@ -56,6 +56,10 @@ def _record_asset_change(session: OrmSession, asset: FileAsset) -> None:
|
||||
"filename",
|
||||
"description",
|
||||
"deleted_at",
|
||||
"retained_until",
|
||||
"legal_hold",
|
||||
"lifecycle_revision",
|
||||
"lifecycle_reason",
|
||||
"metadata_",
|
||||
),
|
||||
)
|
||||
@@ -79,6 +83,9 @@ def _record_asset_change(session: OrmSession, asset: FileAsset) -> None:
|
||||
"previous_path": previous_value(asset, "display_path"),
|
||||
"filename": asset.filename,
|
||||
"deleted_at": _isoformat(asset.deleted_at),
|
||||
"retained_until": _isoformat(asset.retained_until),
|
||||
"legal_hold": asset.legal_hold,
|
||||
"lifecycle_revision": asset.lifecycle_revision,
|
||||
},
|
||||
)
|
||||
|
||||
@@ -117,7 +124,15 @@ def _record_share_visibility_change(session: OrmSession, share: FileShare) -> No
|
||||
return
|
||||
if not state.pending and not has_attr_changes(
|
||||
state,
|
||||
("file_asset_id", "target_type", "target_id", "permission", "revoked_at"),
|
||||
(
|
||||
"file_asset_id",
|
||||
"target_type",
|
||||
"target_id",
|
||||
"permission",
|
||||
"expires_at",
|
||||
"revoked_at",
|
||||
"revoked_by_user_id",
|
||||
),
|
||||
):
|
||||
return
|
||||
_ensure_id(share)
|
||||
@@ -136,7 +151,9 @@ def _record_share_visibility_change(session: OrmSession, share: FileShare) -> No
|
||||
"share_target_type": share.target_type,
|
||||
"share_target_id": share.target_id,
|
||||
"share_permission": share.permission,
|
||||
"share_expires_at": _isoformat(share.expires_at),
|
||||
"share_revoked_at": _isoformat(share.revoked_at),
|
||||
"share_revoked_by_user_id": share.revoked_by_user_id,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@@ -0,0 +1,600 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from govoplan_core.core.configuration_packages import (
|
||||
ConfigurationApplyResult,
|
||||
ConfigurationDiagnostic,
|
||||
ConfigurationExportResult,
|
||||
ConfigurationExportSelection,
|
||||
ConfigurationPackageFragment,
|
||||
ConfigurationPlanItem,
|
||||
ConfigurationPreflightContext,
|
||||
ConfigurationPreflightResult,
|
||||
ConfigurationProvider,
|
||||
ConfigurationProviderDescription,
|
||||
)
|
||||
from govoplan_core.core.infrastructure_capabilities import (
|
||||
InfrastructureCapability,
|
||||
InfrastructureCapabilityReceipt,
|
||||
)
|
||||
from govoplan_files.backend.runtime import settings as runtime_settings
|
||||
|
||||
|
||||
FILES_CONFIGURATION_CAPABILITY = "files.configuration"
|
||||
MANAGED_STORAGE_FRAGMENT = "managed_storage"
|
||||
_PAYLOAD_KEYS = frozenset(
|
||||
{"capability_id", "expected_backend", "expected_source"}
|
||||
)
|
||||
|
||||
|
||||
class FilesConfigurationProvider(ConfigurationProvider):
|
||||
module_id = "files"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
settings: object | None = None,
|
||||
environment: Mapping[str, str] | None = None,
|
||||
) -> None:
|
||||
self._settings = runtime_settings if settings is None else settings
|
||||
self._environment = os.environ if environment is None else environment
|
||||
|
||||
def describe(self) -> ConfigurationProviderDescription:
|
||||
return ConfigurationProviderDescription(
|
||||
module_id=self.module_id,
|
||||
fragment_types=(MANAGED_STORAGE_FRAGMENT,),
|
||||
schema_refs={
|
||||
MANAGED_STORAGE_FRAGMENT: "govoplan/files/configuration/managed-storage.v1"
|
||||
},
|
||||
exported_scopes=("system",),
|
||||
)
|
||||
|
||||
def preflight(
|
||||
self,
|
||||
fragment: ConfigurationPackageFragment,
|
||||
context: ConfigurationPreflightContext,
|
||||
) -> ConfigurationPreflightResult:
|
||||
if fragment.fragment_type != MANAGED_STORAGE_FRAGMENT:
|
||||
return ConfigurationPreflightResult(
|
||||
diagnostics=(_unsupported(fragment),),
|
||||
plan=(_blocked_plan(fragment, "Fragment type is unsupported."),),
|
||||
)
|
||||
diagnostics, binding_ref = self._binding_diagnostics(fragment, context)
|
||||
blocked = any(item.severity == "blocker" for item in diagnostics)
|
||||
return ConfigurationPreflightResult(
|
||||
diagnostics=tuple(diagnostics),
|
||||
plan=(
|
||||
ConfigurationPlanItem(
|
||||
action="blocked" if blocked else "skip",
|
||||
module_id="files",
|
||||
fragment_type=MANAGED_STORAGE_FRAGMENT,
|
||||
fragment_id=fragment.fragment_id or binding_ref,
|
||||
summary=(
|
||||
"Managed storage does not match the deployment receipt."
|
||||
if blocked
|
||||
else "Deployment-owned managed storage already matches the receipt; no module state is rewritten."
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
fragment: ConfigurationPackageFragment,
|
||||
supplied_data: Mapping[str, Any],
|
||||
context: ConfigurationPreflightContext,
|
||||
) -> ConfigurationApplyResult:
|
||||
del supplied_data
|
||||
if fragment.fragment_type != MANAGED_STORAGE_FRAGMENT:
|
||||
return ConfigurationApplyResult(diagnostics=(_unsupported(fragment),))
|
||||
diagnostics, _binding_ref = self._binding_diagnostics(fragment, context)
|
||||
return ConfigurationApplyResult(
|
||||
diagnostics=tuple(
|
||||
item for item in diagnostics if item.severity == "blocker"
|
||||
)
|
||||
)
|
||||
|
||||
def export(
|
||||
self,
|
||||
selection: ConfigurationExportSelection,
|
||||
context: ConfigurationPreflightContext,
|
||||
) -> ConfigurationExportResult:
|
||||
del selection
|
||||
receipt = context.infrastructure_receipt
|
||||
if context.infrastructure_receipt_error:
|
||||
return ConfigurationExportResult(
|
||||
diagnostics=(
|
||||
_receipt_error(
|
||||
context.infrastructure_receipt_error,
|
||||
object_ref="files.storage",
|
||||
),
|
||||
)
|
||||
)
|
||||
if receipt is None:
|
||||
return ConfigurationExportResult(
|
||||
diagnostics=(
|
||||
ConfigurationDiagnostic(
|
||||
severity="blocker",
|
||||
code="infrastructure_receipt_missing",
|
||||
message="Files managed-storage export requires the deployment capability receipt.",
|
||||
module_id="files",
|
||||
object_ref="files.storage",
|
||||
resolution="Mount the installer-generated receipt before exporting deployment configuration.",
|
||||
),
|
||||
)
|
||||
)
|
||||
capability = receipt.capability("files.storage")
|
||||
if capability is None:
|
||||
return ConfigurationExportResult(
|
||||
diagnostics=(
|
||||
ConfigurationDiagnostic(
|
||||
severity="blocker",
|
||||
code="infrastructure_capability_missing",
|
||||
message="The deployment receipt does not declare managed file storage.",
|
||||
module_id="files",
|
||||
object_ref="files.storage",
|
||||
),
|
||||
)
|
||||
)
|
||||
return ConfigurationExportResult(
|
||||
fragments=(
|
||||
ConfigurationPackageFragment(
|
||||
module_id="files",
|
||||
fragment_type=MANAGED_STORAGE_FRAGMENT,
|
||||
fragment_id=f"{receipt.installation_id}:files.storage",
|
||||
payload={
|
||||
"capability_id": "files.storage",
|
||||
"expected_backend": _expected_backend(capability.source),
|
||||
"expected_source": capability.source,
|
||||
},
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
def health(
|
||||
self,
|
||||
import_result: ConfigurationApplyResult,
|
||||
context: ConfigurationPreflightContext,
|
||||
) -> tuple[ConfigurationDiagnostic, ...]:
|
||||
del context
|
||||
return tuple(
|
||||
item for item in import_result.diagnostics if item.severity == "blocker"
|
||||
)
|
||||
|
||||
def _binding_diagnostics(
|
||||
self,
|
||||
fragment: ConfigurationPackageFragment,
|
||||
context: ConfigurationPreflightContext,
|
||||
) -> tuple[list[ConfigurationDiagnostic], str]:
|
||||
diagnostics: list[ConfigurationDiagnostic] = []
|
||||
payload = _files_fragment_payload(fragment, diagnostics)
|
||||
if payload is None:
|
||||
return diagnostics, fragment.fragment_id or "files.storage"
|
||||
receipt_state = _files_receipt_capability(
|
||||
fragment,
|
||||
context,
|
||||
payload,
|
||||
diagnostics,
|
||||
)
|
||||
if receipt_state is None:
|
||||
return diagnostics, fragment.fragment_id or "files.storage"
|
||||
_receipt, capability, binding_ref = receipt_state
|
||||
expected_backend = _expected_backend(capability.source)
|
||||
diagnostics.extend(
|
||||
_package_expectation_diagnostics(
|
||||
payload,
|
||||
capability,
|
||||
expected_backend,
|
||||
binding_ref,
|
||||
)
|
||||
)
|
||||
active_backend = _normalized_backend(
|
||||
getattr(self._settings, "file_storage_backend", "local")
|
||||
)
|
||||
if active_backend != expected_backend:
|
||||
diagnostics.append(
|
||||
ConfigurationDiagnostic(
|
||||
severity="blocker",
|
||||
code="files_storage_runtime_mismatch",
|
||||
message=(
|
||||
f"Files runtime uses {active_backend!r}, while the deployment receipt declares {expected_backend!r}."
|
||||
),
|
||||
module_id="files",
|
||||
object_ref=binding_ref,
|
||||
resolution="Reconcile deployment environment and receipt before starting or importing Files configuration.",
|
||||
)
|
||||
)
|
||||
return diagnostics, binding_ref
|
||||
diagnostics.extend(
|
||||
self._runtime_storage_diagnostics(
|
||||
expected_backend,
|
||||
capability,
|
||||
binding_ref,
|
||||
)
|
||||
)
|
||||
if "files" not in capability.dependent_modules:
|
||||
diagnostics.append(
|
||||
ConfigurationDiagnostic(
|
||||
severity="info",
|
||||
code="infrastructure_consumer_not_declared",
|
||||
message="Files is active but was not selected as a receipt consumer when the deployment plan was generated.",
|
||||
module_id="files",
|
||||
object_ref=binding_ref,
|
||||
resolution="Regenerate the deployment plan so removal-impact inventory includes Files.",
|
||||
)
|
||||
)
|
||||
return diagnostics, binding_ref
|
||||
|
||||
def _runtime_storage_diagnostics(
|
||||
self,
|
||||
expected_backend: str,
|
||||
capability: InfrastructureCapability,
|
||||
binding_ref: str,
|
||||
) -> list[ConfigurationDiagnostic]:
|
||||
if expected_backend == "s3":
|
||||
return self._s3_diagnostics(
|
||||
capability.endpoint,
|
||||
capability.secret_refs,
|
||||
capability.source,
|
||||
binding_ref,
|
||||
)
|
||||
root = Path(
|
||||
str(getattr(self._settings, "file_storage_local_root", "") or "")
|
||||
)
|
||||
if root.is_absolute():
|
||||
return []
|
||||
return [
|
||||
ConfigurationDiagnostic(
|
||||
severity="blocker",
|
||||
code="files_storage_local_root_not_durable",
|
||||
message="Receipt-bound local file storage requires an absolute deployment-managed path.",
|
||||
module_id="files",
|
||||
object_ref=binding_ref,
|
||||
resolution="Set FILE_STORAGE_LOCAL_ROOT to the mounted persistent-volume path.",
|
||||
)
|
||||
]
|
||||
|
||||
def _s3_diagnostics(
|
||||
self,
|
||||
endpoint: Mapping[str, object],
|
||||
secret_refs: tuple[str, ...],
|
||||
source: str,
|
||||
object_ref: str,
|
||||
) -> list[ConfigurationDiagnostic]:
|
||||
diagnostics: list[ConfigurationDiagnostic] = []
|
||||
endpoint_url = str(
|
||||
getattr(self._settings, "file_storage_s3_endpoint_url", "")
|
||||
or getattr(self._settings, "s3_endpoint_url", "")
|
||||
or ""
|
||||
).strip()
|
||||
runtime_endpoint = _redacted_endpoint(
|
||||
endpoint_url,
|
||||
default_port=int(endpoint.get("port") or 443),
|
||||
)
|
||||
receipt_endpoint = {
|
||||
key: endpoint.get(key) for key in ("scheme", "host", "port") if endpoint.get(key) is not None
|
||||
}
|
||||
if receipt_endpoint and runtime_endpoint != receipt_endpoint:
|
||||
diagnostics.append(
|
||||
ConfigurationDiagnostic(
|
||||
severity="blocker",
|
||||
code="files_storage_endpoint_mismatch",
|
||||
message="Files S3 endpoint does not match the sanitized deployment receipt endpoint.",
|
||||
module_id="files",
|
||||
object_ref=object_ref,
|
||||
resolution="Reconcile FILE_STORAGE_S3_ENDPOINT_URL and the deployment plan before apply.",
|
||||
)
|
||||
)
|
||||
bucket = str(
|
||||
getattr(self._settings, "file_storage_s3_bucket", "")
|
||||
or getattr(self._settings, "s3_bucket", "")
|
||||
or ""
|
||||
).strip()
|
||||
if not bucket:
|
||||
diagnostics.append(
|
||||
ConfigurationDiagnostic(
|
||||
severity="blocker",
|
||||
code="files_storage_bucket_missing",
|
||||
message="Files S3 storage has no configured bucket.",
|
||||
module_id="files",
|
||||
object_ref=object_ref,
|
||||
resolution="Set FILE_STORAGE_S3_BUCKET in the deployment environment.",
|
||||
)
|
||||
)
|
||||
for reference in secret_refs:
|
||||
variable = reference.removeprefix("env:")
|
||||
if not variable.startswith("FILE_STORAGE_"):
|
||||
continue
|
||||
if not str(self._environment.get(variable, "")).strip():
|
||||
diagnostics.append(
|
||||
ConfigurationDiagnostic(
|
||||
severity="blocker",
|
||||
code="files_storage_secret_reference_unresolved",
|
||||
message=f"Required storage secret reference {reference} is not available to the runtime.",
|
||||
module_id="files",
|
||||
object_ref=object_ref,
|
||||
resolution="Provide the referenced environment secret without copying its value into the package.",
|
||||
)
|
||||
)
|
||||
deployment_managed = bool(
|
||||
getattr(
|
||||
self._settings,
|
||||
"file_storage_s3_deployment_managed",
|
||||
False,
|
||||
)
|
||||
)
|
||||
endpoint_trusted = bool(
|
||||
getattr(self._settings, "file_storage_s3_endpoint_trusted", False)
|
||||
)
|
||||
if source == "installer-managed-garage" and not deployment_managed:
|
||||
diagnostics.append(
|
||||
ConfigurationDiagnostic(
|
||||
severity="blocker",
|
||||
code="files_storage_management_boundary_mismatch",
|
||||
message="Garage is installer-managed in the receipt but not marked deployment-managed in Files runtime.",
|
||||
module_id="files",
|
||||
object_ref=object_ref,
|
||||
resolution="Set FILE_STORAGE_S3_DEPLOYMENT_MANAGED=true for the installer-managed Garage endpoint.",
|
||||
)
|
||||
)
|
||||
if source == "operator-supplied-s3" and not endpoint_trusted:
|
||||
diagnostics.append(
|
||||
ConfigurationDiagnostic(
|
||||
severity="blocker",
|
||||
code="files_storage_trust_boundary_missing",
|
||||
message="External S3 storage must be explicitly marked as a trusted deployment endpoint.",
|
||||
module_id="files",
|
||||
object_ref=object_ref,
|
||||
resolution="Review the endpoint and set FILE_STORAGE_S3_ENDPOINT_TRUSTED=true in deployment configuration.",
|
||||
)
|
||||
)
|
||||
return diagnostics
|
||||
|
||||
|
||||
def _files_fragment_payload(
|
||||
fragment: ConfigurationPackageFragment,
|
||||
diagnostics: list[ConfigurationDiagnostic],
|
||||
) -> Mapping[str, Any] | None:
|
||||
payload = fragment.payload
|
||||
if not isinstance(payload, Mapping):
|
||||
diagnostics.append(
|
||||
_invalid(fragment, "Managed-storage payload must be an object.")
|
||||
)
|
||||
return None
|
||||
unknown = sorted(set(payload) - _PAYLOAD_KEYS)
|
||||
if not unknown:
|
||||
return payload
|
||||
secret_like = any(
|
||||
marker in key.casefold()
|
||||
for key in unknown
|
||||
for marker in ("password", "secret", "token", "credential", "access_key")
|
||||
)
|
||||
diagnostics.append(
|
||||
ConfigurationDiagnostic(
|
||||
severity="blocker",
|
||||
code=(
|
||||
"files_configuration_secret_forbidden"
|
||||
if secret_like
|
||||
else "files_configuration_payload_invalid"
|
||||
),
|
||||
message=(
|
||||
"Managed-storage fragments accept receipt references and non-secret expectations only."
|
||||
if secret_like
|
||||
else f"Managed-storage payload contains unsupported fields: {', '.join(unknown)}."
|
||||
),
|
||||
module_id="files",
|
||||
object_ref=fragment.fragment_id or MANAGED_STORAGE_FRAGMENT,
|
||||
resolution="Keep storage credentials in deployment environment references, never in a configuration package.",
|
||||
)
|
||||
)
|
||||
return payload
|
||||
|
||||
|
||||
def _files_receipt_capability(
|
||||
fragment: ConfigurationPackageFragment,
|
||||
context: ConfigurationPreflightContext,
|
||||
payload: Mapping[str, Any],
|
||||
diagnostics: list[ConfigurationDiagnostic],
|
||||
) -> tuple[
|
||||
InfrastructureCapabilityReceipt,
|
||||
InfrastructureCapability,
|
||||
str,
|
||||
] | None:
|
||||
if context.infrastructure_receipt_error:
|
||||
diagnostics.append(
|
||||
_receipt_error(
|
||||
context.infrastructure_receipt_error,
|
||||
object_ref=fragment.fragment_id or "files.storage",
|
||||
)
|
||||
)
|
||||
return None
|
||||
receipt = context.infrastructure_receipt
|
||||
if receipt is None:
|
||||
diagnostics.append(
|
||||
ConfigurationDiagnostic(
|
||||
severity="blocker",
|
||||
code="infrastructure_receipt_missing",
|
||||
message="Files managed-storage configuration requires the deployment capability receipt.",
|
||||
module_id="files",
|
||||
object_ref=fragment.fragment_id or "files.storage",
|
||||
resolution="Mount the installer-generated receipt and rerun preflight.",
|
||||
)
|
||||
)
|
||||
return None
|
||||
binding_ref = fragment.fragment_id or f"{receipt.installation_id}:files.storage"
|
||||
capability_id = _text(payload.get("capability_id")) or "files.storage"
|
||||
if capability_id != "files.storage":
|
||||
diagnostics.append(
|
||||
_invalid(
|
||||
fragment,
|
||||
"Files managed-storage fragments must reference capability 'files.storage'.",
|
||||
)
|
||||
)
|
||||
return None
|
||||
capability = receipt.capability(capability_id)
|
||||
if capability is None:
|
||||
diagnostics.append(
|
||||
ConfigurationDiagnostic(
|
||||
severity="blocker",
|
||||
code="infrastructure_capability_missing",
|
||||
message="The deployment receipt does not declare managed file storage.",
|
||||
module_id="files",
|
||||
object_ref=binding_ref,
|
||||
resolution="Regenerate the receipt from a deployment profile that declares managed storage.",
|
||||
)
|
||||
)
|
||||
return None
|
||||
state_diagnostic = _storage_capability_state_diagnostic(capability, binding_ref)
|
||||
if state_diagnostic is not None:
|
||||
diagnostics.append(state_diagnostic)
|
||||
return None
|
||||
return receipt, capability, binding_ref
|
||||
|
||||
|
||||
def _storage_capability_state_diagnostic(
|
||||
capability: InfrastructureCapability,
|
||||
binding_ref: str,
|
||||
) -> ConfigurationDiagnostic | None:
|
||||
if capability.state == "unavailable":
|
||||
return ConfigurationDiagnostic(
|
||||
severity="blocker",
|
||||
code="infrastructure_capability_unavailable",
|
||||
message="The deployment receipt states that managed file storage is unavailable.",
|
||||
module_id="files",
|
||||
object_ref=binding_ref,
|
||||
resolution="Select local, managed Garage, or external S3 storage in the deployment profile.",
|
||||
)
|
||||
if capability.state == "available_unconfigured":
|
||||
return ConfigurationDiagnostic(
|
||||
severity="blocker",
|
||||
code="infrastructure_capability_unconfigured",
|
||||
message="Managed file storage is available but has not been bound by the deployment runtime.",
|
||||
module_id="files",
|
||||
object_ref=binding_ref,
|
||||
resolution="Complete the deployment-owned storage configuration before importing Files configuration.",
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _package_expectation_diagnostics(
|
||||
payload: Mapping[str, Any],
|
||||
capability: InfrastructureCapability,
|
||||
expected_backend: str,
|
||||
binding_ref: str,
|
||||
) -> list[ConfigurationDiagnostic]:
|
||||
diagnostics: list[ConfigurationDiagnostic] = []
|
||||
package_backend = (
|
||||
_text(payload.get("expected_backend")) or expected_backend
|
||||
).casefold()
|
||||
if package_backend != expected_backend:
|
||||
diagnostics.append(
|
||||
ConfigurationDiagnostic(
|
||||
severity="blocker",
|
||||
code="infrastructure_backend_mismatch",
|
||||
message="The package storage backend expectation conflicts with the deployment receipt.",
|
||||
module_id="files",
|
||||
object_ref=binding_ref,
|
||||
resolution="Use the receipt backend or regenerate the deployment package after review.",
|
||||
)
|
||||
)
|
||||
package_source = _text(payload.get("expected_source"))
|
||||
if package_source and package_source != capability.source:
|
||||
diagnostics.append(
|
||||
ConfigurationDiagnostic(
|
||||
severity="blocker",
|
||||
code="infrastructure_source_mismatch",
|
||||
message="The package storage source expectation conflicts with the deployment receipt.",
|
||||
module_id="files",
|
||||
object_ref=binding_ref,
|
||||
resolution="Review the provider replacement and regenerate the package from the active receipt.",
|
||||
)
|
||||
)
|
||||
return diagnostics
|
||||
|
||||
|
||||
def _expected_backend(source: str) -> str:
|
||||
return "local" if source == "host-local" else "s3"
|
||||
|
||||
|
||||
def _normalized_backend(value: object) -> str:
|
||||
clean = str(value or "local").strip().casefold()
|
||||
if clean in {"local", "filesystem", "fs"}:
|
||||
return "local"
|
||||
if clean in {"s3", "garage"}:
|
||||
return "s3"
|
||||
return clean
|
||||
|
||||
|
||||
def _redacted_endpoint(value: str, *, default_port: int) -> dict[str, object]:
|
||||
try:
|
||||
parsed = urlsplit(value)
|
||||
host = parsed.hostname
|
||||
if not parsed.scheme or not host:
|
||||
return {"reference": "unresolved"}
|
||||
port = parsed.port or default_port
|
||||
except ValueError:
|
||||
return {"reference": "unresolved"}
|
||||
return {"scheme": parsed.scheme, "host": host, "port": port}
|
||||
|
||||
|
||||
def _text(value: object) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
text = str(value).strip()
|
||||
return text or None
|
||||
|
||||
|
||||
def _receipt_error(error: str, *, object_ref: str) -> ConfigurationDiagnostic:
|
||||
return ConfigurationDiagnostic(
|
||||
severity="blocker",
|
||||
code="infrastructure_receipt_invalid",
|
||||
message=f"The deployment capability receipt is invalid: {error}",
|
||||
module_id="files",
|
||||
object_ref=object_ref,
|
||||
resolution="Repair or regenerate the deployment receipt before importing Files configuration.",
|
||||
)
|
||||
|
||||
|
||||
def _invalid(
|
||||
fragment: ConfigurationPackageFragment,
|
||||
message: str,
|
||||
) -> ConfigurationDiagnostic:
|
||||
return ConfigurationDiagnostic(
|
||||
severity="blocker",
|
||||
code="files_configuration_payload_invalid",
|
||||
message=message,
|
||||
module_id="files",
|
||||
object_ref=fragment.fragment_id or fragment.fragment_type,
|
||||
resolution="Review the Files configuration-package fragment and rerun preflight.",
|
||||
)
|
||||
|
||||
|
||||
def _unsupported(fragment: ConfigurationPackageFragment) -> ConfigurationDiagnostic:
|
||||
return ConfigurationDiagnostic(
|
||||
severity="blocker",
|
||||
code="fragment_type_unsupported",
|
||||
message=f"Files configuration does not support fragment type {fragment.fragment_type!r}.",
|
||||
module_id="files",
|
||||
object_ref=fragment.fragment_id or fragment.fragment_type,
|
||||
)
|
||||
|
||||
|
||||
def _blocked_plan(
|
||||
fragment: ConfigurationPackageFragment,
|
||||
summary: str,
|
||||
) -> ConfigurationPlanItem:
|
||||
return ConfigurationPlanItem(
|
||||
action="blocked",
|
||||
module_id="files",
|
||||
fragment_type=fragment.fragment_type,
|
||||
fragment_id=fragment.fragment_id,
|
||||
summary=summary,
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["FILES_CONFIGURATION_CAPABILITY", "FilesConfigurationProvider"]
|
||||
@@ -4,7 +4,18 @@ import uuid
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import Boolean, DateTime, ForeignKey, Index, Integer, JSON, String, Text, UniqueConstraint, text
|
||||
from sqlalchemy import (
|
||||
Boolean,
|
||||
DateTime,
|
||||
ForeignKey,
|
||||
Index,
|
||||
Integer,
|
||||
JSON,
|
||||
String,
|
||||
Text,
|
||||
UniqueConstraint,
|
||||
text,
|
||||
)
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from govoplan_core.db.base import Base, TimestampMixin
|
||||
@@ -16,7 +27,15 @@ def new_uuid() -> str:
|
||||
|
||||
class FileBlob(Base, TimestampMixin):
|
||||
__tablename__ = "file_blobs"
|
||||
__table_args__ = (UniqueConstraint("tenant_id", "checksum_sha256", "size_bytes", name="uq_file_blobs_tenant_checksum_size"),)
|
||||
__table_args__ = (
|
||||
UniqueConstraint(
|
||||
"tenant_id",
|
||||
"checksum_sha256",
|
||||
"size_bytes",
|
||||
"protection_discriminator",
|
||||
name="uq_file_blobs_tenant_checksum_size_protection",
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||
@@ -25,9 +44,106 @@ class FileBlob(Base, TimestampMixin):
|
||||
storage_key: Mapped[str] = mapped_column(String(1000), nullable=False)
|
||||
checksum_sha256: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
size_bytes: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
protection_discriminator: Mapped[str] = mapped_column(
|
||||
String(320), default="plaintext", nullable=False, index=True
|
||||
)
|
||||
encryption_envelope_id: Mapped[str | None] = mapped_column(
|
||||
String(255), nullable=True, index=True
|
||||
)
|
||||
storage_checksum_sha256: Mapped[str | None] = mapped_column(
|
||||
String(64), nullable=True
|
||||
)
|
||||
storage_size_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
content_type: Mapped[str | None] = mapped_column(String(255))
|
||||
ref_count: Mapped[int] = mapped_column(Integer, default=1, nullable=False)
|
||||
retained_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
integrity_status: Mapped[str] = mapped_column(
|
||||
String(30), default="unchecked", nullable=False, index=True
|
||||
)
|
||||
integrity_checked_at: Mapped[datetime | None] = mapped_column(
|
||||
DateTime(timezone=True), nullable=True, index=True
|
||||
)
|
||||
integrity_failure: Mapped[str | None] = mapped_column(String(100), nullable=True)
|
||||
quarantined_at: Mapped[datetime | None] = mapped_column(
|
||||
DateTime(timezone=True), nullable=True, index=True
|
||||
)
|
||||
|
||||
|
||||
class FileIntegrityScan(Base, TimestampMixin):
|
||||
__tablename__ = "file_integrity_scans"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||
storage_backend: Mapped[str] = mapped_column(String(50), nullable=False)
|
||||
storage_prefix: Mapped[str] = mapped_column(String(1000), nullable=False)
|
||||
status: Mapped[str] = mapped_column(
|
||||
String(30), default="pending", nullable=False, index=True
|
||||
)
|
||||
revision: Mapped[int] = mapped_column(Integer, default=1, nullable=False)
|
||||
phase: Mapped[str] = mapped_column(String(30), default="blobs", nullable=False)
|
||||
verify_checksums: Mapped[bool] = mapped_column(
|
||||
Boolean, default=True, nullable=False
|
||||
)
|
||||
batch_size: Mapped[int] = mapped_column(Integer, default=100, nullable=False)
|
||||
blob_cursor: Mapped[str | None] = mapped_column(String(36), nullable=True)
|
||||
object_cursor: Mapped[str | None] = mapped_column(String(1000), nullable=True)
|
||||
scanned_blob_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
|
||||
verified_blob_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
|
||||
quarantined_blob_count: Mapped[int] = mapped_column(
|
||||
Integer, default=0, nullable=False
|
||||
)
|
||||
scanned_object_count: Mapped[int] = mapped_column(
|
||||
Integer, default=0, nullable=False
|
||||
)
|
||||
orphan_object_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
|
||||
created_by_user_id: Mapped[str | None] = mapped_column(
|
||||
ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True
|
||||
)
|
||||
started_at: Mapped[datetime | None] = mapped_column(
|
||||
DateTime(timezone=True), nullable=True
|
||||
)
|
||||
completed_at: Mapped[datetime | None] = mapped_column(
|
||||
DateTime(timezone=True), nullable=True
|
||||
)
|
||||
last_error: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
|
||||
|
||||
class FileIntegrityFinding(Base, TimestampMixin):
|
||||
__tablename__ = "file_integrity_findings"
|
||||
__table_args__ = (
|
||||
Index("ix_file_integrity_findings_scan_state", "scan_id", "state"),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||
scan_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("file_integrity_scans.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
index=True,
|
||||
)
|
||||
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||
kind: Mapped[str] = mapped_column(String(40), nullable=False, index=True)
|
||||
state: Mapped[str] = mapped_column(
|
||||
String(30), default="open", nullable=False, index=True
|
||||
)
|
||||
revision: Mapped[int] = mapped_column(Integer, default=1, nullable=False)
|
||||
blob_id: Mapped[str | None] = mapped_column(
|
||||
ForeignKey("file_blobs.id", ondelete="SET NULL"), nullable=True, index=True
|
||||
)
|
||||
storage_key: Mapped[str] = mapped_column(String(1000), nullable=False)
|
||||
expected_size_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
observed_size_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
expected_checksum_sha256: Mapped[str | None] = mapped_column(
|
||||
String(64), nullable=True
|
||||
)
|
||||
observed_checksum_sha256: Mapped[str | None] = mapped_column(
|
||||
String(64), nullable=True
|
||||
)
|
||||
resolved_at: Mapped[datetime | None] = mapped_column(
|
||||
DateTime(timezone=True), nullable=True
|
||||
)
|
||||
resolved_by_user_id: Mapped[str | None] = mapped_column(
|
||||
ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True
|
||||
)
|
||||
|
||||
|
||||
class FileFolder(Base, TimestampMixin):
|
||||
@@ -35,14 +151,18 @@ class FileFolder(Base, TimestampMixin):
|
||||
__table_args__ = (
|
||||
Index(
|
||||
"uq_file_folders_active_user_path",
|
||||
"tenant_id", "owner_user_id", "path",
|
||||
"tenant_id",
|
||||
"owner_user_id",
|
||||
"path",
|
||||
unique=True,
|
||||
sqlite_where=text("owner_type = 'user' AND deleted_at IS NULL"),
|
||||
postgresql_where=text("owner_type = 'user' AND deleted_at IS NULL"),
|
||||
),
|
||||
Index(
|
||||
"uq_file_folders_active_group_path",
|
||||
"tenant_id", "owner_group_id", "path",
|
||||
"tenant_id",
|
||||
"owner_group_id",
|
||||
"path",
|
||||
unique=True,
|
||||
sqlite_where=text("owner_type = 'group' AND deleted_at IS NULL"),
|
||||
postgresql_where=text("owner_type = 'group' AND deleted_at IS NULL"),
|
||||
@@ -52,12 +172,22 @@ class FileFolder(Base, TimestampMixin):
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||
owner_type: Mapped[str] = mapped_column(String(20), nullable=False, index=True)
|
||||
owner_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
|
||||
owner_group_id: Mapped[str | None] = mapped_column(ForeignKey("access_groups.id", ondelete="SET NULL"), nullable=True, index=True)
|
||||
owner_user_id: Mapped[str | None] = mapped_column(
|
||||
ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True
|
||||
)
|
||||
owner_group_id: Mapped[str | None] = mapped_column(
|
||||
ForeignKey("access_groups.id", ondelete="SET NULL"), nullable=True, index=True
|
||||
)
|
||||
path: Mapped[str] = mapped_column(String(1000), nullable=False, index=True)
|
||||
created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
|
||||
deleted_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
|
||||
metadata_: Mapped[dict[str, Any] | None] = mapped_column("metadata", JSON, nullable=True)
|
||||
created_by_user_id: Mapped[str | None] = mapped_column(
|
||||
ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True
|
||||
)
|
||||
deleted_at: Mapped[datetime | None] = mapped_column(
|
||||
DateTime(timezone=True), nullable=True, index=True
|
||||
)
|
||||
metadata_: Mapped[dict[str, Any] | None] = mapped_column(
|
||||
"metadata", JSON, nullable=True
|
||||
)
|
||||
|
||||
|
||||
class FileAsset(Base, TimestampMixin):
|
||||
@@ -66,46 +196,159 @@ class FileAsset(Base, TimestampMixin):
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||
owner_type: Mapped[str] = mapped_column(String(20), nullable=False, index=True)
|
||||
owner_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
|
||||
owner_group_id: Mapped[str | None] = mapped_column(ForeignKey("access_groups.id", ondelete="SET NULL"), nullable=True, index=True)
|
||||
current_version_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
|
||||
owner_user_id: Mapped[str | None] = mapped_column(
|
||||
ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True
|
||||
)
|
||||
owner_group_id: Mapped[str | None] = mapped_column(
|
||||
ForeignKey("access_groups.id", ondelete="SET NULL"), nullable=True, index=True
|
||||
)
|
||||
current_version_id: Mapped[str | None] = mapped_column(
|
||||
String(36), nullable=True, index=True
|
||||
)
|
||||
display_path: Mapped[str] = mapped_column(String(1000), nullable=False, index=True)
|
||||
filename: Mapped[str] = mapped_column(String(500), nullable=False, index=True)
|
||||
description: Mapped[str | None] = mapped_column(Text)
|
||||
created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
|
||||
deleted_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
|
||||
metadata_: Mapped[dict[str, Any] | None] = mapped_column("metadata", JSON, nullable=True)
|
||||
created_by_user_id: Mapped[str | None] = mapped_column(
|
||||
ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True
|
||||
)
|
||||
deleted_at: Mapped[datetime | None] = mapped_column(
|
||||
DateTime(timezone=True), nullable=True, index=True
|
||||
)
|
||||
retained_until: Mapped[datetime | None] = mapped_column(
|
||||
DateTime(timezone=True), nullable=True, index=True
|
||||
)
|
||||
legal_hold: Mapped[bool] = mapped_column(
|
||||
Boolean, default=False, nullable=False, index=True
|
||||
)
|
||||
lifecycle_revision: Mapped[int] = mapped_column(
|
||||
Integer, default=1, nullable=False
|
||||
)
|
||||
lifecycle_reason: Mapped[str | None] = mapped_column(String(500), nullable=True)
|
||||
metadata_: Mapped[dict[str, Any] | None] = mapped_column(
|
||||
"metadata", JSON, nullable=True
|
||||
)
|
||||
|
||||
|
||||
class FileVersion(Base, TimestampMixin):
|
||||
__tablename__ = "file_versions"
|
||||
__table_args__ = (UniqueConstraint("file_asset_id", "version_number", name="uq_file_versions_asset_number"),)
|
||||
__table_args__ = (
|
||||
UniqueConstraint(
|
||||
"file_asset_id", "version_number", name="uq_file_versions_asset_number"
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||
file_asset_id: Mapped[str] = mapped_column(ForeignKey("file_assets.id", ondelete="CASCADE"), nullable=False, index=True)
|
||||
blob_id: Mapped[str] = mapped_column(ForeignKey("file_blobs.id", ondelete="RESTRICT"), nullable=False, index=True)
|
||||
file_asset_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("file_assets.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
blob_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("file_blobs.id", ondelete="RESTRICT"), nullable=False, index=True
|
||||
)
|
||||
version_number: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
filename_at_upload: Mapped[str] = mapped_column(String(500), nullable=False)
|
||||
display_path_at_upload: Mapped[str] = mapped_column(String(1000), nullable=False)
|
||||
content_type: Mapped[str | None] = mapped_column(String(255))
|
||||
size_bytes: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
checksum_sha256: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
|
||||
created_by_user_id: Mapped[str | None] = mapped_column(
|
||||
ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True
|
||||
)
|
||||
|
||||
|
||||
class FileFormEvidenceGrant(Base, TimestampMixin):
|
||||
__tablename__ = "file_form_evidence_grants"
|
||||
__table_args__ = (
|
||||
UniqueConstraint(
|
||||
"tenant_id",
|
||||
"idempotency_key",
|
||||
name="uq_file_form_evidence_grants_idempotency",
|
||||
),
|
||||
Index(
|
||||
"ix_file_form_evidence_grants_form",
|
||||
"tenant_id",
|
||||
"form_instance_id",
|
||||
"form_definition_id",
|
||||
"form_definition_revision",
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||
form_instance_id: Mapped[str] = mapped_column(
|
||||
String(36), nullable=False, index=True
|
||||
)
|
||||
form_definition_id: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
form_definition_revision: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
token_sha256: Mapped[str] = mapped_column(String(64), nullable=False, unique=True)
|
||||
idempotency_key: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
request_sha256: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
custodian_user_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("access_users.id", ondelete="RESTRICT"),
|
||||
nullable=False,
|
||||
index=True,
|
||||
)
|
||||
evidence_kind: Mapped[str] = mapped_column(String(30), nullable=False)
|
||||
purpose: Mapped[str] = mapped_column(String(500), nullable=False)
|
||||
status: Mapped[str] = mapped_column(
|
||||
String(30), default="issued", nullable=False, index=True
|
||||
)
|
||||
expires_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, index=True
|
||||
)
|
||||
max_size_bytes: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
allowed_content_types: Mapped[list[str]] = mapped_column(
|
||||
JSON, default=list, nullable=False
|
||||
)
|
||||
file_asset_id: Mapped[str | None] = mapped_column(
|
||||
ForeignKey("file_assets.id", ondelete="RESTRICT"), nullable=True, index=True
|
||||
)
|
||||
file_version_id: Mapped[str | None] = mapped_column(
|
||||
ForeignKey("file_versions.id", ondelete="RESTRICT"), nullable=True, index=True
|
||||
)
|
||||
uploaded_at: Mapped[datetime | None] = mapped_column(
|
||||
DateTime(timezone=True), nullable=True
|
||||
)
|
||||
revoked_at: Mapped[datetime | None] = mapped_column(
|
||||
DateTime(timezone=True), nullable=True
|
||||
)
|
||||
metadata_: Mapped[dict[str, Any]] = mapped_column(
|
||||
"metadata", JSON, default=dict, nullable=False
|
||||
)
|
||||
|
||||
|
||||
class FileShare(Base, TimestampMixin):
|
||||
__tablename__ = "file_shares"
|
||||
__table_args__ = (UniqueConstraint("file_asset_id", "target_type", "target_id", "revoked_at", name="uq_file_shares_active_target"),)
|
||||
__table_args__ = (
|
||||
UniqueConstraint(
|
||||
"file_asset_id",
|
||||
"target_type",
|
||||
"target_id",
|
||||
"revoked_at",
|
||||
name="uq_file_shares_active_target",
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||
file_asset_id: Mapped[str] = mapped_column(ForeignKey("file_assets.id", ondelete="CASCADE"), nullable=False, index=True)
|
||||
file_asset_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("file_assets.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
target_type: Mapped[str] = mapped_column(String(20), nullable=False, index=True)
|
||||
target_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||
permission: Mapped[str] = mapped_column(String(20), default="read", nullable=False)
|
||||
created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
|
||||
revoked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
|
||||
created_by_user_id: Mapped[str | None] = mapped_column(
|
||||
ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True
|
||||
)
|
||||
expires_at: Mapped[datetime | None] = mapped_column(
|
||||
DateTime(timezone=True), nullable=True, index=True
|
||||
)
|
||||
revoked_at: Mapped[datetime | None] = mapped_column(
|
||||
DateTime(timezone=True), nullable=True, index=True
|
||||
)
|
||||
revoked_by_user_id: Mapped[str | None] = mapped_column(
|
||||
ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True
|
||||
)
|
||||
|
||||
|
||||
class FileConnectorProfile(Base, TimestampMixin):
|
||||
@@ -116,15 +359,23 @@ class FileConnectorProfile(Base, TimestampMixin):
|
||||
|
||||
id: Mapped[str] = mapped_column(String(255), primary_key=True)
|
||||
tenant_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
|
||||
scope_type: Mapped[str] = mapped_column(String(20), default="tenant", nullable=False, index=True)
|
||||
scope_type: Mapped[str] = mapped_column(
|
||||
String(20), default="tenant", nullable=False, index=True
|
||||
)
|
||||
scope_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
|
||||
label: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
provider: Mapped[str] = mapped_column(String(50), nullable=False, index=True)
|
||||
endpoint_url: Mapped[str | None] = mapped_column(String(1000), nullable=True)
|
||||
base_path: Mapped[str | None] = mapped_column(String(1000), nullable=True)
|
||||
enabled: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False, index=True)
|
||||
credential_profile_id: Mapped[str | None] = mapped_column(String(255), nullable=True, index=True)
|
||||
credential_mode: Mapped[str] = mapped_column(String(30), default="none", nullable=False)
|
||||
enabled: Mapped[bool] = mapped_column(
|
||||
Boolean, default=True, nullable=False, index=True
|
||||
)
|
||||
credential_profile_id: Mapped[str | None] = mapped_column(
|
||||
String(255), nullable=True, index=True
|
||||
)
|
||||
credential_mode: Mapped[str] = mapped_column(
|
||||
String(30), default="none", nullable=False
|
||||
)
|
||||
username: Mapped[str | None] = mapped_column(String(320), nullable=True)
|
||||
password_encrypted: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
token_encrypted: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
@@ -133,9 +384,15 @@ class FileConnectorProfile(Base, TimestampMixin):
|
||||
secret_ref: Mapped[str | None] = mapped_column(String(1000), nullable=True)
|
||||
capabilities: Mapped[list[str] | None] = mapped_column(JSON, nullable=True)
|
||||
policy: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
|
||||
metadata_: Mapped[dict[str, Any] | None] = mapped_column("metadata", JSON, nullable=True)
|
||||
created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
|
||||
updated_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
|
||||
metadata_: Mapped[dict[str, Any] | None] = mapped_column(
|
||||
"metadata", JSON, nullable=True
|
||||
)
|
||||
created_by_user_id: Mapped[str | None] = mapped_column(
|
||||
ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True
|
||||
)
|
||||
updated_by_user_id: Mapped[str | None] = mapped_column(
|
||||
ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True
|
||||
)
|
||||
|
||||
|
||||
class FileConnectorCredential(Base, TimestampMixin):
|
||||
@@ -146,12 +403,18 @@ class FileConnectorCredential(Base, TimestampMixin):
|
||||
|
||||
id: Mapped[str] = mapped_column(String(255), primary_key=True)
|
||||
tenant_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
|
||||
scope_type: Mapped[str] = mapped_column(String(20), default="tenant", nullable=False, index=True)
|
||||
scope_type: Mapped[str] = mapped_column(
|
||||
String(20), default="tenant", nullable=False, index=True
|
||||
)
|
||||
scope_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
|
||||
label: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
provider: Mapped[str | None] = mapped_column(String(50), nullable=True, index=True)
|
||||
enabled: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False, index=True)
|
||||
credential_mode: Mapped[str] = mapped_column(String(30), default="none", nullable=False)
|
||||
enabled: Mapped[bool] = mapped_column(
|
||||
Boolean, default=True, nullable=False, index=True
|
||||
)
|
||||
credential_mode: Mapped[str] = mapped_column(
|
||||
String(30), default="none", nullable=False
|
||||
)
|
||||
username: Mapped[str | None] = mapped_column(String(320), nullable=True)
|
||||
password_encrypted: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
token_encrypted: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
@@ -159,15 +422,26 @@ class FileConnectorCredential(Base, TimestampMixin):
|
||||
token_env: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
secret_ref: Mapped[str | None] = mapped_column(String(1000), nullable=True)
|
||||
policy: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
|
||||
metadata_: Mapped[dict[str, Any] | None] = mapped_column("metadata", JSON, nullable=True)
|
||||
created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
|
||||
updated_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
|
||||
metadata_: Mapped[dict[str, Any] | None] = mapped_column(
|
||||
"metadata", JSON, nullable=True
|
||||
)
|
||||
created_by_user_id: Mapped[str | None] = mapped_column(
|
||||
ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True
|
||||
)
|
||||
updated_by_user_id: Mapped[str | None] = mapped_column(
|
||||
ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True
|
||||
)
|
||||
|
||||
|
||||
class FileConnectorPolicy(Base, TimestampMixin):
|
||||
__tablename__ = "file_connector_policies"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("tenant_id", "scope_type", "scope_id", name="uq_file_connector_policies_scope"),
|
||||
UniqueConstraint(
|
||||
"tenant_id",
|
||||
"scope_type",
|
||||
"scope_id",
|
||||
name="uq_file_connector_policies_scope",
|
||||
),
|
||||
Index("ix_file_connector_policies_scope", "scope_type", "scope_id"),
|
||||
)
|
||||
|
||||
@@ -176,24 +450,38 @@ class FileConnectorPolicy(Base, TimestampMixin):
|
||||
scope_type: Mapped[str] = mapped_column(String(20), nullable=False, index=True)
|
||||
scope_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
|
||||
policy: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict, nullable=False)
|
||||
created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
|
||||
updated_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
|
||||
created_by_user_id: Mapped[str | None] = mapped_column(
|
||||
ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True
|
||||
)
|
||||
updated_by_user_id: Mapped[str | None] = mapped_column(
|
||||
ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True
|
||||
)
|
||||
|
||||
|
||||
class FileConnectorSpace(Base, TimestampMixin):
|
||||
__tablename__ = "file_connector_spaces"
|
||||
__table_args__ = (
|
||||
Index("ix_file_connector_spaces_owner", "tenant_id", "owner_type", "owner_user_id", "owner_group_id"),
|
||||
Index(
|
||||
"ix_file_connector_spaces_owner",
|
||||
"tenant_id",
|
||||
"owner_type",
|
||||
"owner_user_id",
|
||||
"owner_group_id",
|
||||
),
|
||||
Index(
|
||||
"uq_file_connector_spaces_active_user_label",
|
||||
"tenant_id", "owner_user_id", "label",
|
||||
"tenant_id",
|
||||
"owner_user_id",
|
||||
"label",
|
||||
unique=True,
|
||||
sqlite_where=text("owner_type = 'user' AND deleted_at IS NULL"),
|
||||
postgresql_where=text("owner_type = 'user' AND deleted_at IS NULL"),
|
||||
),
|
||||
Index(
|
||||
"uq_file_connector_spaces_active_group_label",
|
||||
"tenant_id", "owner_group_id", "label",
|
||||
"tenant_id",
|
||||
"owner_group_id",
|
||||
"label",
|
||||
unique=True,
|
||||
sqlite_where=text("owner_type = 'group' AND deleted_at IS NULL"),
|
||||
postgresql_where=text("owner_type = 'group' AND deleted_at IS NULL"),
|
||||
@@ -203,41 +491,81 @@ class FileConnectorSpace(Base, TimestampMixin):
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||
owner_type: Mapped[str] = mapped_column(String(20), nullable=False, index=True)
|
||||
owner_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
|
||||
owner_group_id: Mapped[str | None] = mapped_column(ForeignKey("access_groups.id", ondelete="SET NULL"), nullable=True, index=True)
|
||||
owner_user_id: Mapped[str | None] = mapped_column(
|
||||
ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True
|
||||
)
|
||||
owner_group_id: Mapped[str | None] = mapped_column(
|
||||
ForeignKey("access_groups.id", ondelete="SET NULL"), nullable=True, index=True
|
||||
)
|
||||
label: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
connector_profile_id: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
|
||||
connector_profile_id: Mapped[str] = mapped_column(
|
||||
String(255), nullable=False, index=True
|
||||
)
|
||||
provider: Mapped[str] = mapped_column(String(50), nullable=False, index=True)
|
||||
library_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
remote_path: Mapped[str] = mapped_column(String(1000), default="", nullable=False)
|
||||
sync_mode: Mapped[str] = mapped_column(String(30), default="manual", nullable=False)
|
||||
read_only: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False)
|
||||
is_active: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False, index=True)
|
||||
created_by_user_id: Mapped[str | None] = mapped_column(ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True)
|
||||
deleted_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
|
||||
metadata_: Mapped[dict[str, Any] | None] = mapped_column("metadata", JSON, nullable=True)
|
||||
is_active: Mapped[bool] = mapped_column(
|
||||
Boolean, default=True, nullable=False, index=True
|
||||
)
|
||||
created_by_user_id: Mapped[str | None] = mapped_column(
|
||||
ForeignKey("access_users.id", ondelete="SET NULL"), nullable=True, index=True
|
||||
)
|
||||
deleted_at: Mapped[datetime | None] = mapped_column(
|
||||
DateTime(timezone=True), nullable=True, index=True
|
||||
)
|
||||
metadata_: Mapped[dict[str, Any] | None] = mapped_column(
|
||||
"metadata", JSON, nullable=True
|
||||
)
|
||||
|
||||
|
||||
class CampaignAttachmentUse(Base, TimestampMixin):
|
||||
__tablename__ = "campaign_attachment_uses"
|
||||
__table_args__ = (UniqueConstraint("campaign_job_id", "file_version_id", "filename_used", "use_stage", name="uq_campaign_attachment_uses_job_file_stage"),)
|
||||
__table_args__ = (
|
||||
UniqueConstraint(
|
||||
"campaign_job_id",
|
||||
"file_version_id",
|
||||
"filename_used",
|
||||
"use_stage",
|
||||
name="uq_campaign_attachment_uses_job_file_stage",
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||
campaign_id: Mapped[str] = mapped_column(ForeignKey("campaigns.id", ondelete="CASCADE"), nullable=False, index=True)
|
||||
campaign_version_id: Mapped[str] = mapped_column(ForeignKey("campaign_versions.id", ondelete="CASCADE"), nullable=False, index=True)
|
||||
campaign_job_id: Mapped[str | None] = mapped_column(ForeignKey("campaign_jobs.id", ondelete="SET NULL"), nullable=True, index=True)
|
||||
campaign_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("campaigns.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
campaign_version_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("campaign_versions.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
index=True,
|
||||
)
|
||||
campaign_job_id: Mapped[str | None] = mapped_column(
|
||||
ForeignKey("campaign_jobs.id", ondelete="SET NULL"), nullable=True, index=True
|
||||
)
|
||||
entry_index: Mapped[int | None] = mapped_column(Integer)
|
||||
entry_id: Mapped[str | None] = mapped_column(String(255), index=True)
|
||||
file_asset_id: Mapped[str] = mapped_column(ForeignKey("file_assets.id", ondelete="RESTRICT"), nullable=False, index=True)
|
||||
file_version_id: Mapped[str] = mapped_column(ForeignKey("file_versions.id", ondelete="RESTRICT"), nullable=False, index=True)
|
||||
file_blob_id: Mapped[str] = mapped_column(ForeignKey("file_blobs.id", ondelete="RESTRICT"), nullable=False, index=True)
|
||||
file_asset_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("file_assets.id", ondelete="RESTRICT"), nullable=False, index=True
|
||||
)
|
||||
file_version_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("file_versions.id", ondelete="RESTRICT"), nullable=False, index=True
|
||||
)
|
||||
file_blob_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("file_blobs.id", ondelete="RESTRICT"), nullable=False, index=True
|
||||
)
|
||||
filename_used: Mapped[str] = mapped_column(String(500), nullable=False)
|
||||
checksum_sha256: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
size_bytes: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
content_type: Mapped[str | None] = mapped_column(String(255))
|
||||
use_stage: Mapped[str] = mapped_column(String(20), default="built", nullable=False, index=True)
|
||||
used_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
|
||||
use_stage: Mapped[str] = mapped_column(
|
||||
String(20), default="built", nullable=False, index=True
|
||||
)
|
||||
used_at: Mapped[datetime | None] = mapped_column(
|
||||
DateTime(timezone=True), nullable=True, index=True
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
@@ -249,6 +577,7 @@ __all__ = [
|
||||
"FileConnectorProfile",
|
||||
"FileConnectorSpace",
|
||||
"FileFolder",
|
||||
"FileFormEvidenceGrant",
|
||||
"FileShare",
|
||||
"FileVersion",
|
||||
]
|
||||
|
||||
@@ -12,7 +12,7 @@ from govoplan_core.core.modules import (
|
||||
DocumentationLink,
|
||||
DocumentationTopic,
|
||||
)
|
||||
from govoplan_files.backend.storage.archives import ZIP_UPLOAD_MAX_FILES
|
||||
from govoplan_files.backend.storage.archives import ARCHIVE_UPLOAD_MAX_ENTRIES
|
||||
from govoplan_files.backend.storage.access import user_group_ids
|
||||
from govoplan_files.backend.storage.connector_visibility import (
|
||||
connector_profile_usable_for_import,
|
||||
@@ -22,6 +22,9 @@ from govoplan_files.backend.storage.connector_visibility import (
|
||||
|
||||
_DEFAULT_UPLOAD_MAX_BYTES = 50 * 1024 * 1024
|
||||
_DEFAULT_ZIP_MAX_BYTES = 250 * 1024 * 1024
|
||||
_DEFAULT_ARCHIVE_MAX_EXPANDED_BYTES = 2 * 1024 * 1024 * 1024
|
||||
_DEFAULT_ARCHIVE_MAX_EXPANSION_RATIO = 100
|
||||
_DEFAULT_ARCHIVE_PREVIEW_TTL_SECONDS = 30 * 60
|
||||
_FILES_READ_SCOPE = "files:file:read"
|
||||
_FILES_UPLOAD_SCOPE = "files:file:upload"
|
||||
|
||||
@@ -42,11 +45,47 @@ def documentation_topics(
|
||||
"file_upload_zip_max_bytes",
|
||||
default=_DEFAULT_ZIP_MAX_BYTES,
|
||||
)
|
||||
archive_expanded_limit = _configured_positive_int(
|
||||
context.settings,
|
||||
"file_archive_max_expanded_bytes",
|
||||
default=_DEFAULT_ARCHIVE_MAX_EXPANDED_BYTES,
|
||||
)
|
||||
archive_entry_limit = _configured_positive_int(
|
||||
context.settings,
|
||||
"file_archive_max_entries",
|
||||
default=ARCHIVE_UPLOAD_MAX_ENTRIES,
|
||||
)
|
||||
archive_ratio_limit = _configured_positive_int(
|
||||
context.settings,
|
||||
"file_archive_max_expansion_ratio",
|
||||
default=_DEFAULT_ARCHIVE_MAX_EXPANSION_RATIO,
|
||||
)
|
||||
archive_preview_ttl = _configured_positive_int(
|
||||
context.settings,
|
||||
"file_archive_preview_ttl_seconds",
|
||||
default=_DEFAULT_ARCHIVE_PREVIEW_TTL_SECONDS,
|
||||
)
|
||||
topics: list[DocumentationTopic] = []
|
||||
if upload_limit is not None:
|
||||
topics.append(_upload_topic(upload_limit))
|
||||
if upload_limit is not None and zip_limit is not None:
|
||||
topics.append(_zip_topic(upload_limit, zip_limit))
|
||||
if (
|
||||
upload_limit is not None
|
||||
and zip_limit is not None
|
||||
and archive_expanded_limit is not None
|
||||
and archive_entry_limit is not None
|
||||
and archive_ratio_limit is not None
|
||||
and archive_preview_ttl is not None
|
||||
):
|
||||
topics.append(
|
||||
_archive_topic(
|
||||
upload_limit,
|
||||
zip_limit,
|
||||
archive_expanded_limit,
|
||||
archive_entry_limit,
|
||||
archive_ratio_limit,
|
||||
archive_preview_ttl,
|
||||
)
|
||||
)
|
||||
topics.append(_connector_import_topic(context))
|
||||
return tuple(topics)
|
||||
|
||||
@@ -118,19 +157,29 @@ def _upload_topic(max_bytes: int) -> DocumentationTopic:
|
||||
)
|
||||
|
||||
|
||||
def _zip_topic(max_file_bytes: int, max_zip_bytes: int) -> DocumentationTopic:
|
||||
def _archive_topic(
|
||||
max_file_bytes: int,
|
||||
max_archive_request_bytes: int,
|
||||
max_expanded_bytes: int,
|
||||
max_entries: int,
|
||||
max_expansion_ratio: int,
|
||||
preview_ttl_seconds: int,
|
||||
) -> DocumentationTopic:
|
||||
member_limit = _format_byte_limit(max_file_bytes)
|
||||
archive_limit = _format_byte_limit(max_zip_bytes)
|
||||
request_limit = _format_byte_limit(max_archive_request_bytes)
|
||||
expanded_limit = _format_byte_limit(max_expanded_bytes)
|
||||
preview_minutes = max(1, preview_ttl_seconds // 60)
|
||||
return DocumentationTopic(
|
||||
id="files.workflow.upload-and-unpack-zip",
|
||||
title="Upload and unpack a ZIP archive",
|
||||
title="Preview and unpack an archive",
|
||||
summary=(
|
||||
f"Safely unpack up to {ZIP_UPLOAD_MAX_FILES:,} files from a ZIP whose request and extracted total are each limited to {archive_limit}."
|
||||
f"Review and selectively unpack up to {max_entries:,} entries from ZIP or TAR archives before any managed file is created."
|
||||
),
|
||||
body=(
|
||||
f"The current deployment limits the ZIP request and actual extracted total to {archive_limit}, each member to {member_limit}, "
|
||||
f"and the archive to {ZIP_UPLOAD_MAX_FILES:,} non-directory members. Encrypted archives and unsafe member paths are rejected. "
|
||||
"Actual extracted bytes are counted instead of trusting ZIP headers."
|
||||
f"The deployment accepts ZIP, TAR, TAR.GZ, TAR.BZ2, and TAR.XZ requests up to {request_limit}, limits actual expanded data to {expanded_limit}, "
|
||||
f"each member to {member_limit}, the archive to {max_entries:,} entries, and expansion to {max_expansion_ratio}:1. "
|
||||
f"The server-issued preview expires after {preview_minutes} minutes. Password-protected ZIP archives are supported; passwords remain request-only. "
|
||||
"Unsafe paths and special filesystem entries are rejected. Actual extracted bytes are counted instead of trusting archive headers."
|
||||
),
|
||||
layer="configured",
|
||||
documentation_types=("user",),
|
||||
@@ -161,35 +210,42 @@ def _zip_topic(max_file_bytes: int, max_zip_bytes: int) -> DocumentationTopic:
|
||||
"help_contexts": ["files.list"],
|
||||
"prerequisites": [
|
||||
"You may view and upload managed files.",
|
||||
"The archive is not encrypted and fits the current configured limits.",
|
||||
"The archive fits the configured request, expansion, size, and entry limits.",
|
||||
"The destination personal or group space grants this account write access.",
|
||||
],
|
||||
"steps": [
|
||||
"Open Files and choose the managed destination space and folder.",
|
||||
"Enable Unpack ZIP uploads, then choose or drag the ZIP archive.",
|
||||
"Enable Preview and unpack archive, then choose or drag one supported archive.",
|
||||
"Review the discovered entries, supply a ZIP password when required, and select the files or folders to import.",
|
||||
"Resolve every destination conflict explicitly.",
|
||||
"Wait for extraction and finalization to finish before leaving the page.",
|
||||
"Confirm the selection, then wait for extraction and finalization to finish before leaving the page.",
|
||||
],
|
||||
"outcome": "Accepted archive members are stored as separate governed managed assets below the selected folder.",
|
||||
"verification": "Confirm the expected member paths and inspect representative file sizes, checksums, and versions.",
|
||||
"constraints": [
|
||||
{
|
||||
"id": "zip-request-and-total",
|
||||
"label": "Maximum ZIP request and extracted total",
|
||||
"description": f"Both the compressed request and the actual extracted total are limited to {archive_limit}.",
|
||||
"values": [archive_limit],
|
||||
"id": "archive-request-and-total",
|
||||
"label": "Maximum archive request and expanded total",
|
||||
"description": f"The compressed request is limited to {request_limit}; actual expanded data is limited to {expanded_limit}.",
|
||||
"values": [request_limit, expanded_limit],
|
||||
},
|
||||
{
|
||||
"id": "zip-member-size",
|
||||
"id": "archive-member-size",
|
||||
"label": "Maximum extracted member size",
|
||||
"description": f"Each extracted file is limited to {member_limit}.",
|
||||
"values": [member_limit],
|
||||
},
|
||||
{
|
||||
"id": "zip-member-count",
|
||||
"label": "Maximum file count",
|
||||
"description": f"A ZIP may contain at most {ZIP_UPLOAD_MAX_FILES:,} non-directory members.",
|
||||
"values": [f"{ZIP_UPLOAD_MAX_FILES:,} files"],
|
||||
"id": "archive-entry-count",
|
||||
"label": "Maximum entry count",
|
||||
"description": f"An archive may contain at most {max_entries:,} declared entries.",
|
||||
"values": [f"{max_entries:,} entries"],
|
||||
},
|
||||
{
|
||||
"id": "archive-expansion-ratio",
|
||||
"label": "Maximum expansion ratio",
|
||||
"description": f"Declared and actual output may not exceed {max_expansion_ratio} times the compressed request size.",
|
||||
"values": [f"{max_expansion_ratio}:1"],
|
||||
},
|
||||
],
|
||||
"related_topic_ids": [
|
||||
|
||||
@@ -0,0 +1,879 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Sequence
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import inspect, or_
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from govoplan_core.core.dsar import (
|
||||
DsarErasureActionRef,
|
||||
DsarExecutionResultRef,
|
||||
DsarRecordRef,
|
||||
DsarSubjectRef,
|
||||
dsar_capability_name,
|
||||
)
|
||||
from govoplan_files.backend.db.models import (
|
||||
CampaignAttachmentUse,
|
||||
FileAsset,
|
||||
FileConnectorCredential,
|
||||
FileConnectorPolicy,
|
||||
FileConnectorProfile,
|
||||
FileConnectorSpace,
|
||||
FileFolder,
|
||||
FileFormEvidenceGrant,
|
||||
FileIntegrityFinding,
|
||||
FileIntegrityScan,
|
||||
FileShare,
|
||||
FileVersion,
|
||||
)
|
||||
|
||||
|
||||
FILES_DSAR_CAPABILITY = dsar_capability_name("files")
|
||||
_MAX_RECORDS = 5_000
|
||||
|
||||
|
||||
class FilesDsarProvider:
|
||||
provider_id = "files"
|
||||
module_id = "files"
|
||||
|
||||
def search_subject(
|
||||
self,
|
||||
session: object,
|
||||
*,
|
||||
tenant_id: str,
|
||||
subject: DsarSubjectRef,
|
||||
) -> Sequence[DsarRecordRef]:
|
||||
db = _session(session)
|
||||
subject_user_id = _subject_user_id(subject)
|
||||
if subject_user_id is None:
|
||||
return ()
|
||||
|
||||
records: list[DsarRecordRef] = []
|
||||
form_evidence_available = _has_table(db, FileFormEvidenceGrant)
|
||||
campaign_evidence_available = _has_table(db, CampaignAttachmentUse)
|
||||
retention_reasons: dict[str, str | None] = {}
|
||||
|
||||
def retention_reason(asset: FileAsset) -> str | None:
|
||||
if asset.id not in retention_reasons:
|
||||
retention_reasons[asset.id] = _asset_retention_reason(
|
||||
db,
|
||||
asset,
|
||||
form_evidence_available=form_evidence_available,
|
||||
campaign_evidence_available=campaign_evidence_available,
|
||||
)
|
||||
return retention_reasons[asset.id]
|
||||
|
||||
def append(record: DsarRecordRef) -> None:
|
||||
if len(records) >= _MAX_RECORDS:
|
||||
raise ValueError(
|
||||
"Files DSAR match limit exceeded; narrow the subject selectors."
|
||||
)
|
||||
records.append(record)
|
||||
|
||||
assets = _bounded_rows(
|
||||
db.query(FileAsset)
|
||||
.filter(
|
||||
FileAsset.tenant_id == tenant_id,
|
||||
or_(
|
||||
FileAsset.owner_user_id == subject_user_id,
|
||||
FileAsset.created_by_user_id == subject_user_id,
|
||||
),
|
||||
)
|
||||
.order_by(FileAsset.id)
|
||||
)
|
||||
for asset in assets:
|
||||
match_fields = _matching_fields(
|
||||
asset,
|
||||
subject_user_id,
|
||||
("owner_user_id", "created_by_user_id"),
|
||||
)
|
||||
asset_retention_reason = retention_reason(asset)
|
||||
append(
|
||||
_record(
|
||||
"file_asset",
|
||||
asset.id,
|
||||
"managed_file",
|
||||
asset.filename,
|
||||
{
|
||||
"match_fields": match_fields,
|
||||
"owner_type": asset.owner_type,
|
||||
"display_path": asset.display_path,
|
||||
"filename": asset.filename,
|
||||
"description": asset.description,
|
||||
"deleted_at": _iso(asset.deleted_at),
|
||||
"retained_until": _iso(asset.retained_until),
|
||||
"legal_hold": asset.legal_hold,
|
||||
"lifecycle_reason": asset.lifecycle_reason,
|
||||
"lifecycle_revision": asset.lifecycle_revision,
|
||||
},
|
||||
observed_at=asset.updated_at,
|
||||
immutable=asset_retention_reason is not None,
|
||||
retention_reason=asset_retention_reason,
|
||||
source_path=f"/files?file={asset.id}",
|
||||
)
|
||||
)
|
||||
|
||||
version_rows = _bounded_rows(
|
||||
db.query(FileVersion, FileAsset)
|
||||
.join(FileAsset, FileAsset.id == FileVersion.file_asset_id)
|
||||
.filter(
|
||||
FileVersion.tenant_id == tenant_id,
|
||||
FileAsset.tenant_id == tenant_id,
|
||||
or_(
|
||||
FileAsset.owner_user_id == subject_user_id,
|
||||
FileVersion.created_by_user_id == subject_user_id,
|
||||
),
|
||||
)
|
||||
.order_by(FileVersion.id)
|
||||
)
|
||||
for version, asset in version_rows:
|
||||
asset_retention_reason = retention_reason(asset)
|
||||
append(
|
||||
_record(
|
||||
"file_version",
|
||||
version.id,
|
||||
"managed_file_version",
|
||||
version.filename_at_upload,
|
||||
{
|
||||
"match_fields": (
|
||||
["asset.owner_user_id"]
|
||||
if asset.owner_user_id == subject_user_id
|
||||
else []
|
||||
)
|
||||
+ (
|
||||
["created_by_user_id"]
|
||||
if version.created_by_user_id == subject_user_id
|
||||
else []
|
||||
),
|
||||
"file_asset_id": version.file_asset_id,
|
||||
"version_number": version.version_number,
|
||||
"filename_at_upload": version.filename_at_upload,
|
||||
"display_path_at_upload": version.display_path_at_upload,
|
||||
"content_type": version.content_type,
|
||||
"size_bytes": version.size_bytes,
|
||||
"checksum_sha256": version.checksum_sha256,
|
||||
"created_at": _iso(version.created_at),
|
||||
},
|
||||
observed_at=version.updated_at,
|
||||
immutable=asset_retention_reason is not None,
|
||||
retention_reason=asset_retention_reason,
|
||||
source_path=(
|
||||
f"/api/v1/files/{version.file_asset_id}/versions/"
|
||||
f"{version.id}/download"
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
for folder in _bounded_rows(
|
||||
db.query(FileFolder)
|
||||
.filter(
|
||||
FileFolder.tenant_id == tenant_id,
|
||||
or_(
|
||||
FileFolder.owner_user_id == subject_user_id,
|
||||
FileFolder.created_by_user_id == subject_user_id,
|
||||
),
|
||||
)
|
||||
.order_by(FileFolder.id)
|
||||
):
|
||||
append(
|
||||
_record(
|
||||
"file_folder",
|
||||
folder.id,
|
||||
"managed_folder",
|
||||
folder.path,
|
||||
{
|
||||
"match_fields": _matching_fields(
|
||||
folder,
|
||||
subject_user_id,
|
||||
("owner_user_id", "created_by_user_id"),
|
||||
),
|
||||
"owner_type": folder.owner_type,
|
||||
"path": folder.path,
|
||||
"deleted_at": _iso(folder.deleted_at),
|
||||
},
|
||||
observed_at=folder.updated_at,
|
||||
source_path="/files",
|
||||
)
|
||||
)
|
||||
|
||||
for share in _bounded_rows(
|
||||
db.query(FileShare)
|
||||
.filter(
|
||||
FileShare.tenant_id == tenant_id,
|
||||
or_(
|
||||
(FileShare.target_type == "user")
|
||||
& (FileShare.target_id == subject_user_id),
|
||||
FileShare.created_by_user_id == subject_user_id,
|
||||
FileShare.revoked_by_user_id == subject_user_id,
|
||||
),
|
||||
)
|
||||
.order_by(FileShare.id)
|
||||
):
|
||||
target_matches = (
|
||||
share.target_type == "user" and share.target_id == subject_user_id
|
||||
)
|
||||
match_fields = _matching_fields(
|
||||
share,
|
||||
subject_user_id,
|
||||
("created_by_user_id", "revoked_by_user_id"),
|
||||
)
|
||||
if target_matches:
|
||||
match_fields.insert(0, "target_id")
|
||||
append(
|
||||
_record(
|
||||
"file_share",
|
||||
share.id,
|
||||
"file_access_evidence",
|
||||
f"Share for file {share.file_asset_id}",
|
||||
{
|
||||
"match_fields": match_fields,
|
||||
"file_asset_id": share.file_asset_id,
|
||||
"target_type": share.target_type,
|
||||
"target_is_subject": target_matches,
|
||||
"permission": share.permission,
|
||||
"expires_at": _iso(share.expires_at),
|
||||
"revoked_at": _iso(share.revoked_at),
|
||||
},
|
||||
observed_at=share.updated_at,
|
||||
immutable=True,
|
||||
retention_reason=(
|
||||
"File-sharing history is institutional access evidence."
|
||||
),
|
||||
source_path=f"/files?file={share.file_asset_id}",
|
||||
)
|
||||
)
|
||||
|
||||
if form_evidence_available:
|
||||
evidence_rows = _bounded_rows(
|
||||
db.query(FileFormEvidenceGrant)
|
||||
.outerjoin(
|
||||
FileAsset,
|
||||
FileAsset.id == FileFormEvidenceGrant.file_asset_id,
|
||||
)
|
||||
.filter(
|
||||
FileFormEvidenceGrant.tenant_id == tenant_id,
|
||||
or_(
|
||||
FileFormEvidenceGrant.custodian_user_id == subject_user_id,
|
||||
FileAsset.owner_user_id == subject_user_id,
|
||||
),
|
||||
)
|
||||
.order_by(FileFormEvidenceGrant.id)
|
||||
)
|
||||
for grant in evidence_rows:
|
||||
append(
|
||||
_record(
|
||||
"file_form_evidence",
|
||||
grant.id,
|
||||
"form_evidence",
|
||||
f"Form evidence {grant.form_definition_id}",
|
||||
{
|
||||
"match_fields": (
|
||||
["custodian_user_id"]
|
||||
if grant.custodian_user_id == subject_user_id
|
||||
else ["asset.owner_user_id"]
|
||||
),
|
||||
"form_instance_id": grant.form_instance_id,
|
||||
"form_definition_id": grant.form_definition_id,
|
||||
"form_definition_revision": grant.form_definition_revision,
|
||||
"evidence_kind": grant.evidence_kind,
|
||||
"purpose": grant.purpose,
|
||||
"status": grant.status,
|
||||
"expires_at": _iso(grant.expires_at),
|
||||
"file_asset_id": grant.file_asset_id,
|
||||
"file_version_id": grant.file_version_id,
|
||||
},
|
||||
observed_at=grant.updated_at,
|
||||
immutable=True,
|
||||
retention_reason=(
|
||||
"Submitted Form attachment evidence follows the owning "
|
||||
"process retention and cannot be erased through Files alone."
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
if campaign_evidence_available:
|
||||
campaign_rows = _bounded_rows(
|
||||
db.query(CampaignAttachmentUse)
|
||||
.join(FileAsset, FileAsset.id == CampaignAttachmentUse.file_asset_id)
|
||||
.filter(
|
||||
CampaignAttachmentUse.tenant_id == tenant_id,
|
||||
FileAsset.owner_user_id == subject_user_id,
|
||||
)
|
||||
.order_by(CampaignAttachmentUse.id)
|
||||
)
|
||||
for use in campaign_rows:
|
||||
append(
|
||||
_record(
|
||||
"campaign_attachment_use",
|
||||
use.id,
|
||||
"delivery_evidence",
|
||||
use.filename_used,
|
||||
{
|
||||
"match_fields": ["asset.owner_user_id"],
|
||||
"campaign_id": use.campaign_id,
|
||||
"campaign_version_id": use.campaign_version_id,
|
||||
"campaign_job_id": use.campaign_job_id,
|
||||
"file_asset_id": use.file_asset_id,
|
||||
"file_version_id": use.file_version_id,
|
||||
"filename_used": use.filename_used,
|
||||
"checksum_sha256": use.checksum_sha256,
|
||||
"size_bytes": use.size_bytes,
|
||||
"use_stage": use.use_stage,
|
||||
"used_at": _iso(use.used_at),
|
||||
},
|
||||
observed_at=use.updated_at,
|
||||
immutable=True,
|
||||
retention_reason=(
|
||||
"Campaign attachment use is immutable delivery evidence."
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
self._append_configuration_references(
|
||||
db,
|
||||
append=append,
|
||||
tenant_id=tenant_id,
|
||||
subject_user_id=subject_user_id,
|
||||
)
|
||||
self._append_integrity_references(
|
||||
db,
|
||||
append=append,
|
||||
tenant_id=tenant_id,
|
||||
subject_user_id=subject_user_id,
|
||||
)
|
||||
return tuple(records)
|
||||
|
||||
def plan_erasure(
|
||||
self,
|
||||
session: object,
|
||||
*,
|
||||
tenant_id: str,
|
||||
subject: DsarSubjectRef,
|
||||
records: Sequence[DsarRecordRef],
|
||||
) -> Sequence[DsarErasureActionRef]:
|
||||
del session
|
||||
subject_user_id = _subject_user_id(subject)
|
||||
if subject_user_id is None:
|
||||
return ()
|
||||
actions: list[DsarErasureActionRef] = []
|
||||
for record in records:
|
||||
if record.provider_id != self.provider_id or record.module_id != self.module_id:
|
||||
raise ValueError("Files DSAR received a foreign provider record.")
|
||||
match_fields = {
|
||||
str(value) for value in record.data.get("match_fields", ())
|
||||
}
|
||||
if record.immutable_evidence:
|
||||
actions.append(
|
||||
_action(
|
||||
f"files:retain:{record.resource_type}:{record.resource_id}",
|
||||
"retain",
|
||||
record,
|
||||
f"Retain {record.title}",
|
||||
record.retention_reason
|
||||
or "Institutional evidence must be retained.",
|
||||
executable=False,
|
||||
)
|
||||
)
|
||||
elif record.resource_type in {
|
||||
"file_asset",
|
||||
"file_version",
|
||||
"file_folder",
|
||||
}:
|
||||
actions.append(
|
||||
_action(
|
||||
f"files:review:{record.resource_type}:{record.resource_id}",
|
||||
"manual_review",
|
||||
record,
|
||||
f"Review {record.title}",
|
||||
(
|
||||
"Managed file content, names, paths, ownership, and shared "
|
||||
"references require a case decision. Approved byte erasure "
|
||||
"must use the separate Files purge workflow."
|
||||
),
|
||||
executable=False,
|
||||
)
|
||||
)
|
||||
if (
|
||||
record.resource_type == "file_share"
|
||||
and "target_id" in match_fields
|
||||
and record.data.get("revoked_at") is None
|
||||
):
|
||||
actions.append(
|
||||
_action(
|
||||
f"files:revoke:file_share:{record.resource_id}",
|
||||
"revoke",
|
||||
record,
|
||||
"Revoke active file share",
|
||||
"The active user-targeted share can be revoked without deleting file evidence.",
|
||||
executable=True,
|
||||
metadata={"subject_user_id": subject_user_id},
|
||||
)
|
||||
)
|
||||
for field_name in sorted(
|
||||
match_fields.intersection(_detachable_fields(record.resource_type))
|
||||
):
|
||||
actions.append(
|
||||
_action(
|
||||
(
|
||||
f"files:detach:{record.resource_type}:"
|
||||
f"{field_name}:{record.resource_id}"
|
||||
),
|
||||
"detach",
|
||||
record,
|
||||
f"Detach {field_name.replace('_', ' ')}",
|
||||
(
|
||||
"Remove the mutable subject reference while preserving the "
|
||||
"governed resource and DSAR evidence."
|
||||
),
|
||||
executable=True,
|
||||
metadata={
|
||||
"field": field_name,
|
||||
"subject_user_id": subject_user_id,
|
||||
},
|
||||
)
|
||||
)
|
||||
action_ids = [action.action_id for action in actions]
|
||||
if len(action_ids) != len(set(action_ids)):
|
||||
raise ValueError("Files DSAR produced duplicate action ids.")
|
||||
return tuple(actions)
|
||||
|
||||
def execute_erasure(
|
||||
self,
|
||||
session: object,
|
||||
*,
|
||||
tenant_id: str,
|
||||
subject: DsarSubjectRef,
|
||||
actions: Sequence[DsarErasureActionRef],
|
||||
request_id: str,
|
||||
) -> Sequence[DsarExecutionResultRef]:
|
||||
db = _session(session)
|
||||
subject_user_id = _subject_user_id(subject)
|
||||
if subject_user_id is None:
|
||||
return tuple(
|
||||
_blocked(action, "Files requires a direct membership subject reference.")
|
||||
for action in actions
|
||||
)
|
||||
results: list[DsarExecutionResultRef] = []
|
||||
for action in actions:
|
||||
if (
|
||||
action.provider_id != self.provider_id
|
||||
or action.module_id != self.module_id
|
||||
or action.metadata.get("subject_user_id") != subject_user_id
|
||||
):
|
||||
results.append(_blocked(action, "The Files DSAR action is stale or invalid."))
|
||||
continue
|
||||
if action.action_id.startswith("files:revoke:file_share:"):
|
||||
results.append(
|
||||
_revoke_share(
|
||||
db,
|
||||
tenant_id=tenant_id,
|
||||
subject_user_id=subject_user_id,
|
||||
action=action,
|
||||
request_id=request_id,
|
||||
)
|
||||
)
|
||||
elif action.action_id.startswith("files:detach:"):
|
||||
results.append(
|
||||
_detach_reference(
|
||||
db,
|
||||
tenant_id=tenant_id,
|
||||
subject_user_id=subject_user_id,
|
||||
action=action,
|
||||
request_id=request_id,
|
||||
)
|
||||
)
|
||||
else:
|
||||
results.append(_blocked(action, "Files does not execute this action kind."))
|
||||
db.flush()
|
||||
return tuple(results)
|
||||
|
||||
def _append_configuration_references(
|
||||
self,
|
||||
db: Session,
|
||||
*,
|
||||
append: object,
|
||||
tenant_id: str,
|
||||
subject_user_id: str,
|
||||
) -> None:
|
||||
configurations = (
|
||||
(FileConnectorProfile, "connector_profile", ("created_by_user_id", "updated_by_user_id")),
|
||||
(FileConnectorCredential, "connector_credential", ("created_by_user_id", "updated_by_user_id")),
|
||||
(FileConnectorPolicy, "connector_policy", ("created_by_user_id", "updated_by_user_id")),
|
||||
(FileConnectorSpace, "connector_space", ("owner_user_id", "created_by_user_id")),
|
||||
)
|
||||
for model, resource_type, fields in configurations:
|
||||
if not _has_table(db, model):
|
||||
continue
|
||||
conditions = [getattr(model, field) == subject_user_id for field in fields]
|
||||
query = db.query(model).filter(or_(*conditions))
|
||||
if hasattr(model, "tenant_id"):
|
||||
query = query.filter(model.tenant_id == tenant_id)
|
||||
for row in _bounded_rows(query.order_by(model.id)):
|
||||
match_fields = _matching_fields(row, subject_user_id, fields)
|
||||
data: dict[str, object] = {
|
||||
"match_fields": match_fields,
|
||||
"label": getattr(row, "label", None),
|
||||
"provider": getattr(row, "provider", None),
|
||||
}
|
||||
if isinstance(row, FileConnectorCredential):
|
||||
data["credential_mode"] = row.credential_mode
|
||||
elif isinstance(row, FileConnectorSpace):
|
||||
data.update(
|
||||
{
|
||||
"remote_path": row.remote_path,
|
||||
"sync_mode": row.sync_mode,
|
||||
"read_only": row.read_only,
|
||||
"deleted_at": _iso(row.deleted_at),
|
||||
}
|
||||
)
|
||||
append( # type: ignore[operator]
|
||||
_record(
|
||||
resource_type,
|
||||
row.id,
|
||||
"connector_configuration_evidence",
|
||||
getattr(row, "label", None) or resource_type.replace("_", " "),
|
||||
data,
|
||||
observed_at=row.updated_at,
|
||||
immutable=True,
|
||||
retention_reason=(
|
||||
"Connector configuration history is institutional evidence; "
|
||||
"credential secrets are excluded from the DSAR export."
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
def _append_integrity_references(
|
||||
self,
|
||||
db: Session,
|
||||
*,
|
||||
append: object,
|
||||
tenant_id: str,
|
||||
subject_user_id: str,
|
||||
) -> None:
|
||||
if _has_table(db, FileIntegrityScan):
|
||||
for scan in _bounded_rows(
|
||||
db.query(FileIntegrityScan)
|
||||
.filter(
|
||||
FileIntegrityScan.tenant_id == tenant_id,
|
||||
FileIntegrityScan.created_by_user_id == subject_user_id,
|
||||
)
|
||||
.order_by(FileIntegrityScan.id)
|
||||
):
|
||||
append( # type: ignore[operator]
|
||||
_record(
|
||||
"file_integrity_scan",
|
||||
scan.id,
|
||||
"storage_integrity_evidence",
|
||||
f"Integrity scan {scan.id}",
|
||||
{
|
||||
"match_fields": ["created_by_user_id"],
|
||||
"storage_backend": scan.storage_backend,
|
||||
"status": scan.status,
|
||||
"started_at": _iso(scan.started_at),
|
||||
"completed_at": _iso(scan.completed_at),
|
||||
},
|
||||
observed_at=scan.updated_at,
|
||||
immutable=True,
|
||||
retention_reason="Storage integrity scans are operator evidence.",
|
||||
)
|
||||
)
|
||||
if _has_table(db, FileIntegrityFinding):
|
||||
for finding in _bounded_rows(
|
||||
db.query(FileIntegrityFinding)
|
||||
.filter(
|
||||
FileIntegrityFinding.tenant_id == tenant_id,
|
||||
FileIntegrityFinding.resolved_by_user_id == subject_user_id,
|
||||
)
|
||||
.order_by(FileIntegrityFinding.id)
|
||||
):
|
||||
append( # type: ignore[operator]
|
||||
_record(
|
||||
"file_integrity_finding",
|
||||
finding.id,
|
||||
"storage_integrity_evidence",
|
||||
f"Integrity finding {finding.kind}",
|
||||
{
|
||||
"match_fields": ["resolved_by_user_id"],
|
||||
"kind": finding.kind,
|
||||
"state": finding.state,
|
||||
"resolved_at": _iso(finding.resolved_at),
|
||||
},
|
||||
observed_at=finding.updated_at,
|
||||
immutable=True,
|
||||
retention_reason="Integrity resolution is operator evidence.",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _asset_retention_reason(
|
||||
session: Session,
|
||||
asset: FileAsset,
|
||||
*,
|
||||
form_evidence_available: bool,
|
||||
campaign_evidence_available: bool,
|
||||
) -> str | None:
|
||||
reasons: list[str] = []
|
||||
if asset.legal_hold:
|
||||
reasons.append("The file is under legal hold.")
|
||||
retained_until = _aware(asset.retained_until)
|
||||
if retained_until is not None and retained_until > datetime.now(timezone.utc):
|
||||
reasons.append(f"The file is retained until {retained_until.isoformat()}.")
|
||||
if form_evidence_available:
|
||||
has_form_evidence = (
|
||||
session.query(FileFormEvidenceGrant.id)
|
||||
.filter(
|
||||
FileFormEvidenceGrant.tenant_id == asset.tenant_id,
|
||||
FileFormEvidenceGrant.file_asset_id == asset.id,
|
||||
)
|
||||
.first()
|
||||
is not None
|
||||
)
|
||||
if has_form_evidence:
|
||||
reasons.append("The file is referenced by submitted Form evidence.")
|
||||
if campaign_evidence_available:
|
||||
has_campaign_evidence = (
|
||||
session.query(CampaignAttachmentUse.id)
|
||||
.filter(
|
||||
CampaignAttachmentUse.tenant_id == asset.tenant_id,
|
||||
CampaignAttachmentUse.file_asset_id == asset.id,
|
||||
)
|
||||
.first()
|
||||
is not None
|
||||
)
|
||||
if has_campaign_evidence:
|
||||
reasons.append("The file is referenced by Campaign delivery evidence.")
|
||||
return " ".join(reasons) or None
|
||||
|
||||
|
||||
_DETACHABLE_MODELS: dict[str, tuple[type[object], frozenset[str]]] = {
|
||||
"file_asset": (FileAsset, frozenset({"created_by_user_id"})),
|
||||
"file_version": (FileVersion, frozenset({"created_by_user_id"})),
|
||||
"file_folder": (FileFolder, frozenset({"created_by_user_id"})),
|
||||
"file_share": (
|
||||
FileShare,
|
||||
frozenset({"created_by_user_id", "revoked_by_user_id"}),
|
||||
),
|
||||
"connector_profile": (
|
||||
FileConnectorProfile,
|
||||
frozenset({"created_by_user_id", "updated_by_user_id"}),
|
||||
),
|
||||
"connector_credential": (
|
||||
FileConnectorCredential,
|
||||
frozenset({"created_by_user_id", "updated_by_user_id"}),
|
||||
),
|
||||
"connector_policy": (
|
||||
FileConnectorPolicy,
|
||||
frozenset({"created_by_user_id", "updated_by_user_id"}),
|
||||
),
|
||||
"connector_space": (FileConnectorSpace, frozenset({"created_by_user_id"})),
|
||||
}
|
||||
|
||||
|
||||
def _detachable_fields(resource_type: str) -> frozenset[str]:
|
||||
entry = _DETACHABLE_MODELS.get(resource_type)
|
||||
return entry[1] if entry is not None else frozenset()
|
||||
|
||||
|
||||
def _revoke_share(
|
||||
session: Session,
|
||||
*,
|
||||
tenant_id: str,
|
||||
subject_user_id: str,
|
||||
action: DsarErasureActionRef,
|
||||
request_id: str,
|
||||
) -> DsarExecutionResultRef:
|
||||
row = (
|
||||
session.query(FileShare)
|
||||
.filter(FileShare.id == action.resource_id)
|
||||
.with_for_update()
|
||||
.one_or_none()
|
||||
)
|
||||
if (
|
||||
row is None
|
||||
or row.tenant_id != tenant_id
|
||||
or row.target_type != "user"
|
||||
or row.target_id != subject_user_id
|
||||
):
|
||||
return _blocked(action, "The subject-targeted file share is no longer available.")
|
||||
if row.revoked_at is not None:
|
||||
return _result(
|
||||
action,
|
||||
"unchanged",
|
||||
"The file share was already revoked.",
|
||||
{"request_id": request_id, "revoked_at": _iso(row.revoked_at)},
|
||||
)
|
||||
row.revoked_at = datetime.now(timezone.utc)
|
||||
row.revoked_by_user_id = None
|
||||
return _result(
|
||||
action,
|
||||
"executed",
|
||||
"The subject-targeted file share was revoked.",
|
||||
{"request_id": request_id, "revoked_at": _iso(row.revoked_at)},
|
||||
)
|
||||
|
||||
|
||||
def _detach_reference(
|
||||
session: Session,
|
||||
*,
|
||||
tenant_id: str,
|
||||
subject_user_id: str,
|
||||
action: DsarErasureActionRef,
|
||||
request_id: str,
|
||||
) -> DsarExecutionResultRef:
|
||||
entry = _DETACHABLE_MODELS.get(action.resource_type)
|
||||
field_name = str(action.metadata.get("field") or "")
|
||||
if entry is None or field_name not in entry[1]:
|
||||
return _blocked(action, "The requested Files subject reference is not detachable.")
|
||||
model = entry[0]
|
||||
row = (
|
||||
session.query(model)
|
||||
.filter(getattr(model, "id") == action.resource_id)
|
||||
.with_for_update()
|
||||
.one_or_none()
|
||||
)
|
||||
if row is None or getattr(row, "tenant_id", None) != tenant_id:
|
||||
return _blocked(action, "The Files resource is no longer available.")
|
||||
current = getattr(row, field_name)
|
||||
if current is None:
|
||||
return _result(
|
||||
action,
|
||||
"unchanged",
|
||||
"The subject reference was already detached.",
|
||||
{"request_id": request_id, "field": field_name},
|
||||
)
|
||||
if current != subject_user_id:
|
||||
return _blocked(action, "The Files subject reference changed after planning.")
|
||||
setattr(row, field_name, None)
|
||||
return _result(
|
||||
action,
|
||||
"executed",
|
||||
"The mutable subject reference was detached.",
|
||||
{"request_id": request_id, "field": field_name},
|
||||
)
|
||||
|
||||
|
||||
def _subject_user_id(subject: DsarSubjectRef) -> str | None:
|
||||
candidates: list[str] = []
|
||||
if subject.membership_id:
|
||||
candidates.append(subject.membership_id)
|
||||
for key, value in subject.external_references.items():
|
||||
if key in {
|
||||
"files.user",
|
||||
"files.membership",
|
||||
"access.membership",
|
||||
"membership_id",
|
||||
}:
|
||||
candidates.append(value)
|
||||
normalized = {value.strip() for value in candidates if value.strip()}
|
||||
if len(normalized) != 1:
|
||||
return None
|
||||
return normalized.pop()
|
||||
|
||||
|
||||
def _matching_fields(
|
||||
row: object,
|
||||
subject_user_id: str,
|
||||
fields: Sequence[str],
|
||||
) -> list[str]:
|
||||
return [field for field in fields if getattr(row, field) == subject_user_id]
|
||||
|
||||
|
||||
def _record(
|
||||
resource_type: str,
|
||||
resource_id: str,
|
||||
category: str,
|
||||
title: str,
|
||||
data: dict[str, object],
|
||||
*,
|
||||
observed_at: datetime | None = None,
|
||||
immutable: bool = False,
|
||||
retention_reason: str | None = None,
|
||||
source_path: str | None = None,
|
||||
) -> DsarRecordRef:
|
||||
return DsarRecordRef(
|
||||
provider_id="files",
|
||||
module_id="files",
|
||||
resource_type=resource_type,
|
||||
resource_id=resource_id,
|
||||
category=category,
|
||||
title=title,
|
||||
data=data,
|
||||
observed_at=observed_at,
|
||||
immutable_evidence=immutable,
|
||||
retention_reason=retention_reason,
|
||||
source_path=source_path,
|
||||
)
|
||||
|
||||
|
||||
def _action(
|
||||
action_id: str,
|
||||
kind: str,
|
||||
record: DsarRecordRef,
|
||||
title: str,
|
||||
rationale: str,
|
||||
*,
|
||||
executable: bool,
|
||||
metadata: dict[str, object] | None = None,
|
||||
) -> DsarErasureActionRef:
|
||||
return DsarErasureActionRef(
|
||||
action_id=action_id,
|
||||
provider_id="files",
|
||||
module_id="files",
|
||||
kind=kind, # type: ignore[arg-type]
|
||||
resource_type=record.resource_type,
|
||||
resource_id=record.resource_id,
|
||||
title=title,
|
||||
rationale=rationale,
|
||||
executable=executable,
|
||||
metadata=metadata or {},
|
||||
)
|
||||
|
||||
|
||||
def _result(
|
||||
action: DsarErasureActionRef,
|
||||
status: str,
|
||||
summary: str,
|
||||
evidence: dict[str, object] | None = None,
|
||||
) -> DsarExecutionResultRef:
|
||||
return DsarExecutionResultRef(
|
||||
action_id=action.action_id,
|
||||
status=status, # type: ignore[arg-type]
|
||||
summary=summary,
|
||||
evidence=evidence or {},
|
||||
)
|
||||
|
||||
|
||||
def _blocked(action: DsarErasureActionRef, summary: str) -> DsarExecutionResultRef:
|
||||
return _result(action, "blocked", summary)
|
||||
|
||||
|
||||
def _session(value: object) -> Session:
|
||||
if not isinstance(value, Session):
|
||||
raise TypeError("Files DSAR provider requires a SQLAlchemy session.")
|
||||
return value
|
||||
|
||||
|
||||
def _bounded_rows(query: object) -> list[object]:
|
||||
rows = query.limit(_MAX_RECORDS + 1).all() # type: ignore[attr-defined]
|
||||
if len(rows) > _MAX_RECORDS:
|
||||
raise ValueError("Files DSAR match limit exceeded; narrow the subject selectors.")
|
||||
return rows
|
||||
|
||||
|
||||
def _has_table(session: Session, model: type[object]) -> bool:
|
||||
return inspect(session.connection()).has_table(model.__tablename__)
|
||||
|
||||
|
||||
def _aware(value: datetime | None) -> datetime | None:
|
||||
if value is None or value.tzinfo is not None:
|
||||
return value
|
||||
return value.replace(tzinfo=timezone.utc)
|
||||
|
||||
|
||||
def _iso(value: datetime | None) -> str | None:
|
||||
aware = _aware(value)
|
||||
return aware.isoformat() if aware else None
|
||||
|
||||
|
||||
__all__ = ["FILES_DSAR_CAPABILITY", "FilesDsarProvider"]
|
||||
@@ -0,0 +1,449 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping, Sequence
|
||||
from datetime import UTC, datetime, timedelta
|
||||
import hashlib
|
||||
import json
|
||||
import secrets
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from govoplan_core.core.access import CAPABILITY_ACCESS_DIRECTORY, AccessDirectory
|
||||
from govoplan_core.core.form_evidence import (
|
||||
FormEvidenceContractError,
|
||||
FormEvidenceGrant,
|
||||
FormEvidenceGrantRequest,
|
||||
FormEvidenceInspection,
|
||||
FormEvidenceInspectionRequest,
|
||||
FormEvidenceState,
|
||||
)
|
||||
from govoplan_core.core.institutional import EvidenceReference
|
||||
from govoplan_core.core.modules import ModuleContext
|
||||
from govoplan_files.backend.db.models import (
|
||||
FileAsset,
|
||||
FileBlob,
|
||||
FileFormEvidenceGrant,
|
||||
FileVersion,
|
||||
)
|
||||
from govoplan_files.backend.runtime import configure_runtime
|
||||
|
||||
|
||||
PROVIDER_ID = "files"
|
||||
CAPABILITY_FORM_EVIDENCE_FILES = "forms_runtime.evidence.files"
|
||||
MAX_GRANT_TTL = timedelta(minutes=15)
|
||||
|
||||
|
||||
class FilesFormEvidenceProvider:
|
||||
provider_id = PROVIDER_ID
|
||||
|
||||
def __init__(self, registry: object | None, settings: object) -> None:
|
||||
self._registry = registry
|
||||
self._settings = settings
|
||||
|
||||
def supported_kinds(self) -> Sequence[str]:
|
||||
return ("document",)
|
||||
|
||||
def create_upload_grant(
|
||||
self,
|
||||
session: object,
|
||||
principal: object,
|
||||
*,
|
||||
request: FormEvidenceGrantRequest,
|
||||
) -> FormEvidenceGrant:
|
||||
db = _session(session)
|
||||
_assert_tenant(principal, request.tenant_id)
|
||||
if request.evidence_kind != "document":
|
||||
raise FormEvidenceContractError(
|
||||
"Files can accept only document evidence for Forms Runtime."
|
||||
)
|
||||
custodian_user_id = _custodian_user_id(request.custodian_ref)
|
||||
self._assert_active_custodian(
|
||||
tenant_id=request.tenant_id,
|
||||
user_id=custodian_user_id,
|
||||
)
|
||||
now = datetime.now(UTC)
|
||||
if request.expires_at <= now:
|
||||
raise FormEvidenceContractError(
|
||||
"Form evidence upload grant expiry must be in the future."
|
||||
)
|
||||
allowed_content_types = _content_types(request.allowed_content_types)
|
||||
configured_max = int(getattr(self._settings, "file_upload_max_bytes"))
|
||||
max_size_bytes = min(request.max_size_bytes or configured_max, configured_max)
|
||||
request_sha256 = _request_sha256(
|
||||
request,
|
||||
custodian_user_id=custodian_user_id,
|
||||
max_size_bytes=max_size_bytes,
|
||||
allowed_content_types=allowed_content_types,
|
||||
)
|
||||
existing = (
|
||||
db.query(FileFormEvidenceGrant)
|
||||
.filter(
|
||||
FileFormEvidenceGrant.tenant_id == request.tenant_id,
|
||||
FileFormEvidenceGrant.idempotency_key == request.idempotency_key,
|
||||
)
|
||||
.one_or_none()
|
||||
)
|
||||
if existing is not None:
|
||||
if not secrets.compare_digest(existing.request_sha256, request_sha256):
|
||||
raise FormEvidenceContractError(
|
||||
"Form evidence grant idempotency conflict."
|
||||
)
|
||||
if (
|
||||
existing.status in {"expired", "revoked"}
|
||||
or _aware(existing.expires_at) <= now
|
||||
):
|
||||
raise FormEvidenceContractError(
|
||||
"The existing Form evidence upload grant is no longer usable; "
|
||||
"request a new grant with a new idempotency key."
|
||||
)
|
||||
return _grant_response(existing, upload_token=None, replayed=True)
|
||||
|
||||
metadata = _bounded_metadata(request.metadata)
|
||||
remaining_attachments = metadata.get("remaining_attachments")
|
||||
if isinstance(remaining_attachments, int):
|
||||
existing_attachment_ids = set(metadata.get("existing_attachment_ids", ()))
|
||||
active_grants = (
|
||||
db.query(FileFormEvidenceGrant)
|
||||
.filter(
|
||||
FileFormEvidenceGrant.tenant_id == request.tenant_id,
|
||||
FileFormEvidenceGrant.form_instance_id == request.instance_id,
|
||||
FileFormEvidenceGrant.form_definition_id
|
||||
== request.definition_ref.object_id,
|
||||
FileFormEvidenceGrant.form_definition_revision
|
||||
== str(request.definition_ref.version),
|
||||
FileFormEvidenceGrant.status.in_(("issued", "uploaded")),
|
||||
FileFormEvidenceGrant.expires_at > now,
|
||||
)
|
||||
.with_for_update()
|
||||
.all()
|
||||
)
|
||||
outstanding = sum(
|
||||
1
|
||||
for item in active_grants
|
||||
if not item.file_asset_id
|
||||
or item.file_asset_id not in existing_attachment_ids
|
||||
)
|
||||
if outstanding >= remaining_attachments:
|
||||
raise FormEvidenceContractError(
|
||||
"This Form already has the maximum number of active attachment uploads."
|
||||
)
|
||||
|
||||
upload_token = secrets.token_urlsafe(32)
|
||||
grant = FileFormEvidenceGrant(
|
||||
tenant_id=request.tenant_id,
|
||||
form_instance_id=request.instance_id,
|
||||
form_definition_id=request.definition_ref.object_id,
|
||||
form_definition_revision=str(request.definition_ref.version),
|
||||
token_sha256=_token_sha256(upload_token),
|
||||
idempotency_key=request.idempotency_key,
|
||||
request_sha256=request_sha256,
|
||||
custodian_user_id=custodian_user_id,
|
||||
evidence_kind=request.evidence_kind,
|
||||
purpose=request.purpose,
|
||||
status="issued",
|
||||
expires_at=min(request.expires_at, now + MAX_GRANT_TTL),
|
||||
max_size_bytes=max_size_bytes,
|
||||
allowed_content_types=list(allowed_content_types),
|
||||
metadata_=metadata,
|
||||
)
|
||||
db.add(grant)
|
||||
db.flush()
|
||||
return _grant_response(grant, upload_token=upload_token, replayed=False)
|
||||
|
||||
def inspect_evidence(
|
||||
self,
|
||||
session: object,
|
||||
principal: object,
|
||||
*,
|
||||
request: FormEvidenceInspectionRequest,
|
||||
) -> FormEvidenceInspection:
|
||||
db = _session(session)
|
||||
_assert_tenant(principal, request.tenant_id)
|
||||
now = datetime.now(UTC)
|
||||
if request.evidence.owner_module != PROVIDER_ID:
|
||||
return _inspection(
|
||||
request.evidence,
|
||||
state="rejected",
|
||||
observed_at=now,
|
||||
reason="The evidence owner does not match the Files provider.",
|
||||
)
|
||||
if request.evidence.kind != "document" or not request.evidence.version:
|
||||
return _inspection(
|
||||
request.evidence,
|
||||
state="rejected",
|
||||
observed_at=now,
|
||||
reason="Files evidence requires an exact document version.",
|
||||
)
|
||||
grant = (
|
||||
db.query(FileFormEvidenceGrant)
|
||||
.filter(
|
||||
FileFormEvidenceGrant.tenant_id == request.tenant_id,
|
||||
FileFormEvidenceGrant.form_instance_id == request.instance_id,
|
||||
FileFormEvidenceGrant.form_definition_id
|
||||
== request.definition_ref.object_id,
|
||||
FileFormEvidenceGrant.form_definition_revision
|
||||
== str(request.definition_ref.version),
|
||||
FileFormEvidenceGrant.file_asset_id == request.evidence.evidence_id,
|
||||
FileFormEvidenceGrant.file_version_id == request.evidence.version,
|
||||
)
|
||||
.one_or_none()
|
||||
)
|
||||
if grant is None:
|
||||
return _inspection(
|
||||
request.evidence,
|
||||
state="rejected",
|
||||
observed_at=now,
|
||||
reason="The document was not captured for this exact Form submission.",
|
||||
)
|
||||
if grant.status == "revoked":
|
||||
return _inspection(
|
||||
request.evidence,
|
||||
state="revoked",
|
||||
observed_at=now,
|
||||
reason="The Form evidence grant was revoked.",
|
||||
)
|
||||
if grant.status != "uploaded":
|
||||
return _inspection(
|
||||
request.evidence,
|
||||
state="pending",
|
||||
observed_at=now,
|
||||
retryable=True,
|
||||
reason="The Form evidence upload has not completed.",
|
||||
)
|
||||
asset = db.get(FileAsset, grant.file_asset_id)
|
||||
version = db.get(FileVersion, grant.file_version_id)
|
||||
blob = db.get(FileBlob, version.blob_id) if version is not None else None
|
||||
if asset is None or version is None or blob is None:
|
||||
return _inspection(
|
||||
request.evidence,
|
||||
state="unavailable",
|
||||
observed_at=now,
|
||||
retryable=True,
|
||||
reason="The managed document cannot currently be reconstructed.",
|
||||
)
|
||||
if (
|
||||
asset.tenant_id != request.tenant_id
|
||||
or version.tenant_id != request.tenant_id
|
||||
or blob.tenant_id != request.tenant_id
|
||||
or version.file_asset_id != asset.id
|
||||
):
|
||||
return _inspection(
|
||||
request.evidence,
|
||||
state="rejected",
|
||||
observed_at=now,
|
||||
reason="The managed document crosses an evidence ownership boundary.",
|
||||
)
|
||||
if asset.deleted_at is not None:
|
||||
return _inspection(
|
||||
request.evidence,
|
||||
state="revoked",
|
||||
observed_at=now,
|
||||
reason="The managed document is no longer active.",
|
||||
)
|
||||
if blob.quarantined_at is not None or blob.integrity_status == "quarantined":
|
||||
return _inspection(
|
||||
request.evidence,
|
||||
state="rejected",
|
||||
observed_at=now,
|
||||
reason="The managed document failed its integrity gate.",
|
||||
)
|
||||
if blob.integrity_status != "verified":
|
||||
return _inspection(
|
||||
request.evidence,
|
||||
state="pending",
|
||||
observed_at=now,
|
||||
retryable=True,
|
||||
reason="The managed document is awaiting integrity verification.",
|
||||
)
|
||||
if (
|
||||
not request.evidence.checksum
|
||||
or not secrets.compare_digest(
|
||||
request.evidence.checksum,
|
||||
version.checksum_sha256,
|
||||
)
|
||||
or not secrets.compare_digest(
|
||||
version.checksum_sha256,
|
||||
blob.checksum_sha256,
|
||||
)
|
||||
):
|
||||
return _inspection(
|
||||
request.evidence,
|
||||
state="rejected",
|
||||
observed_at=now,
|
||||
reason="The managed document checksum does not match the evidence.",
|
||||
)
|
||||
return _inspection(
|
||||
request.evidence,
|
||||
state="accepted",
|
||||
observed_at=now,
|
||||
metadata={
|
||||
"content_type": version.content_type,
|
||||
"size_bytes": version.size_bytes,
|
||||
"integrity_status": blob.integrity_status,
|
||||
"grant_id": grant.id,
|
||||
},
|
||||
)
|
||||
|
||||
def _assert_active_custodian(self, *, tenant_id: str, user_id: str) -> None:
|
||||
registry = self._registry
|
||||
if registry is None or not hasattr(registry, "has_capability"):
|
||||
raise FormEvidenceContractError(
|
||||
"The Access directory is unavailable for Form evidence custody."
|
||||
)
|
||||
if not registry.has_capability(CAPABILITY_ACCESS_DIRECTORY):
|
||||
raise FormEvidenceContractError(
|
||||
"The Access directory is unavailable for Form evidence custody."
|
||||
)
|
||||
directory = registry.require_capability(CAPABILITY_ACCESS_DIRECTORY)
|
||||
if not isinstance(directory, AccessDirectory):
|
||||
raise FormEvidenceContractError(
|
||||
"The Access directory capability is invalid."
|
||||
)
|
||||
user = directory.get_user(user_id)
|
||||
if user is None or user.tenant_id != tenant_id or user.status != "active":
|
||||
raise FormEvidenceContractError(
|
||||
"Form evidence requires an active same-tenant custodian."
|
||||
)
|
||||
|
||||
|
||||
def create_files_form_evidence_provider(
|
||||
context: ModuleContext,
|
||||
) -> FilesFormEvidenceProvider:
|
||||
configure_runtime(registry=context.registry, settings=context.settings)
|
||||
return FilesFormEvidenceProvider(context.registry, context.settings)
|
||||
|
||||
|
||||
def _grant_response(
|
||||
grant: FileFormEvidenceGrant,
|
||||
*,
|
||||
upload_token: str | None,
|
||||
replayed: bool,
|
||||
) -> FormEvidenceGrant:
|
||||
return FormEvidenceGrant(
|
||||
provider_id=PROVIDER_ID,
|
||||
grant_id=grant.id,
|
||||
upload_token=upload_token,
|
||||
upload_url="/api/v1/files/form-evidence/upload",
|
||||
expires_at=_aware(grant.expires_at),
|
||||
max_size_bytes=grant.max_size_bytes,
|
||||
allowed_content_types=tuple(grant.allowed_content_types),
|
||||
replayed=replayed,
|
||||
)
|
||||
|
||||
|
||||
def _inspection(
|
||||
reference: EvidenceReference,
|
||||
*,
|
||||
state: FormEvidenceState,
|
||||
observed_at: datetime,
|
||||
retryable: bool = False,
|
||||
reason: str | None = None,
|
||||
metadata: Mapping[str, object] | None = None,
|
||||
) -> FormEvidenceInspection:
|
||||
return FormEvidenceInspection(
|
||||
provider_id=PROVIDER_ID,
|
||||
reference=reference,
|
||||
state=state,
|
||||
observed_at=observed_at,
|
||||
retryable=retryable,
|
||||
reason=reason,
|
||||
metadata=dict(metadata or {}),
|
||||
)
|
||||
|
||||
|
||||
def _session(value: object) -> Session:
|
||||
if not isinstance(value, Session):
|
||||
raise TypeError("Files Form evidence requires a SQLAlchemy session.")
|
||||
return value
|
||||
|
||||
|
||||
def _assert_tenant(principal: object, tenant_id: str) -> None:
|
||||
principal_tenant = str(getattr(principal, "tenant_id", "") or "").strip()
|
||||
if not principal_tenant or principal_tenant != tenant_id:
|
||||
raise PermissionError("Form evidence cannot cross tenants.")
|
||||
|
||||
|
||||
def _custodian_user_id(value: str | None) -> str:
|
||||
clean = str(value or "").strip()
|
||||
if not clean.startswith("user:") or len(clean) <= len("user:"):
|
||||
raise FormEvidenceContractError(
|
||||
"Files Form evidence requires a user custodian."
|
||||
)
|
||||
return clean.removeprefix("user:")
|
||||
|
||||
|
||||
def _content_types(values: Sequence[str]) -> tuple[str, ...]:
|
||||
cleaned = tuple(
|
||||
dict.fromkeys(
|
||||
value.split(";", 1)[0].strip().casefold()
|
||||
for value in values
|
||||
if value.strip()
|
||||
)
|
||||
)
|
||||
if len(cleaned) > 50 or any(len(value) > 255 for value in cleaned):
|
||||
raise FormEvidenceContractError(
|
||||
"Form evidence content-type restrictions are too large."
|
||||
)
|
||||
return cleaned
|
||||
|
||||
|
||||
def _bounded_metadata(value: Mapping[str, object]) -> dict[str, object]:
|
||||
remaining = value.get("remaining_attachments")
|
||||
result: dict[str, object] = {}
|
||||
if isinstance(remaining, int) and remaining >= 0:
|
||||
result["remaining_attachments"] = remaining
|
||||
raw_ids = value.get("existing_attachment_ids")
|
||||
if isinstance(raw_ids, (list, tuple)):
|
||||
clean_ids = tuple(
|
||||
dict.fromkeys(
|
||||
item.strip()
|
||||
for item in raw_ids
|
||||
if isinstance(item, str) and item.strip() and len(item.strip()) <= 255
|
||||
)
|
||||
)
|
||||
if len(clean_ids) <= 1000:
|
||||
result["existing_attachment_ids"] = clean_ids
|
||||
return result
|
||||
|
||||
|
||||
def _request_sha256(
|
||||
request: FormEvidenceGrantRequest,
|
||||
*,
|
||||
custodian_user_id: str,
|
||||
max_size_bytes: int,
|
||||
allowed_content_types: Sequence[str],
|
||||
) -> str:
|
||||
return hashlib.sha256(
|
||||
json.dumps(
|
||||
{
|
||||
"tenant_id": request.tenant_id,
|
||||
"instance_id": request.instance_id,
|
||||
"definition_ref": request.definition_ref.to_dict(),
|
||||
"evidence_kind": request.evidence_kind,
|
||||
"purpose": request.purpose,
|
||||
"expires_at": request.expires_at.isoformat(),
|
||||
"custodian_user_id": custodian_user_id,
|
||||
"max_size_bytes": max_size_bytes,
|
||||
"allowed_content_types": list(allowed_content_types),
|
||||
"metadata": _bounded_metadata(request.metadata),
|
||||
},
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
).encode("utf-8")
|
||||
).hexdigest()
|
||||
|
||||
|
||||
def _token_sha256(token: str) -> str:
|
||||
return hashlib.sha256(token.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _aware(value: datetime) -> datetime:
|
||||
return value if value.tzinfo is not None else value.replace(tzinfo=UTC)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"CAPABILITY_FORM_EVIDENCE_FILES",
|
||||
"FilesFormEvidenceProvider",
|
||||
"PROVIDER_ID",
|
||||
"create_files_form_evidence_provider",
|
||||
]
|
||||
File diff suppressed because it is too large
Load Diff
+147
@@ -0,0 +1,147 @@
|
||||
"""add purpose-bound Form evidence upload grants
|
||||
|
||||
Revision ID: a2b3c4d5e6f8
|
||||
Revises: f1a2b3c4d5e7
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
|
||||
revision = "a2b3c4d5e6f8"
|
||||
down_revision = "f1a2b3c4d5e7"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"file_form_evidence_grants",
|
||||
sa.Column("id", sa.String(length=36), nullable=False),
|
||||
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
||||
sa.Column("form_instance_id", sa.String(length=36), nullable=False),
|
||||
sa.Column("form_definition_id", sa.String(length=255), nullable=False),
|
||||
sa.Column("form_definition_revision", sa.String(length=255), nullable=False),
|
||||
sa.Column("token_sha256", sa.String(length=64), nullable=False),
|
||||
sa.Column("idempotency_key", sa.String(length=255), nullable=False),
|
||||
sa.Column("request_sha256", sa.String(length=64), nullable=False),
|
||||
sa.Column("custodian_user_id", sa.String(length=36), nullable=False),
|
||||
sa.Column("evidence_kind", sa.String(length=30), nullable=False),
|
||||
sa.Column("purpose", sa.String(length=500), nullable=False),
|
||||
sa.Column("status", sa.String(length=30), nullable=False),
|
||||
sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("max_size_bytes", sa.Integer(), nullable=False),
|
||||
sa.Column("allowed_content_types", sa.JSON(), nullable=False),
|
||||
sa.Column("file_asset_id", sa.String(length=36), nullable=True),
|
||||
sa.Column("file_version_id", sa.String(length=36), nullable=True),
|
||||
sa.Column("uploaded_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("revoked_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("metadata", sa.JSON(), nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(
|
||||
["custodian_user_id"],
|
||||
["access_users.id"],
|
||||
ondelete="RESTRICT",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["file_asset_id"],
|
||||
["file_assets.id"],
|
||||
ondelete="RESTRICT",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["file_version_id"],
|
||||
["file_versions.id"],
|
||||
ondelete="RESTRICT",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
sa.UniqueConstraint(
|
||||
"tenant_id",
|
||||
"idempotency_key",
|
||||
name="uq_file_form_evidence_grants_idempotency",
|
||||
),
|
||||
sa.UniqueConstraint("token_sha256"),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_file_form_evidence_grants_tenant_id",
|
||||
"file_form_evidence_grants",
|
||||
["tenant_id"],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_file_form_evidence_grants_form_instance_id",
|
||||
"file_form_evidence_grants",
|
||||
["form_instance_id"],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_file_form_evidence_grants_custodian_user_id",
|
||||
"file_form_evidence_grants",
|
||||
["custodian_user_id"],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_file_form_evidence_grants_status",
|
||||
"file_form_evidence_grants",
|
||||
["status"],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_file_form_evidence_grants_expires_at",
|
||||
"file_form_evidence_grants",
|
||||
["expires_at"],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_file_form_evidence_grants_file_asset_id",
|
||||
"file_form_evidence_grants",
|
||||
["file_asset_id"],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_file_form_evidence_grants_file_version_id",
|
||||
"file_form_evidence_grants",
|
||||
["file_version_id"],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_file_form_evidence_grants_form",
|
||||
"file_form_evidence_grants",
|
||||
[
|
||||
"tenant_id",
|
||||
"form_instance_id",
|
||||
"form_definition_id",
|
||||
"form_definition_revision",
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index(
|
||||
"ix_file_form_evidence_grants_form",
|
||||
table_name="file_form_evidence_grants",
|
||||
)
|
||||
op.drop_index(
|
||||
"ix_file_form_evidence_grants_file_version_id",
|
||||
table_name="file_form_evidence_grants",
|
||||
)
|
||||
op.drop_index(
|
||||
"ix_file_form_evidence_grants_file_asset_id",
|
||||
table_name="file_form_evidence_grants",
|
||||
)
|
||||
op.drop_index(
|
||||
"ix_file_form_evidence_grants_expires_at",
|
||||
table_name="file_form_evidence_grants",
|
||||
)
|
||||
op.drop_index(
|
||||
"ix_file_form_evidence_grants_status",
|
||||
table_name="file_form_evidence_grants",
|
||||
)
|
||||
op.drop_index(
|
||||
"ix_file_form_evidence_grants_custodian_user_id",
|
||||
table_name="file_form_evidence_grants",
|
||||
)
|
||||
op.drop_index(
|
||||
"ix_file_form_evidence_grants_form_instance_id",
|
||||
table_name="file_form_evidence_grants",
|
||||
)
|
||||
op.drop_index(
|
||||
"ix_file_form_evidence_grants_tenant_id",
|
||||
table_name="file_form_evidence_grants",
|
||||
)
|
||||
op.drop_table("file_form_evidence_grants")
|
||||
@@ -0,0 +1,52 @@
|
||||
"""govern managed file retention and legal holds
|
||||
|
||||
Revision ID: a2b3c4d5e6f9
|
||||
Revises: f1a2b3c4d5e7
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
|
||||
revision = "a2b3c4d5e6f9"
|
||||
down_revision = "f1a2b3c4d5e7"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
with op.batch_alter_table("file_assets") as batch_op:
|
||||
batch_op.add_column(
|
||||
sa.Column("retained_until", sa.DateTime(timezone=True), nullable=True)
|
||||
)
|
||||
batch_op.add_column(
|
||||
sa.Column(
|
||||
"legal_hold", sa.Boolean(), nullable=False, server_default=sa.false()
|
||||
)
|
||||
)
|
||||
batch_op.add_column(
|
||||
sa.Column(
|
||||
"lifecycle_revision", sa.Integer(), nullable=False, server_default="1"
|
||||
)
|
||||
)
|
||||
batch_op.add_column(
|
||||
sa.Column("lifecycle_reason", sa.String(length=500), nullable=True)
|
||||
)
|
||||
batch_op.create_index(
|
||||
op.f("ix_file_assets_retained_until"), ["retained_until"], unique=False
|
||||
)
|
||||
batch_op.create_index(
|
||||
op.f("ix_file_assets_legal_hold"), ["legal_hold"], unique=False
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
with op.batch_alter_table("file_assets") as batch_op:
|
||||
batch_op.drop_index(op.f("ix_file_assets_legal_hold"))
|
||||
batch_op.drop_index(op.f("ix_file_assets_retained_until"))
|
||||
batch_op.drop_column("lifecycle_reason")
|
||||
batch_op.drop_column("lifecycle_revision")
|
||||
batch_op.drop_column("legal_hold")
|
||||
batch_op.drop_column("retained_until")
|
||||
@@ -0,0 +1,53 @@
|
||||
"""file share lifecycle
|
||||
|
||||
Revision ID: b8c9d0e1f2a4
|
||||
Revises: a7b8c9d0e1f3
|
||||
Create Date: 2026-07-30 00:00:00.000000
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
revision = "b8c9d0e1f2a4"
|
||||
down_revision = "a7b8c9d0e1f3"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
with op.batch_alter_table("file_shares") as batch_op:
|
||||
batch_op.add_column(
|
||||
sa.Column("expires_at", sa.DateTime(timezone=True), nullable=True)
|
||||
)
|
||||
batch_op.add_column(
|
||||
sa.Column("revoked_by_user_id", sa.String(length=36), nullable=True)
|
||||
)
|
||||
batch_op.create_foreign_key(
|
||||
op.f("fk_file_shares_revoked_by_user_id_access_users"),
|
||||
"access_users",
|
||||
["revoked_by_user_id"],
|
||||
["id"],
|
||||
ondelete="SET NULL",
|
||||
)
|
||||
batch_op.create_index(
|
||||
op.f("ix_file_shares_expires_at"), ["expires_at"], unique=False
|
||||
)
|
||||
batch_op.create_index(
|
||||
op.f("ix_file_shares_revoked_by_user_id"),
|
||||
["revoked_by_user_id"],
|
||||
unique=False,
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
with op.batch_alter_table("file_shares") as batch_op:
|
||||
batch_op.drop_index(op.f("ix_file_shares_revoked_by_user_id"))
|
||||
batch_op.drop_index(op.f("ix_file_shares_expires_at"))
|
||||
batch_op.drop_constraint(
|
||||
op.f("fk_file_shares_revoked_by_user_id_access_users"),
|
||||
type_="foreignkey",
|
||||
)
|
||||
batch_op.drop_column("revoked_by_user_id")
|
||||
batch_op.drop_column("expires_at")
|
||||
+168
@@ -0,0 +1,168 @@
|
||||
"""file integrity reconciliation
|
||||
|
||||
Revision ID: c9d0e1f2a3b5
|
||||
Revises: b8c9d0e1f2a4
|
||||
Create Date: 2026-07-30 00:00:00.000000
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
revision = "c9d0e1f2a3b5"
|
||||
down_revision = "b8c9d0e1f2a4"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
with op.batch_alter_table("file_blobs") as batch_op:
|
||||
batch_op.add_column(
|
||||
sa.Column(
|
||||
"integrity_status",
|
||||
sa.String(length=30),
|
||||
nullable=False,
|
||||
server_default="unchecked",
|
||||
)
|
||||
)
|
||||
batch_op.add_column(
|
||||
sa.Column("integrity_checked_at", sa.DateTime(timezone=True), nullable=True)
|
||||
)
|
||||
batch_op.add_column(
|
||||
sa.Column("integrity_failure", sa.String(length=100), nullable=True)
|
||||
)
|
||||
batch_op.add_column(
|
||||
sa.Column("quarantined_at", sa.DateTime(timezone=True), nullable=True)
|
||||
)
|
||||
batch_op.create_index(
|
||||
op.f("ix_file_blobs_integrity_status"),
|
||||
["integrity_status"],
|
||||
unique=False,
|
||||
)
|
||||
batch_op.create_index(
|
||||
op.f("ix_file_blobs_integrity_checked_at"),
|
||||
["integrity_checked_at"],
|
||||
unique=False,
|
||||
)
|
||||
batch_op.create_index(
|
||||
op.f("ix_file_blobs_quarantined_at"),
|
||||
["quarantined_at"],
|
||||
unique=False,
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"file_integrity_scans",
|
||||
sa.Column("id", sa.String(length=36), nullable=False),
|
||||
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
||||
sa.Column("storage_backend", sa.String(length=50), nullable=False),
|
||||
sa.Column("storage_prefix", sa.String(length=1000), nullable=False),
|
||||
sa.Column("status", sa.String(length=30), nullable=False),
|
||||
sa.Column("phase", sa.String(length=30), nullable=False),
|
||||
sa.Column("verify_checksums", sa.Boolean(), nullable=False),
|
||||
sa.Column("batch_size", sa.Integer(), nullable=False),
|
||||
sa.Column("blob_cursor", sa.String(length=36), nullable=True),
|
||||
sa.Column("object_cursor", sa.String(length=1000), nullable=True),
|
||||
sa.Column("scanned_blob_count", sa.Integer(), nullable=False),
|
||||
sa.Column("verified_blob_count", sa.Integer(), nullable=False),
|
||||
sa.Column("quarantined_blob_count", sa.Integer(), nullable=False),
|
||||
sa.Column("scanned_object_count", sa.Integer(), nullable=False),
|
||||
sa.Column("orphan_object_count", sa.Integer(), nullable=False),
|
||||
sa.Column("created_by_user_id", sa.String(length=36), nullable=True),
|
||||
sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("last_error", sa.String(length=255), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(
|
||||
["created_by_user_id"],
|
||||
["access_users.id"],
|
||||
name=op.f(
|
||||
"fk_file_integrity_scans_created_by_user_id_access_users"
|
||||
),
|
||||
ondelete="SET NULL",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_file_integrity_scans")),
|
||||
)
|
||||
for column in ("tenant_id", "status", "created_by_user_id"):
|
||||
op.create_index(
|
||||
op.f(f"ix_file_integrity_scans_{column}"),
|
||||
"file_integrity_scans",
|
||||
[column],
|
||||
unique=False,
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"file_integrity_findings",
|
||||
sa.Column("id", sa.String(length=36), nullable=False),
|
||||
sa.Column("scan_id", sa.String(length=36), nullable=False),
|
||||
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
||||
sa.Column("kind", sa.String(length=40), nullable=False),
|
||||
sa.Column("state", sa.String(length=30), nullable=False),
|
||||
sa.Column("blob_id", sa.String(length=36), nullable=True),
|
||||
sa.Column("storage_key", sa.String(length=1000), nullable=False),
|
||||
sa.Column("expected_size_bytes", sa.Integer(), nullable=True),
|
||||
sa.Column("observed_size_bytes", sa.Integer(), nullable=True),
|
||||
sa.Column("expected_checksum_sha256", sa.String(length=64), nullable=True),
|
||||
sa.Column("observed_checksum_sha256", sa.String(length=64), nullable=True),
|
||||
sa.Column("resolved_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("resolved_by_user_id", sa.String(length=36), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(
|
||||
["blob_id"],
|
||||
["file_blobs.id"],
|
||||
name=op.f("fk_file_integrity_findings_blob_id_file_blobs"),
|
||||
ondelete="SET NULL",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["resolved_by_user_id"],
|
||||
["access_users.id"],
|
||||
name=op.f(
|
||||
"fk_file_integrity_findings_resolved_by_user_id_access_users"
|
||||
),
|
||||
ondelete="SET NULL",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["scan_id"],
|
||||
["file_integrity_scans.id"],
|
||||
name=op.f(
|
||||
"fk_file_integrity_findings_scan_id_file_integrity_scans"
|
||||
),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_file_integrity_findings")),
|
||||
)
|
||||
for column in (
|
||||
"scan_id",
|
||||
"tenant_id",
|
||||
"kind",
|
||||
"state",
|
||||
"blob_id",
|
||||
"resolved_by_user_id",
|
||||
):
|
||||
op.create_index(
|
||||
op.f(f"ix_file_integrity_findings_{column}"),
|
||||
"file_integrity_findings",
|
||||
[column],
|
||||
unique=False,
|
||||
)
|
||||
op.create_index(
|
||||
"ix_file_integrity_findings_scan_state",
|
||||
"file_integrity_findings",
|
||||
["scan_id", "state"],
|
||||
unique=False,
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table("file_integrity_findings")
|
||||
op.drop_table("file_integrity_scans")
|
||||
with op.batch_alter_table("file_blobs") as batch_op:
|
||||
batch_op.drop_index(op.f("ix_file_blobs_quarantined_at"))
|
||||
batch_op.drop_index(op.f("ix_file_blobs_integrity_checked_at"))
|
||||
batch_op.drop_index(op.f("ix_file_blobs_integrity_status"))
|
||||
batch_op.drop_column("quarantined_at")
|
||||
batch_op.drop_column("integrity_failure")
|
||||
batch_op.drop_column("integrity_checked_at")
|
||||
batch_op.drop_column("integrity_status")
|
||||
+79
@@ -0,0 +1,79 @@
|
||||
"""file content-protection metadata
|
||||
|
||||
Revision ID: d0e1f2a3b4c6
|
||||
Revises: c9d0e1f2a3b5
|
||||
Create Date: 2026-08-02 00:00:00.000000
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
revision = "d0e1f2a3b4c6"
|
||||
down_revision = "c9d0e1f2a3b5"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
with op.batch_alter_table("file_blobs") as batch_op:
|
||||
batch_op.add_column(
|
||||
sa.Column(
|
||||
"protection_discriminator",
|
||||
sa.String(length=320),
|
||||
nullable=False,
|
||||
server_default="plaintext",
|
||||
)
|
||||
)
|
||||
batch_op.add_column(
|
||||
sa.Column("encryption_envelope_id", sa.String(length=255), nullable=True)
|
||||
)
|
||||
batch_op.add_column(
|
||||
sa.Column("storage_checksum_sha256", sa.String(length=64), nullable=True)
|
||||
)
|
||||
batch_op.add_column(
|
||||
sa.Column("storage_size_bytes", sa.Integer(), nullable=True)
|
||||
)
|
||||
batch_op.drop_constraint(
|
||||
"uq_file_blobs_tenant_checksum_size",
|
||||
type_="unique",
|
||||
)
|
||||
batch_op.create_unique_constraint(
|
||||
"uq_file_blobs_tenant_checksum_size_protection",
|
||||
[
|
||||
"tenant_id",
|
||||
"checksum_sha256",
|
||||
"size_bytes",
|
||||
"protection_discriminator",
|
||||
],
|
||||
)
|
||||
batch_op.create_index(
|
||||
op.f("ix_file_blobs_protection_discriminator"),
|
||||
["protection_discriminator"],
|
||||
unique=False,
|
||||
)
|
||||
batch_op.create_index(
|
||||
op.f("ix_file_blobs_encryption_envelope_id"),
|
||||
["encryption_envelope_id"],
|
||||
unique=False,
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
with op.batch_alter_table("file_blobs") as batch_op:
|
||||
batch_op.drop_index(op.f("ix_file_blobs_encryption_envelope_id"))
|
||||
batch_op.drop_index(op.f("ix_file_blobs_protection_discriminator"))
|
||||
batch_op.drop_constraint(
|
||||
"uq_file_blobs_tenant_checksum_size_protection",
|
||||
type_="unique",
|
||||
)
|
||||
batch_op.create_unique_constraint(
|
||||
"uq_file_blobs_tenant_checksum_size",
|
||||
["tenant_id", "checksum_sha256", "size_bytes"],
|
||||
)
|
||||
batch_op.drop_column("storage_size_bytes")
|
||||
batch_op.drop_column("storage_checksum_sha256")
|
||||
batch_op.drop_column("encryption_envelope_id")
|
||||
batch_op.drop_column("protection_discriminator")
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
"""add stale-action revisions to Files integrity operations
|
||||
|
||||
Revision ID: f1a2b3c4d5e7
|
||||
Revises: d0e1f2a3b4c6
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
|
||||
revision = "f1a2b3c4d5e7"
|
||||
down_revision = "d0e1f2a3b4c6"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
with op.batch_alter_table("file_integrity_scans") as batch_op:
|
||||
batch_op.add_column(
|
||||
sa.Column("revision", sa.Integer(), nullable=False, server_default="1")
|
||||
)
|
||||
with op.batch_alter_table("file_integrity_findings") as batch_op:
|
||||
batch_op.add_column(
|
||||
sa.Column("revision", sa.Integer(), nullable=False, server_default="1")
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
with op.batch_alter_table("file_integrity_findings") as batch_op:
|
||||
batch_op.drop_column("revision")
|
||||
with op.batch_alter_table("file_integrity_scans") as batch_op:
|
||||
batch_op.drop_column("revision")
|
||||
@@ -0,0 +1,67 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import secrets
|
||||
from uuid import uuid4
|
||||
|
||||
from govoplan_core.core.operations import OperationalCheck
|
||||
from govoplan_files.backend.storage.backends import (
|
||||
StorageBackendError,
|
||||
get_storage_backend,
|
||||
)
|
||||
|
||||
|
||||
def managed_storage_roundtrip_check() -> OperationalCheck:
|
||||
"""Exercise the configured managed store without retaining probe data."""
|
||||
|
||||
backend = get_storage_backend()
|
||||
key = f".govoplan-health/probes/{uuid4().hex}.bin"
|
||||
payload = secrets.token_bytes(64)
|
||||
expected_digest = hashlib.sha256(payload).hexdigest()
|
||||
delete_error: Exception | None = None
|
||||
try:
|
||||
backend.put_bytes(key, payload, content_type="application/octet-stream")
|
||||
stored = backend.get_bytes(key)
|
||||
info = backend.stat(key)
|
||||
if info.size_bytes != len(payload):
|
||||
raise StorageBackendError("Managed storage returned an unexpected object size")
|
||||
if hashlib.sha256(stored).hexdigest() != expected_digest:
|
||||
raise StorageBackendError("Managed storage returned different bytes than were written")
|
||||
except Exception as exc: # noqa: BLE001 - operational boundary reports provider failures.
|
||||
return OperationalCheck(
|
||||
id="files.managed_storage_roundtrip",
|
||||
label="Managed file storage",
|
||||
state="error",
|
||||
detail=(
|
||||
"The configured managed file store failed a bounded write/read/stat/delete "
|
||||
f"probe ({type(exc).__name__})."
|
||||
),
|
||||
readiness_critical=True,
|
||||
metrics={"backend": backend.name, "probe_bytes": len(payload)},
|
||||
)
|
||||
finally:
|
||||
try:
|
||||
backend.delete(key)
|
||||
except Exception as exc: # noqa: BLE001 - reported below when the data probe passed.
|
||||
delete_error = exc
|
||||
|
||||
if delete_error is not None:
|
||||
return OperationalCheck(
|
||||
id="files.managed_storage_roundtrip",
|
||||
label="Managed file storage",
|
||||
state="error",
|
||||
detail=(
|
||||
"Managed file bytes round-tripped, but probe cleanup failed "
|
||||
f"({type(delete_error).__name__})."
|
||||
),
|
||||
readiness_critical=True,
|
||||
metrics={"backend": backend.name, "probe_bytes": len(payload)},
|
||||
)
|
||||
return OperationalCheck(
|
||||
id="files.managed_storage_roundtrip",
|
||||
label="Managed file storage",
|
||||
state="ok",
|
||||
detail="The configured managed file store passed write, read, stat, integrity, and delete checks.",
|
||||
metrics={"backend": backend.name, "probe_bytes": len(payload)},
|
||||
)
|
||||
|
||||
@@ -0,0 +1,133 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import defaultdict
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import or_, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from govoplan_core.core.provider_governance import (
|
||||
ExternalProviderRuntimeState,
|
||||
ExternalProviderStateContext,
|
||||
)
|
||||
from govoplan_files.backend.db.models import FileConnectorProfile, FileConnectorSpace
|
||||
from govoplan_files.backend.storage.connector_providers import (
|
||||
ConnectorProviderDescriptor,
|
||||
connector_provider_descriptors,
|
||||
)
|
||||
|
||||
|
||||
REMOTE_STORAGE_PROVIDER_ID = "files.remote_storage"
|
||||
|
||||
|
||||
def remote_storage_provider_states(
|
||||
context: ExternalProviderStateContext,
|
||||
) -> tuple[ExternalProviderRuntimeState, ...]:
|
||||
if not isinstance(context.session, Session):
|
||||
raise RuntimeError("Files provider state requires a database session.")
|
||||
statement = select(FileConnectorProfile)
|
||||
if context.tenant_id is not None:
|
||||
statement = statement.where(
|
||||
or_(
|
||||
FileConnectorProfile.tenant_id.is_(None),
|
||||
FileConnectorProfile.tenant_id == context.tenant_id,
|
||||
)
|
||||
)
|
||||
profiles = tuple(
|
||||
context.session.scalars(
|
||||
statement.order_by(
|
||||
FileConnectorProfile.tenant_id,
|
||||
FileConnectorProfile.id,
|
||||
).limit(context.max_items + 1)
|
||||
)
|
||||
)
|
||||
if not profiles:
|
||||
return ()
|
||||
|
||||
profile_ids = tuple(item.id for item in profiles)
|
||||
space_statement = select(FileConnectorSpace).where(
|
||||
FileConnectorSpace.connector_profile_id.in_(profile_ids),
|
||||
FileConnectorSpace.deleted_at.is_(None),
|
||||
)
|
||||
if context.tenant_id is not None:
|
||||
space_statement = space_statement.where(
|
||||
FileConnectorSpace.tenant_id == context.tenant_id
|
||||
)
|
||||
spaces_by_profile: dict[str, list[FileConnectorSpace]] = defaultdict(list)
|
||||
for space in context.session.scalars(space_statement):
|
||||
spaces_by_profile[space.connector_profile_id].append(space)
|
||||
|
||||
descriptors = {
|
||||
item.provider: item for item in connector_provider_descriptors()
|
||||
}
|
||||
observed_at = datetime.now(UTC)
|
||||
return tuple(
|
||||
_profile_state(
|
||||
profile,
|
||||
spaces=spaces_by_profile.get(profile.id, []),
|
||||
descriptor=descriptors.get(profile.provider),
|
||||
observed_at=observed_at,
|
||||
)
|
||||
for profile in profiles
|
||||
)
|
||||
|
||||
|
||||
def _profile_state(
|
||||
profile: FileConnectorProfile,
|
||||
*,
|
||||
spaces: list[FileConnectorSpace],
|
||||
descriptor: ConnectorProviderDescriptor | None,
|
||||
observed_at: datetime,
|
||||
) -> ExternalProviderRuntimeState:
|
||||
active_spaces = tuple(item for item in spaces if item.is_active)
|
||||
active = bool(profile.enabled)
|
||||
implementation_ready = bool(
|
||||
descriptor is not None and descriptor.implemented and descriptor.installed
|
||||
)
|
||||
health = (
|
||||
"inactive"
|
||||
if not active
|
||||
else "error"
|
||||
if not implementation_ready
|
||||
else "unknown"
|
||||
)
|
||||
recovery = (
|
||||
"not_applicable"
|
||||
if not active
|
||||
else "unsupported"
|
||||
if not implementation_ready
|
||||
else "attention"
|
||||
)
|
||||
detail = (
|
||||
"Remote-storage connector profile is disabled."
|
||||
if not active
|
||||
else "The configured provider is not available in this runtime."
|
||||
if not implementation_ready
|
||||
else "Software support is available; no live remote health observation is retained."
|
||||
)
|
||||
return ExternalProviderRuntimeState(
|
||||
provider_id=REMOTE_STORAGE_PROVIDER_ID,
|
||||
binding_ref=f"files:connector-profile:{profile.id}",
|
||||
authority_mode="external_mirror",
|
||||
observed_at=observed_at,
|
||||
configured=True,
|
||||
active=active,
|
||||
health=health,
|
||||
freshness="unknown" if active else "not_applicable",
|
||||
conflict="not_applicable",
|
||||
recovery=recovery,
|
||||
detail=detail,
|
||||
metrics={
|
||||
"provider": profile.provider,
|
||||
"configured_spaces": len(spaces),
|
||||
"active_spaces": len(active_spaces),
|
||||
"write_requested_spaces": sum(
|
||||
1 for item in active_spaces if not item.read_only
|
||||
),
|
||||
"software_implemented": bool(descriptor and descriptor.implemented),
|
||||
"optional_dependency_available": bool(descriptor and descriptor.installed),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["REMOTE_STORAGE_PROVIDER_ID", "remote_storage_provider_states"]
|
||||
@@ -0,0 +1,113 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Sequence
|
||||
from urllib.parse import quote
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from govoplan_core.core.records import (
|
||||
RecordContractError,
|
||||
RecordSourceLocator,
|
||||
RecordSourceReference,
|
||||
)
|
||||
from govoplan_files.backend.db.models import FileBlob, FileVersion
|
||||
from govoplan_files.backend.storage.common import FileStorageError
|
||||
from govoplan_files.backend.storage.files import get_asset_for_user
|
||||
|
||||
|
||||
CAPABILITY_RECORD_SOURCE_FILES = "records.source.files"
|
||||
|
||||
|
||||
class FilesRecordSource:
|
||||
provider_id = "files"
|
||||
|
||||
def resource_types(self) -> Sequence[str]:
|
||||
return ("file_version",)
|
||||
|
||||
def resolve(
|
||||
self,
|
||||
session: object,
|
||||
principal: object,
|
||||
*,
|
||||
locator: RecordSourceLocator,
|
||||
purpose: str,
|
||||
) -> RecordSourceReference:
|
||||
if not isinstance(session, Session):
|
||||
raise RecordContractError(
|
||||
"Files record references require a database session."
|
||||
)
|
||||
tenant_id = str(getattr(principal, "tenant_id", "") or "").strip()
|
||||
if not tenant_id or locator.tenant_id != tenant_id:
|
||||
raise RecordContractError("Files record references cannot cross tenants.")
|
||||
if locator.source_module != "files" or locator.resource_type != "file_version":
|
||||
raise RecordContractError("Unsupported Files record source type.")
|
||||
if not str(purpose or "").strip():
|
||||
raise RecordContractError("Files record references require a purpose.")
|
||||
if not hasattr(principal, "has") or not (
|
||||
principal.has("files:file:read") or principal.has("files:file:admin")
|
||||
):
|
||||
raise RecordContractError("Current Files read permission is required.")
|
||||
user = getattr(principal, "user", None)
|
||||
user_id = str(
|
||||
getattr(user, "id", "") or getattr(principal, "membership_id", "") or ""
|
||||
).strip()
|
||||
if not user_id:
|
||||
raise RecordContractError(
|
||||
"Files record references require a tenant user principal."
|
||||
)
|
||||
try:
|
||||
asset = get_asset_for_user(
|
||||
session,
|
||||
tenant_id=tenant_id,
|
||||
user_id=user_id,
|
||||
asset_id=locator.resource_id,
|
||||
is_admin=principal.has("files:file:admin"),
|
||||
)
|
||||
except FileStorageError as exc:
|
||||
raise RecordContractError(str(exc)) from exc
|
||||
version_query = session.query(FileVersion).filter(
|
||||
FileVersion.tenant_id == tenant_id,
|
||||
FileVersion.file_asset_id == asset.id,
|
||||
)
|
||||
revision = locator.source_revision.strip()
|
||||
version = version_query.filter(FileVersion.id == revision).one_or_none()
|
||||
if version is None:
|
||||
raise RecordContractError("The exact file version does not exist.")
|
||||
blob = session.get(FileBlob, version.blob_id)
|
||||
if blob is None or blob.tenant_id != tenant_id:
|
||||
raise RecordContractError(
|
||||
"The exact file version has no managed content object."
|
||||
)
|
||||
if blob.quarantined_at is not None or blob.integrity_status == "failed":
|
||||
raise RecordContractError(
|
||||
"The exact file version failed the current integrity gate."
|
||||
)
|
||||
return RecordSourceReference(
|
||||
locator=locator,
|
||||
label=version.filename_at_upload,
|
||||
authority_mode="external_authoritative",
|
||||
content_sha256=version.checksum_sha256,
|
||||
content_type=version.content_type,
|
||||
size_bytes=version.size_bytes,
|
||||
recorded_at=version.created_at,
|
||||
launch_url=(
|
||||
f"/files?fileId={quote(asset.id, safe='')}&versionId={quote(version.id, safe='')}"
|
||||
),
|
||||
metadata={
|
||||
"display_path": version.display_path_at_upload,
|
||||
"version_number": version.version_number,
|
||||
"integrity_status": blob.integrity_status,
|
||||
"protection": blob.protection_discriminator,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def create_files_record_source(_context: object) -> FilesRecordSource:
|
||||
return FilesRecordSource()
|
||||
|
||||
|
||||
__all__ = [
|
||||
"CAPABILITY_RECORD_SOURCE_FILES",
|
||||
"FilesRecordSource",
|
||||
"create_files_record_source",
|
||||
]
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1 @@
|
||||
"""Focused HTTP route modules for the Files API."""
|
||||
@@ -0,0 +1,175 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from io import BytesIO
|
||||
from fastapi import APIRouter, Depends
|
||||
from fastapi.responses import StreamingResponse
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from govoplan_core.auth import ApiPrincipal, require_scope
|
||||
from govoplan_files.backend.schemas import (
|
||||
BulkDeleteRequest,
|
||||
BulkDeleteResponse,
|
||||
FileAssetResponse,
|
||||
)
|
||||
from govoplan_core.db.session import get_session
|
||||
from govoplan_files.backend.storage.common import FileStorageError
|
||||
from govoplan_files.backend.storage.files import (
|
||||
get_asset_for_user,
|
||||
read_asset_bytes,
|
||||
read_asset_version_bytes,
|
||||
soft_delete_assets,
|
||||
)
|
||||
|
||||
|
||||
from govoplan_files.backend.route_support import (
|
||||
_asset_response,
|
||||
_attachment_disposition,
|
||||
_audit_connector_event,
|
||||
_http_error,
|
||||
_is_admin,
|
||||
)
|
||||
|
||||
router = APIRouter(prefix="/files", tags=["files"])
|
||||
|
||||
|
||||
@router.get("/{file_id}", response_model=FileAssetResponse)
|
||||
def get_file(
|
||||
file_id: str,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(require_scope("files:file:read")),
|
||||
):
|
||||
try:
|
||||
asset = get_asset_for_user(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
user_id=principal.user.id,
|
||||
asset_id=file_id,
|
||||
is_admin=_is_admin(principal),
|
||||
)
|
||||
return _asset_response(session, asset, include_shares=True)
|
||||
except FileStorageError as exc:
|
||||
raise _http_error(exc, not_found=True) from exc
|
||||
|
||||
|
||||
@router.get("/{file_id}/download")
|
||||
def download_file(
|
||||
file_id: str,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(require_scope("files:file:download")),
|
||||
):
|
||||
try:
|
||||
asset = get_asset_for_user(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
user_id=principal.user.id,
|
||||
asset_id=file_id,
|
||||
is_admin=_is_admin(principal),
|
||||
)
|
||||
data, version, blob = read_asset_bytes(session, asset)
|
||||
_audit_connector_event(
|
||||
session,
|
||||
principal,
|
||||
action="files.connector.accessed",
|
||||
asset=asset,
|
||||
version=version,
|
||||
blob=blob,
|
||||
operation="download",
|
||||
commit=True,
|
||||
)
|
||||
except FileStorageError as exc:
|
||||
raise _http_error(exc, not_found=True) from exc
|
||||
headers = {"Content-Disposition": _attachment_disposition(asset.filename)}
|
||||
return StreamingResponse(
|
||||
BytesIO(data),
|
||||
media_type=blob.content_type or "application/octet-stream",
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{file_id}/versions/{version_id}/download")
|
||||
def download_file_version(
|
||||
file_id: str,
|
||||
version_id: str,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(require_scope("files:file:download")),
|
||||
):
|
||||
try:
|
||||
asset = get_asset_for_user(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
user_id=principal.user.id,
|
||||
asset_id=file_id,
|
||||
is_admin=_is_admin(principal),
|
||||
)
|
||||
data, version, blob = read_asset_version_bytes(session, asset, version_id)
|
||||
_audit_connector_event(
|
||||
session,
|
||||
principal,
|
||||
action="files.connector.accessed",
|
||||
asset=asset,
|
||||
version=version,
|
||||
blob=blob,
|
||||
operation="download-version",
|
||||
commit=True,
|
||||
)
|
||||
except FileStorageError as exc:
|
||||
raise _http_error(exc, not_found=True) from exc
|
||||
headers = {
|
||||
"Content-Disposition": _attachment_disposition(
|
||||
version.filename_at_upload or asset.filename
|
||||
)
|
||||
}
|
||||
return StreamingResponse(
|
||||
BytesIO(data),
|
||||
media_type=blob.content_type or "application/octet-stream",
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
|
||||
@router.delete("/{file_id}", response_model=BulkDeleteResponse)
|
||||
def delete_file(
|
||||
file_id: str,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(require_scope("files:file:delete")),
|
||||
):
|
||||
try:
|
||||
asset = get_asset_for_user(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
user_id=principal.user.id,
|
||||
asset_id=file_id,
|
||||
require_write=True,
|
||||
is_admin=_is_admin(principal),
|
||||
)
|
||||
count = soft_delete_assets(session, [asset])
|
||||
session.commit()
|
||||
return BulkDeleteResponse(deleted_count=count)
|
||||
except FileStorageError as exc:
|
||||
session.rollback()
|
||||
raise _http_error(exc, not_found=True) from exc
|
||||
|
||||
|
||||
@router.post("/bulk-delete", response_model=BulkDeleteResponse)
|
||||
def bulk_delete_files(
|
||||
payload: BulkDeleteRequest,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(require_scope("files:file:delete")),
|
||||
):
|
||||
try:
|
||||
assets = [
|
||||
get_asset_for_user(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
user_id=principal.user.id,
|
||||
asset_id=file_id,
|
||||
require_write=True,
|
||||
is_admin=_is_admin(principal),
|
||||
)
|
||||
for file_id in payload.file_ids
|
||||
]
|
||||
count = soft_delete_assets(session, assets)
|
||||
session.commit()
|
||||
return BulkDeleteResponse(deleted_count=count)
|
||||
except FileStorageError as exc:
|
||||
session.rollback()
|
||||
raise _http_error(exc) from exc
|
||||
@@ -0,0 +1,367 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from govoplan_core.auth import ApiPrincipal, require_any_scope, require_scope
|
||||
from govoplan_core.audit.logging import audit_from_principal
|
||||
from govoplan_files.backend.schemas import (
|
||||
FileConnectorBrowseItem,
|
||||
FileConnectorBrowseResponse,
|
||||
FileConnectorImportRequest,
|
||||
FileConnectorSyncResponse,
|
||||
FileConnectorWriteRequest,
|
||||
FileConnectorWriteResponse,
|
||||
FileUploadResponse,
|
||||
)
|
||||
from govoplan_core.db.session import get_session
|
||||
from govoplan_files.backend.storage.paths import UnsafeFilePathError
|
||||
from govoplan_files.backend.storage.common import FileStorageError
|
||||
from govoplan_files.backend.storage.connector_browse import (
|
||||
ConnectorBrowseError,
|
||||
ConnectorBrowseUnsupported,
|
||||
browse_connector_profile,
|
||||
normalize_connector_browse_path,
|
||||
)
|
||||
from govoplan_files.backend.storage.connector_imports import (
|
||||
ConnectorImportError,
|
||||
ConnectorImportUnsupported,
|
||||
)
|
||||
from govoplan_files.backend.storage.connector_deployment import (
|
||||
connector_effective_endpoint_url,
|
||||
)
|
||||
from govoplan_files.backend.storage.connector_policy import (
|
||||
ConnectorAccessRequest,
|
||||
ConnectorPolicyDenied,
|
||||
connector_policy_decision,
|
||||
)
|
||||
from govoplan_files.backend.storage.files import (
|
||||
create_file_asset,
|
||||
get_asset_for_user,
|
||||
read_asset_bytes,
|
||||
sync_file_asset_from_source,
|
||||
)
|
||||
from govoplan_files.backend.storage.connector_spaces import (
|
||||
get_connector_space_for_user,
|
||||
)
|
||||
from govoplan_files.backend.storage.connector_writes import write_connector_file
|
||||
|
||||
|
||||
from govoplan_files.backend.route_support import (
|
||||
_asset_response,
|
||||
_audit_connector_imports,
|
||||
_audit_connector_sync,
|
||||
_connector_browse_next_token,
|
||||
_connector_policy_error,
|
||||
_download_connector_payload,
|
||||
_ensure_campaign_file_access,
|
||||
_http_error,
|
||||
_is_admin,
|
||||
_visible_connector_profile,
|
||||
)
|
||||
|
||||
router = APIRouter(prefix="/files", tags=["files"])
|
||||
|
||||
|
||||
@router.post(
|
||||
"/connector-spaces/{space_id}/write-back",
|
||||
response_model=FileConnectorWriteResponse,
|
||||
)
|
||||
def write_back_connector_file(
|
||||
space_id: str,
|
||||
payload: FileConnectorWriteRequest,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(require_scope("files:connector:write")),
|
||||
):
|
||||
try:
|
||||
space = get_connector_space_for_user(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
user_id=principal.user.id,
|
||||
space_id=space_id,
|
||||
is_admin=_is_admin(principal),
|
||||
)
|
||||
if space.read_only:
|
||||
raise FileStorageError("This connector space is read-only")
|
||||
profile = _visible_connector_profile(
|
||||
session, principal, space.connector_profile_id
|
||||
)
|
||||
requested_path = normalize_connector_browse_path(payload.remote_path)
|
||||
remote_path = normalize_connector_browse_path(
|
||||
"/".join(part for part in (space.remote_path, requested_path) if part)
|
||||
)
|
||||
decision = connector_policy_decision(
|
||||
ConnectorAccessRequest(
|
||||
connector_id=profile.id,
|
||||
credential_id=profile.credential_profile_id,
|
||||
provider=profile.provider,
|
||||
external_path=remote_path,
|
||||
external_url=connector_effective_endpoint_url(
|
||||
provider=profile.provider,
|
||||
endpoint_url=profile.endpoint_url,
|
||||
metadata=profile.metadata,
|
||||
),
|
||||
operation="write",
|
||||
),
|
||||
profile.policy_sources,
|
||||
)
|
||||
if not decision.allowed:
|
||||
raise ConnectorPolicyDenied(decision)
|
||||
asset = get_asset_for_user(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
user_id=principal.user.id,
|
||||
asset_id=payload.file_id,
|
||||
require_write=True,
|
||||
is_admin=_is_admin(principal),
|
||||
)
|
||||
data, version, blob = read_asset_bytes(session, asset)
|
||||
connector_library_id = space.library_id
|
||||
content_type = blob.content_type
|
||||
# Close the read snapshot before the independent recovery transaction
|
||||
# records authority for the external effect. This avoids upgrading an
|
||||
# older SQLite read snapshot after the ledger commit.
|
||||
session.commit()
|
||||
result = write_connector_file(
|
||||
profile,
|
||||
tenant_id=principal.tenant_id,
|
||||
library_id=connector_library_id,
|
||||
remote_path=remote_path,
|
||||
data=data,
|
||||
content_type=content_type,
|
||||
expected_revision=payload.expected_revision,
|
||||
idempotency_key=payload.idempotency_key,
|
||||
)
|
||||
audit_from_principal(
|
||||
session,
|
||||
principal,
|
||||
action="files.connector.written",
|
||||
object_type="file",
|
||||
object_id=asset.id,
|
||||
details={
|
||||
"file_version_id": version.id,
|
||||
"file_blob_id": blob.id,
|
||||
"checksum_sha256": blob.checksum_sha256,
|
||||
"connector_space_id": space.id,
|
||||
"connector_profile_id": profile.id,
|
||||
"remote_path": remote_path,
|
||||
"recovery_operation_id": result.recovery_operation_id,
|
||||
"recovery_status": result.status,
|
||||
"revision": result.revision,
|
||||
},
|
||||
)
|
||||
session.commit()
|
||||
return FileConnectorWriteResponse(
|
||||
recovery_operation_id=result.recovery_operation_id,
|
||||
status=result.status,
|
||||
replayed=result.replayed,
|
||||
provider=result.provider,
|
||||
remote_path=result.remote_path,
|
||||
revision=result.revision,
|
||||
checksum_sha256=result.checksum_sha256,
|
||||
size_bytes=result.size_bytes,
|
||||
)
|
||||
except ConnectorPolicyDenied as exc:
|
||||
session.rollback()
|
||||
raise _connector_policy_error(exc) from exc
|
||||
except (FileStorageError, ConnectorBrowseError) as exc:
|
||||
session.rollback()
|
||||
raise _http_error(exc) from exc
|
||||
|
||||
|
||||
@router.post(
|
||||
"/connectors/profiles/{profile_id}/import", response_model=FileUploadResponse
|
||||
)
|
||||
def import_connector_file(
|
||||
profile_id: str,
|
||||
payload: FileConnectorImportRequest,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(require_scope("files:file:upload")),
|
||||
):
|
||||
try:
|
||||
if payload.campaign_id:
|
||||
_ensure_campaign_file_access(session, principal, payload.campaign_id)
|
||||
profile = _visible_connector_profile(
|
||||
session, principal, profile_id, campaign_id=payload.campaign_id
|
||||
)
|
||||
_source_path, downloaded, metadata = _download_connector_payload(
|
||||
profile, payload, operation="import"
|
||||
)
|
||||
target_owner = payload.owner_id or principal.user.id
|
||||
stored = create_file_asset(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
owner_type=payload.owner_type,
|
||||
owner_id=target_owner,
|
||||
user_id=principal.user.id,
|
||||
filename=downloaded.filename,
|
||||
data=downloaded.data,
|
||||
folder=payload.target_folder,
|
||||
display_path=payload.target_path,
|
||||
content_type=downloaded.content_type,
|
||||
metadata=metadata,
|
||||
campaign_id=payload.campaign_id,
|
||||
conflict_strategy=payload.conflict_strategy,
|
||||
is_admin=_is_admin(principal),
|
||||
)
|
||||
_audit_connector_imports(session, principal, [stored.asset])
|
||||
session.commit()
|
||||
except ConnectorPolicyDenied as exc:
|
||||
session.rollback()
|
||||
raise _connector_policy_error(exc) from exc
|
||||
except ConnectorImportUnsupported as exc:
|
||||
session.rollback()
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_501_NOT_IMPLEMENTED, detail=str(exc)
|
||||
) from exc
|
||||
except (
|
||||
ConnectorImportError,
|
||||
FileStorageError,
|
||||
UnsafeFilePathError,
|
||||
ValueError,
|
||||
json.JSONDecodeError,
|
||||
) as exc:
|
||||
session.rollback()
|
||||
raise _http_error(exc) from exc
|
||||
return FileUploadResponse(
|
||||
files=[_asset_response(session, stored.asset, include_shares=True)]
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/connectors/profiles/{profile_id}/sync", response_model=FileConnectorSyncResponse
|
||||
)
|
||||
def sync_connector_file(
|
||||
profile_id: str,
|
||||
payload: FileConnectorImportRequest,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(require_scope("files:file:upload")),
|
||||
):
|
||||
try:
|
||||
if payload.campaign_id:
|
||||
_ensure_campaign_file_access(session, principal, payload.campaign_id)
|
||||
profile = _visible_connector_profile(
|
||||
session, principal, profile_id, campaign_id=payload.campaign_id
|
||||
)
|
||||
_source_path, downloaded, metadata = _download_connector_payload(
|
||||
profile, payload, operation="sync"
|
||||
)
|
||||
target_owner = payload.owner_id or principal.user.id
|
||||
stored, sync_action, previous_version_id = sync_file_asset_from_source(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
owner_type=payload.owner_type,
|
||||
owner_id=target_owner,
|
||||
user_id=principal.user.id,
|
||||
filename=downloaded.filename,
|
||||
data=downloaded.data,
|
||||
folder=payload.target_folder,
|
||||
display_path=payload.target_path,
|
||||
content_type=downloaded.content_type,
|
||||
metadata=metadata,
|
||||
campaign_id=payload.campaign_id,
|
||||
conflict_strategy=payload.conflict_strategy,
|
||||
is_admin=_is_admin(principal),
|
||||
)
|
||||
_audit_connector_sync(
|
||||
session,
|
||||
principal,
|
||||
stored.asset,
|
||||
sync_action=sync_action,
|
||||
previous_version_id=previous_version_id,
|
||||
)
|
||||
session.commit()
|
||||
except ConnectorPolicyDenied as exc:
|
||||
session.rollback()
|
||||
raise _connector_policy_error(exc) from exc
|
||||
except ConnectorImportUnsupported as exc:
|
||||
session.rollback()
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_501_NOT_IMPLEMENTED, detail=str(exc)
|
||||
) from exc
|
||||
except (
|
||||
ConnectorImportError,
|
||||
FileStorageError,
|
||||
UnsafeFilePathError,
|
||||
ValueError,
|
||||
json.JSONDecodeError,
|
||||
) as exc:
|
||||
session.rollback()
|
||||
raise _http_error(exc) from exc
|
||||
return FileConnectorSyncResponse(
|
||||
file=_asset_response(session, stored.asset, include_shares=True),
|
||||
action=sync_action,
|
||||
previous_version_id=previous_version_id,
|
||||
current_version_id=stored.version.id,
|
||||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/connectors/profiles/{profile_id}/browse",
|
||||
response_model=FileConnectorBrowseResponse,
|
||||
)
|
||||
def browse_connector_profile_items(
|
||||
profile_id: str,
|
||||
path: str | None = None,
|
||||
library_id: str | None = None,
|
||||
continuation_token: str | None = None,
|
||||
campaign_id: str | None = None,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(
|
||||
require_any_scope(
|
||||
"files:file:read",
|
||||
"files:file:upload",
|
||||
"files:file:download",
|
||||
"files:file:admin",
|
||||
"system:settings:read",
|
||||
"admin:settings:read",
|
||||
)
|
||||
),
|
||||
):
|
||||
try:
|
||||
profile = _visible_connector_profile(
|
||||
session, principal, profile_id, campaign_id=campaign_id
|
||||
)
|
||||
browse_path = normalize_connector_browse_path(path)
|
||||
decision = connector_policy_decision(
|
||||
ConnectorAccessRequest(
|
||||
connector_id=profile.id,
|
||||
credential_id=profile.credential_profile_id,
|
||||
provider=profile.provider,
|
||||
external_path=browse_path,
|
||||
external_url=connector_effective_endpoint_url(
|
||||
provider=profile.provider,
|
||||
endpoint_url=profile.endpoint_url,
|
||||
metadata=profile.metadata,
|
||||
),
|
||||
operation="browse",
|
||||
),
|
||||
profile.policy_sources,
|
||||
)
|
||||
if not decision.allowed:
|
||||
raise ConnectorPolicyDenied(decision)
|
||||
items = browse_connector_profile(
|
||||
profile,
|
||||
path=browse_path,
|
||||
library_id=library_id,
|
||||
continuation_token=continuation_token,
|
||||
)
|
||||
except ConnectorPolicyDenied as exc:
|
||||
raise _connector_policy_error(exc) from exc
|
||||
except ConnectorBrowseUnsupported as exc:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_501_NOT_IMPLEMENTED, detail=str(exc)
|
||||
) from exc
|
||||
except (ConnectorBrowseError, OSError, ValueError, json.JSONDecodeError) as exc:
|
||||
raise _http_error(exc) from exc
|
||||
return FileConnectorBrowseResponse(
|
||||
profile_id=profile.id,
|
||||
provider=profile.provider,
|
||||
path=browse_path,
|
||||
library_id=library_id,
|
||||
next_continuation_token=_connector_browse_next_token(items),
|
||||
has_more=any(bool(item.metadata.get("listing_truncated")) for item in items),
|
||||
decision=decision.to_dict(),
|
||||
items=[FileConnectorBrowseItem(**item.to_response()) for item in items],
|
||||
)
|
||||
@@ -0,0 +1,261 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from fastapi import APIRouter, Depends
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from govoplan_core.auth import ApiPrincipal, require_any_scope
|
||||
from govoplan_files.backend.change_tracking import (
|
||||
FILES_CONNECTOR_PROFILES_COLLECTION,
|
||||
)
|
||||
from govoplan_files.backend.schemas import (
|
||||
FileConnectorProfileResponse,
|
||||
FileConnectorProfileUpdateRequest,
|
||||
)
|
||||
from govoplan_core.db.session import get_session
|
||||
from govoplan_files.backend.storage.common import FileStorageError
|
||||
from govoplan_files.backend.storage.connector_credential_deletion import (
|
||||
delete_connector_profile_row,
|
||||
)
|
||||
from govoplan_files.backend.storage.connector_deployment import (
|
||||
connector_effective_endpoint_url,
|
||||
)
|
||||
from govoplan_files.backend.storage.connector_profile_store import (
|
||||
connector_profile_from_row,
|
||||
get_connector_profile_row,
|
||||
update_connector_profile_row,
|
||||
)
|
||||
from govoplan_files.backend.storage.connector_policy import (
|
||||
ConnectorPolicyDenied,
|
||||
)
|
||||
|
||||
|
||||
from govoplan_files.backend.route_support import (
|
||||
FILES_CONNECTOR_PROFILE_RESOURCE,
|
||||
_can_read_disabled_connector_profiles,
|
||||
_connector_policy_error,
|
||||
_credential_row_for_profile,
|
||||
_ensure_connector_configuration_allowed,
|
||||
_ensure_connector_local_policy_allowed,
|
||||
_http_error,
|
||||
_record_connector_settings_change,
|
||||
_require_connector_profile_write,
|
||||
_visible_connector_profile,
|
||||
)
|
||||
|
||||
router = APIRouter(prefix="/files", tags=["files"])
|
||||
|
||||
|
||||
@router.get(
|
||||
"/connectors/profiles/{profile_id}", response_model=FileConnectorProfileResponse
|
||||
)
|
||||
def get_connector_profile(
|
||||
profile_id: str,
|
||||
campaign_id: str | None = None,
|
||||
include_disabled: bool = False,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(
|
||||
require_any_scope(
|
||||
"files:file:read",
|
||||
"files:file:upload",
|
||||
"files:file:download",
|
||||
"files:file:admin",
|
||||
"system:settings:read",
|
||||
"admin:settings:read",
|
||||
)
|
||||
),
|
||||
):
|
||||
try:
|
||||
profile = _visible_connector_profile(
|
||||
session,
|
||||
principal,
|
||||
profile_id,
|
||||
campaign_id=campaign_id,
|
||||
include_disabled=include_disabled
|
||||
and _can_read_disabled_connector_profiles(principal),
|
||||
include_effective_policy=False,
|
||||
)
|
||||
except (OSError, ValueError, json.JSONDecodeError) as exc:
|
||||
raise _http_error(exc) from exc
|
||||
return FileConnectorProfileResponse(**profile.to_response())
|
||||
|
||||
|
||||
@router.patch(
|
||||
"/connectors/profiles/{profile_id}", response_model=FileConnectorProfileResponse
|
||||
)
|
||||
def update_connector_profile(
|
||||
profile_id: str,
|
||||
payload: FileConnectorProfileUpdateRequest,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(
|
||||
require_any_scope(
|
||||
"files:file:admin", "system:settings:write", "admin:settings:write"
|
||||
)
|
||||
),
|
||||
):
|
||||
try:
|
||||
row = get_connector_profile_row(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
profile_id=profile_id,
|
||||
include_disabled=True,
|
||||
)
|
||||
except FileStorageError as exc:
|
||||
raise _http_error(exc, not_found=True) from exc
|
||||
_require_connector_profile_write(principal, row.scope_type)
|
||||
credentials = payload.credentials
|
||||
try:
|
||||
credential_profile_id = (
|
||||
payload.credential_profile_id
|
||||
if payload.credential_profile_id is not None
|
||||
else row.credential_profile_id
|
||||
)
|
||||
provider = payload.provider if payload.provider is not None else row.provider
|
||||
credential_row = _credential_row_for_profile(
|
||||
session,
|
||||
principal,
|
||||
credential_profile_id=credential_profile_id,
|
||||
provider=provider,
|
||||
profile_id=row.id,
|
||||
scope_type=row.scope_type,
|
||||
scope_id=row.scope_id,
|
||||
include_disabled=True,
|
||||
)
|
||||
_ensure_connector_configuration_allowed(
|
||||
session,
|
||||
principal,
|
||||
connector_id=row.id,
|
||||
credential_id=credential_profile_id,
|
||||
provider=provider,
|
||||
endpoint_url=connector_effective_endpoint_url(
|
||||
provider=provider,
|
||||
endpoint_url=payload.endpoint_url
|
||||
if payload.endpoint_url is not None
|
||||
else row.endpoint_url,
|
||||
metadata=payload.metadata
|
||||
if payload.metadata is not None
|
||||
else row.metadata_,
|
||||
),
|
||||
base_path=payload.base_path
|
||||
if payload.base_path is not None
|
||||
else row.base_path,
|
||||
scope_type=row.scope_type,
|
||||
scope_id=row.scope_id,
|
||||
operation="configure",
|
||||
)
|
||||
if payload.policy is not None:
|
||||
_ensure_connector_local_policy_allowed(
|
||||
session,
|
||||
principal,
|
||||
scope_type=row.scope_type,
|
||||
scope_id=row.scope_id,
|
||||
policy=payload.policy,
|
||||
)
|
||||
update_connector_profile_row(
|
||||
session,
|
||||
row,
|
||||
user_id=principal.user.id,
|
||||
label=payload.label,
|
||||
provider=payload.provider,
|
||||
endpoint_url=payload.endpoint_url,
|
||||
base_path=payload.base_path,
|
||||
enabled=payload.enabled,
|
||||
credential_profile_id=payload.credential_profile_id,
|
||||
credential_mode=payload.credential_mode,
|
||||
username=credentials.username if credentials else None,
|
||||
password=credentials.password if credentials else None,
|
||||
token=credentials.token if credentials else None,
|
||||
password_env=credentials.password_env if credentials else None,
|
||||
token_env=credentials.token_env if credentials else None,
|
||||
secret_ref=credentials.secret_ref if credentials else None,
|
||||
clear_password=payload.clear_password,
|
||||
clear_token=payload.clear_token,
|
||||
capabilities=payload.capabilities,
|
||||
policy=payload.policy,
|
||||
metadata=payload.metadata,
|
||||
)
|
||||
_record_connector_settings_change(
|
||||
session,
|
||||
collection=FILES_CONNECTOR_PROFILES_COLLECTION,
|
||||
resource_type=FILES_CONNECTOR_PROFILE_RESOURCE,
|
||||
resource_id=row.id,
|
||||
operation="updated",
|
||||
principal=principal,
|
||||
tenant_id=row.tenant_id,
|
||||
payload={
|
||||
"scope_type": row.scope_type,
|
||||
"scope_id": row.scope_id,
|
||||
"provider": row.provider,
|
||||
},
|
||||
)
|
||||
session.commit()
|
||||
session.refresh(row)
|
||||
return FileConnectorProfileResponse(
|
||||
**connector_profile_from_row(
|
||||
row, credential_row=credential_row
|
||||
).to_response()
|
||||
)
|
||||
except ConnectorPolicyDenied as exc:
|
||||
session.rollback()
|
||||
raise _connector_policy_error(exc) from exc
|
||||
except (FileStorageError, ValueError, json.JSONDecodeError) as exc:
|
||||
session.rollback()
|
||||
raise _http_error(exc) from exc
|
||||
|
||||
|
||||
@router.delete(
|
||||
"/connectors/profiles/{profile_id}", response_model=FileConnectorProfileResponse
|
||||
)
|
||||
def deactivate_connector_profile(
|
||||
profile_id: str,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(
|
||||
require_any_scope(
|
||||
"files:file:admin", "system:settings:write", "admin:settings:write"
|
||||
)
|
||||
),
|
||||
):
|
||||
try:
|
||||
row = get_connector_profile_row(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
profile_id=profile_id,
|
||||
include_disabled=True,
|
||||
)
|
||||
except FileStorageError as exc:
|
||||
raise _http_error(exc, not_found=True) from exc
|
||||
_require_connector_profile_write(principal, row.scope_type)
|
||||
try:
|
||||
changed = delete_connector_profile_row(
|
||||
session,
|
||||
row,
|
||||
deletion_reason="api_delete",
|
||||
user_id=principal.user.id,
|
||||
api_key_id=principal.api_key.id if principal.api_key else None,
|
||||
)
|
||||
if changed:
|
||||
_record_connector_settings_change(
|
||||
session,
|
||||
collection=FILES_CONNECTOR_PROFILES_COLLECTION,
|
||||
resource_type=FILES_CONNECTOR_PROFILE_RESOURCE,
|
||||
resource_id=row.id,
|
||||
operation="deleted",
|
||||
principal=principal,
|
||||
tenant_id=row.tenant_id,
|
||||
payload={
|
||||
"scope_type": row.scope_type,
|
||||
"scope_id": row.scope_id,
|
||||
"provider": row.provider,
|
||||
},
|
||||
)
|
||||
session.commit()
|
||||
session.refresh(row)
|
||||
return FileConnectorProfileResponse(
|
||||
**connector_profile_from_row(row).to_response()
|
||||
)
|
||||
except FileStorageError as exc:
|
||||
session.rollback()
|
||||
raise _http_error(exc) from exc
|
||||
except Exception:
|
||||
session.rollback()
|
||||
raise
|
||||
@@ -0,0 +1,852 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Literal
|
||||
from fastapi import APIRouter, Depends, Query, status
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from govoplan_core.auth import ApiPrincipal, require_any_scope
|
||||
from govoplan_files.backend.change_tracking import (
|
||||
FILES_CONNECTOR_CREDENTIALS_COLLECTION,
|
||||
FILES_CONNECTOR_POLICIES_COLLECTION,
|
||||
FILES_CONNECTOR_PROFILES_COLLECTION,
|
||||
)
|
||||
from govoplan_files.backend.schemas import (
|
||||
FileConnectorCredentialCreateRequest,
|
||||
FileConnectorCredentialResponse,
|
||||
FileConnectorCredentialsResponse,
|
||||
FileConnectorCredentialUpdateRequest,
|
||||
FileConnectorDiscoveryRequest,
|
||||
FileConnectorDiscoveryResponse,
|
||||
FileConnectorSettingsDeltaResponse,
|
||||
FileConnectorPolicyEvaluateRequest,
|
||||
FileConnectorPolicyEvaluateResponse,
|
||||
FileConnectorPolicyResponse,
|
||||
FileConnectorPolicyUpdateRequest,
|
||||
FileConnectorProfileCreateRequest,
|
||||
FileConnectorProfileResponse,
|
||||
FileConnectorProfilesResponse,
|
||||
FileConnectorProviderResponse,
|
||||
FileConnectorProvidersResponse,
|
||||
)
|
||||
from govoplan_core.db.session import get_session
|
||||
from govoplan_files.backend.storage.common import FileStorageError
|
||||
from govoplan_files.backend.storage.connector_credential_store import (
|
||||
create_connector_credential_row,
|
||||
get_connector_credential_row,
|
||||
list_database_connector_credentials,
|
||||
update_connector_credential_row,
|
||||
)
|
||||
from govoplan_files.backend.storage.connector_credential_deletion import (
|
||||
delete_connector_credential_row,
|
||||
)
|
||||
from govoplan_files.backend.storage.connector_browse import (
|
||||
ConnectorBrowseError,
|
||||
ConnectorBrowseUnsupported,
|
||||
browse_connector_profile,
|
||||
)
|
||||
from govoplan_files.backend.storage.connector_deployment import (
|
||||
connector_effective_endpoint_url,
|
||||
reject_api_controlled_deployment_references,
|
||||
)
|
||||
from govoplan_files.backend.storage.connector_profile_store import (
|
||||
connector_profile_from_row,
|
||||
create_connector_profile_row,
|
||||
)
|
||||
from govoplan_files.backend.storage.connector_providers import (
|
||||
connector_provider_descriptors,
|
||||
)
|
||||
from govoplan_files.backend.storage.connector_policy import (
|
||||
ConnectorAccessRequest,
|
||||
ConnectorPolicyDenied,
|
||||
connector_policy_decision,
|
||||
connector_policy_sources_from_payload,
|
||||
)
|
||||
from govoplan_files.backend.storage.connector_policy_store import (
|
||||
connector_policy_response,
|
||||
set_connector_policy,
|
||||
)
|
||||
|
||||
|
||||
from govoplan_files.backend.route_support import (
|
||||
FILES_CONNECTOR_CREDENTIAL_RESOURCE,
|
||||
FILES_CONNECTOR_POLICY_RESOURCE,
|
||||
FILES_CONNECTOR_PROFILE_RESOURCE,
|
||||
_audit_connector_discovery_attempt,
|
||||
_can_read_disabled_connector_profiles,
|
||||
_connector_credential_response,
|
||||
_connector_policy_error,
|
||||
_credential_row_for_profile,
|
||||
_discovery_profile_from_payload,
|
||||
_ensure_campaign_file_access,
|
||||
_ensure_connector_configuration_allowed,
|
||||
_ensure_connector_credential_configuration_allowed,
|
||||
_ensure_connector_local_policy_allowed,
|
||||
_file_connector_policy_resource_id,
|
||||
_file_connector_settings_entries,
|
||||
_http_error,
|
||||
_record_connector_settings_change,
|
||||
_require_connector_credential_write,
|
||||
_require_connector_policy_read,
|
||||
_require_connector_profile_write,
|
||||
_same_endpoint,
|
||||
_visible_connector_profiles,
|
||||
_webdav_discovery_candidates,
|
||||
)
|
||||
from govoplan_files.backend.services.connector_settings_delta import (
|
||||
_full_file_connector_settings_delta_response,
|
||||
_incremental_file_connector_settings_delta_response,
|
||||
)
|
||||
|
||||
router = APIRouter(prefix="/files", tags=["files"])
|
||||
|
||||
|
||||
@router.post(
|
||||
"/connector-policy/evaluate", response_model=FileConnectorPolicyEvaluateResponse
|
||||
)
|
||||
def evaluate_connector_policy(
|
||||
payload: FileConnectorPolicyEvaluateRequest,
|
||||
principal: ApiPrincipal = Depends(
|
||||
require_any_scope("files:file:read", "files:file:upload", "files:file:download")
|
||||
),
|
||||
):
|
||||
del principal
|
||||
sources = connector_policy_sources_from_payload(
|
||||
[item.model_dump(mode="json") for item in payload.policy_sources]
|
||||
)
|
||||
request = ConnectorAccessRequest.from_provenance(
|
||||
payload.source_provenance.model_dump(mode="json", exclude_none=True),
|
||||
operation=payload.operation,
|
||||
)
|
||||
return FileConnectorPolicyEvaluateResponse(
|
||||
decision=connector_policy_decision(request, sources).to_dict()
|
||||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/connectors/settings/delta", response_model=FileConnectorSettingsDeltaResponse
|
||||
)
|
||||
def connector_settings_delta(
|
||||
scope_type: str = Query(default="tenant"),
|
||||
scope_id: str | None = Query(default=None),
|
||||
provider: str | None = None,
|
||||
campaign_id: str | None = None,
|
||||
include_disabled: bool = False,
|
||||
include_inactive: bool = False,
|
||||
owner_type: Literal["user", "group"] | None = None,
|
||||
owner_id: str | None = None,
|
||||
since: str | None = None,
|
||||
limit: int = Query(default=100, ge=1, le=500),
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(
|
||||
require_any_scope(
|
||||
"files:file:admin", "system:settings:read", "admin:settings:read"
|
||||
)
|
||||
),
|
||||
):
|
||||
scope_type = scope_type.strip().casefold()
|
||||
_require_connector_policy_read(principal, scope_type)
|
||||
try:
|
||||
if since is None:
|
||||
return _full_file_connector_settings_delta_response(
|
||||
session,
|
||||
principal,
|
||||
scope_type=scope_type,
|
||||
scope_id=scope_id,
|
||||
provider=provider,
|
||||
campaign_id=campaign_id,
|
||||
include_disabled=include_disabled,
|
||||
include_inactive=include_inactive,
|
||||
owner_type=owner_type,
|
||||
owner_id=owner_id,
|
||||
)
|
||||
entries, has_more = _file_connector_settings_entries(
|
||||
session, tenant_id=principal.tenant_id, since=since, limit=limit
|
||||
)
|
||||
if entries is None:
|
||||
return _full_file_connector_settings_delta_response(
|
||||
session,
|
||||
principal,
|
||||
scope_type=scope_type,
|
||||
scope_id=scope_id,
|
||||
provider=provider,
|
||||
campaign_id=campaign_id,
|
||||
include_disabled=include_disabled,
|
||||
include_inactive=include_inactive,
|
||||
owner_type=owner_type,
|
||||
owner_id=owner_id,
|
||||
)
|
||||
return _incremental_file_connector_settings_delta_response(
|
||||
session,
|
||||
principal,
|
||||
entries=entries,
|
||||
has_more=has_more,
|
||||
scope_type=scope_type,
|
||||
scope_id=scope_id,
|
||||
provider=provider,
|
||||
campaign_id=campaign_id,
|
||||
include_disabled=include_disabled,
|
||||
include_inactive=include_inactive,
|
||||
owner_type=owner_type,
|
||||
owner_id=owner_id,
|
||||
)
|
||||
except (FileStorageError, ValueError, json.JSONDecodeError) as exc:
|
||||
raise _http_error(exc) from exc
|
||||
|
||||
|
||||
@router.get("/connectors/providers", response_model=FileConnectorProvidersResponse)
|
||||
def list_connector_providers(
|
||||
principal: ApiPrincipal = Depends(
|
||||
require_any_scope("files:file:read", "files:file:upload", "files:file:admin")
|
||||
),
|
||||
):
|
||||
del principal
|
||||
return FileConnectorProvidersResponse(
|
||||
providers=[
|
||||
FileConnectorProviderResponse(**item.to_response())
|
||||
for item in connector_provider_descriptors()
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@router.post("/connectors/discover", response_model=FileConnectorDiscoveryResponse)
|
||||
def discover_connector_endpoint(
|
||||
payload: FileConnectorDiscoveryRequest,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(
|
||||
require_any_scope(
|
||||
"files:file:admin", "system:settings:write", "admin:settings:write"
|
||||
)
|
||||
),
|
||||
):
|
||||
try:
|
||||
reject_api_controlled_deployment_references(
|
||||
password_env=payload.credentials.password_env,
|
||||
token_env=payload.credentials.token_env,
|
||||
secret_ref=payload.credentials.secret_ref,
|
||||
metadata=payload.metadata,
|
||||
)
|
||||
except ValueError as exc:
|
||||
raise _http_error(exc) from exc
|
||||
if payload.provider not in {"webdav", "nextcloud"}:
|
||||
return FileConnectorDiscoveryResponse(
|
||||
provider=payload.provider,
|
||||
endpoint_url=None,
|
||||
base_path=payload.base_path,
|
||||
status="unsupported",
|
||||
message=f"Discovery is not implemented for {payload.provider} connectors yet",
|
||||
)
|
||||
candidates: list[dict[str, str]] = []
|
||||
for endpoint_url in _webdav_discovery_candidates(payload):
|
||||
profile = _discovery_profile_from_payload(
|
||||
payload, endpoint_url, principal=principal
|
||||
)
|
||||
try:
|
||||
_ensure_connector_configuration_allowed(
|
||||
session,
|
||||
principal,
|
||||
connector_id=None,
|
||||
credential_id=None,
|
||||
provider=profile.provider,
|
||||
endpoint_url=endpoint_url,
|
||||
base_path=profile.base_path,
|
||||
scope_type="tenant",
|
||||
scope_id=principal.tenant_id,
|
||||
operation="discover",
|
||||
)
|
||||
_audit_connector_discovery_attempt(
|
||||
session,
|
||||
principal,
|
||||
provider=profile.provider,
|
||||
endpoint_url=endpoint_url,
|
||||
base_path=profile.base_path,
|
||||
)
|
||||
browse_connector_profile(profile, path=payload.base_path or "")
|
||||
except ConnectorPolicyDenied as exc:
|
||||
raise _connector_policy_error(exc) from exc
|
||||
except (
|
||||
ConnectorBrowseError,
|
||||
ConnectorBrowseUnsupported,
|
||||
OSError,
|
||||
ValueError,
|
||||
json.JSONDecodeError,
|
||||
) as exc:
|
||||
message = str(exc)
|
||||
if "credentials were rejected" in message.casefold():
|
||||
if payload.require_valid_credentials:
|
||||
candidates.append(
|
||||
{
|
||||
"endpoint_url": endpoint_url,
|
||||
"status": "credentials_rejected",
|
||||
"message": "The endpoint exists, but the credentials were rejected.",
|
||||
}
|
||||
)
|
||||
return FileConnectorDiscoveryResponse(
|
||||
provider=payload.provider,
|
||||
endpoint_url=endpoint_url,
|
||||
base_path=payload.base_path,
|
||||
status="credentials_rejected",
|
||||
message="The endpoint was found, but login failed with these credentials.",
|
||||
candidates=candidates,
|
||||
metadata={"discovered_by": "webdav-auth-challenge"},
|
||||
)
|
||||
candidates.append(
|
||||
{
|
||||
"endpoint_url": endpoint_url,
|
||||
"status": "found",
|
||||
"message": "The endpoint exists, but credentials are required or were rejected.",
|
||||
}
|
||||
)
|
||||
return FileConnectorDiscoveryResponse(
|
||||
provider=payload.provider,
|
||||
endpoint_url=endpoint_url,
|
||||
base_path=payload.base_path,
|
||||
status="found",
|
||||
message="The endpoint was found. Add working credentials before saving or testing the connection.",
|
||||
candidates=candidates,
|
||||
metadata={"discovered_by": "webdav-auth-challenge"},
|
||||
)
|
||||
candidates.append(
|
||||
{"endpoint_url": endpoint_url, "status": "failed", "message": message}
|
||||
)
|
||||
continue
|
||||
status_value = (
|
||||
"usable" if _same_endpoint(endpoint_url, payload.endpoint_url) else "found"
|
||||
)
|
||||
message = (
|
||||
"The supplied URL is directly usable."
|
||||
if status_value == "usable"
|
||||
else "A usable connector endpoint was discovered."
|
||||
)
|
||||
candidates.append(
|
||||
{"endpoint_url": endpoint_url, "status": status_value, "message": message}
|
||||
)
|
||||
return FileConnectorDiscoveryResponse(
|
||||
provider=payload.provider,
|
||||
endpoint_url=endpoint_url,
|
||||
base_path=payload.base_path,
|
||||
status=status_value,
|
||||
message=message,
|
||||
candidates=candidates,
|
||||
metadata={"discovered_by": "webdav-propfind"},
|
||||
)
|
||||
return FileConnectorDiscoveryResponse(
|
||||
provider=payload.provider,
|
||||
endpoint_url=None,
|
||||
base_path=payload.base_path,
|
||||
status="not_found",
|
||||
message="No usable WebDAV endpoint was found for this server URL.",
|
||||
candidates=candidates,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/connectors/credentials", response_model=FileConnectorCredentialsResponse)
|
||||
def list_connector_credentials(
|
||||
provider: str | None = None,
|
||||
include_disabled: bool = False,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(
|
||||
require_any_scope(
|
||||
"files:file:admin", "system:settings:read", "admin:settings:read"
|
||||
)
|
||||
),
|
||||
):
|
||||
provider_norm = provider.strip().casefold() if provider else None
|
||||
credentials = list_database_connector_credentials(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
include_disabled=include_disabled
|
||||
and _can_read_disabled_connector_profiles(principal),
|
||||
)
|
||||
return FileConnectorCredentialsResponse(
|
||||
credentials=[
|
||||
FileConnectorCredentialResponse(**credential.to_response())
|
||||
for credential in credentials
|
||||
if provider_norm is None or credential.provider in {None, provider_norm}
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/connectors/policies/{scope_type}", response_model=FileConnectorPolicyResponse
|
||||
)
|
||||
def read_connector_policy(
|
||||
scope_type: str,
|
||||
scope_id: str | None = Query(default=None),
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(
|
||||
require_any_scope(
|
||||
"files:file:admin", "system:settings:read", "admin:settings:read"
|
||||
)
|
||||
),
|
||||
):
|
||||
_require_connector_policy_read(principal, scope_type)
|
||||
try:
|
||||
return FileConnectorPolicyResponse(
|
||||
**connector_policy_response(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
scope_type=scope_type,
|
||||
scope_id=scope_id,
|
||||
)
|
||||
)
|
||||
except (FileStorageError, ValueError, json.JSONDecodeError) as exc:
|
||||
raise _http_error(exc, not_found=True) from exc
|
||||
|
||||
|
||||
@router.put(
|
||||
"/connectors/policies/{scope_type}", response_model=FileConnectorPolicyResponse
|
||||
)
|
||||
def write_connector_policy(
|
||||
scope_type: str,
|
||||
payload: FileConnectorPolicyUpdateRequest,
|
||||
scope_id: str | None = Query(default=None),
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(
|
||||
require_any_scope(
|
||||
"files:file:admin", "system:settings:write", "admin:settings:write"
|
||||
)
|
||||
),
|
||||
):
|
||||
_require_connector_profile_write(principal, scope_type)
|
||||
try:
|
||||
set_connector_policy(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
user_id=principal.user.id,
|
||||
scope_type=scope_type,
|
||||
scope_id=scope_id,
|
||||
policy=payload.policy,
|
||||
)
|
||||
_record_connector_settings_change(
|
||||
session,
|
||||
collection=FILES_CONNECTOR_POLICIES_COLLECTION,
|
||||
resource_type=FILES_CONNECTOR_POLICY_RESOURCE,
|
||||
resource_id=_file_connector_policy_resource_id(scope_type, scope_id),
|
||||
operation="updated",
|
||||
principal=principal,
|
||||
tenant_id=None
|
||||
if scope_type.strip().casefold() == "system"
|
||||
else principal.tenant_id,
|
||||
payload={"scope_type": scope_type.strip().casefold(), "scope_id": scope_id},
|
||||
)
|
||||
session.commit()
|
||||
return FileConnectorPolicyResponse(
|
||||
**connector_policy_response(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
scope_type=scope_type,
|
||||
scope_id=scope_id,
|
||||
)
|
||||
)
|
||||
except (FileStorageError, ValueError, json.JSONDecodeError) as exc:
|
||||
session.rollback()
|
||||
raise _http_error(exc) from exc
|
||||
|
||||
|
||||
@router.post(
|
||||
"/connectors/credentials",
|
||||
response_model=FileConnectorCredentialResponse,
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
)
|
||||
def create_connector_credential(
|
||||
payload: FileConnectorCredentialCreateRequest,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(
|
||||
require_any_scope(
|
||||
"files:file:admin", "system:settings:write", "admin:settings:write"
|
||||
)
|
||||
),
|
||||
):
|
||||
_require_connector_credential_write(principal, payload.scope_type)
|
||||
credentials = payload.credentials
|
||||
try:
|
||||
_ensure_connector_credential_configuration_allowed(
|
||||
session,
|
||||
principal,
|
||||
credential_id=payload.id,
|
||||
provider=payload.provider,
|
||||
scope_type=payload.scope_type,
|
||||
scope_id=payload.scope_id,
|
||||
operation="configure_credentials",
|
||||
)
|
||||
_ensure_connector_local_policy_allowed(
|
||||
session,
|
||||
principal,
|
||||
scope_type=payload.scope_type,
|
||||
scope_id=payload.scope_id,
|
||||
policy=payload.policy,
|
||||
)
|
||||
row = create_connector_credential_row(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
user_id=principal.user.id,
|
||||
credential_id=payload.id,
|
||||
label=payload.label,
|
||||
provider=payload.provider,
|
||||
scope_type=payload.scope_type,
|
||||
scope_id=payload.scope_id,
|
||||
enabled=payload.enabled,
|
||||
credential_mode=payload.credential_mode,
|
||||
username=credentials.username,
|
||||
password=credentials.password,
|
||||
token=credentials.token,
|
||||
password_env=credentials.password_env,
|
||||
token_env=credentials.token_env,
|
||||
secret_ref=credentials.secret_ref,
|
||||
policy=payload.policy,
|
||||
metadata=payload.metadata,
|
||||
)
|
||||
_record_connector_settings_change(
|
||||
session,
|
||||
collection=FILES_CONNECTOR_CREDENTIALS_COLLECTION,
|
||||
resource_type=FILES_CONNECTOR_CREDENTIAL_RESOURCE,
|
||||
resource_id=row.id,
|
||||
operation="created",
|
||||
principal=principal,
|
||||
tenant_id=row.tenant_id,
|
||||
payload={
|
||||
"scope_type": row.scope_type,
|
||||
"scope_id": row.scope_id,
|
||||
"provider": row.provider,
|
||||
},
|
||||
)
|
||||
session.commit()
|
||||
session.refresh(row)
|
||||
return _connector_credential_response(row)
|
||||
except ConnectorPolicyDenied as exc:
|
||||
session.rollback()
|
||||
raise _connector_policy_error(exc) from exc
|
||||
except (FileStorageError, ValueError, json.JSONDecodeError) as exc:
|
||||
session.rollback()
|
||||
raise _http_error(exc) from exc
|
||||
|
||||
|
||||
@router.get(
|
||||
"/connectors/credentials/{credential_id}",
|
||||
response_model=FileConnectorCredentialResponse,
|
||||
)
|
||||
def get_connector_credential(
|
||||
credential_id: str,
|
||||
include_disabled: bool = False,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(
|
||||
require_any_scope(
|
||||
"files:file:admin", "system:settings:read", "admin:settings:read"
|
||||
)
|
||||
),
|
||||
):
|
||||
try:
|
||||
row = get_connector_credential_row(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
credential_id=credential_id,
|
||||
include_disabled=include_disabled
|
||||
and _can_read_disabled_connector_profiles(principal),
|
||||
)
|
||||
return _connector_credential_response(row)
|
||||
except FileStorageError as exc:
|
||||
raise _http_error(exc, not_found=True) from exc
|
||||
|
||||
|
||||
@router.patch(
|
||||
"/connectors/credentials/{credential_id}",
|
||||
response_model=FileConnectorCredentialResponse,
|
||||
)
|
||||
def update_connector_credential(
|
||||
credential_id: str,
|
||||
payload: FileConnectorCredentialUpdateRequest,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(
|
||||
require_any_scope(
|
||||
"files:file:admin", "system:settings:write", "admin:settings:write"
|
||||
)
|
||||
),
|
||||
):
|
||||
try:
|
||||
row = get_connector_credential_row(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
credential_id=credential_id,
|
||||
include_disabled=True,
|
||||
)
|
||||
except FileStorageError as exc:
|
||||
raise _http_error(exc, not_found=True) from exc
|
||||
_require_connector_credential_write(principal, row.scope_type)
|
||||
credentials = payload.credentials
|
||||
try:
|
||||
provider = payload.provider if payload.provider is not None else row.provider
|
||||
_ensure_connector_credential_configuration_allowed(
|
||||
session,
|
||||
principal,
|
||||
credential_id=row.id,
|
||||
provider=provider,
|
||||
scope_type=row.scope_type,
|
||||
scope_id=row.scope_id,
|
||||
operation="configure_credentials",
|
||||
)
|
||||
if payload.policy is not None:
|
||||
_ensure_connector_local_policy_allowed(
|
||||
session,
|
||||
principal,
|
||||
scope_type=row.scope_type,
|
||||
scope_id=row.scope_id,
|
||||
policy=payload.policy,
|
||||
)
|
||||
update_connector_credential_row(
|
||||
session,
|
||||
row,
|
||||
user_id=principal.user.id,
|
||||
label=payload.label,
|
||||
provider=payload.provider,
|
||||
enabled=payload.enabled,
|
||||
credential_mode=payload.credential_mode,
|
||||
username=credentials.username if credentials else None,
|
||||
password=credentials.password if credentials else None,
|
||||
token=credentials.token if credentials else None,
|
||||
password_env=credentials.password_env if credentials else None,
|
||||
token_env=credentials.token_env if credentials else None,
|
||||
secret_ref=credentials.secret_ref if credentials else None,
|
||||
clear_password=payload.clear_password,
|
||||
clear_token=payload.clear_token,
|
||||
policy=payload.policy,
|
||||
metadata=payload.metadata,
|
||||
)
|
||||
_record_connector_settings_change(
|
||||
session,
|
||||
collection=FILES_CONNECTOR_CREDENTIALS_COLLECTION,
|
||||
resource_type=FILES_CONNECTOR_CREDENTIAL_RESOURCE,
|
||||
resource_id=row.id,
|
||||
operation="updated",
|
||||
principal=principal,
|
||||
tenant_id=row.tenant_id,
|
||||
payload={
|
||||
"scope_type": row.scope_type,
|
||||
"scope_id": row.scope_id,
|
||||
"provider": row.provider,
|
||||
},
|
||||
)
|
||||
session.commit()
|
||||
session.refresh(row)
|
||||
return _connector_credential_response(row)
|
||||
except ConnectorPolicyDenied as exc:
|
||||
session.rollback()
|
||||
raise _connector_policy_error(exc) from exc
|
||||
except (FileStorageError, ValueError, json.JSONDecodeError) as exc:
|
||||
session.rollback()
|
||||
raise _http_error(exc) from exc
|
||||
|
||||
|
||||
@router.delete(
|
||||
"/connectors/credentials/{credential_id}",
|
||||
response_model=FileConnectorCredentialResponse,
|
||||
)
|
||||
def deactivate_connector_credential(
|
||||
credential_id: str,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(
|
||||
require_any_scope(
|
||||
"files:file:admin", "system:settings:write", "admin:settings:write"
|
||||
)
|
||||
),
|
||||
):
|
||||
try:
|
||||
row = get_connector_credential_row(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
credential_id=credential_id,
|
||||
include_disabled=True,
|
||||
)
|
||||
except FileStorageError as exc:
|
||||
raise _http_error(exc, not_found=True) from exc
|
||||
_require_connector_credential_write(principal, row.scope_type)
|
||||
try:
|
||||
deletion = delete_connector_credential_row(
|
||||
session,
|
||||
row,
|
||||
deletion_reason="api_delete",
|
||||
user_id=principal.user.id,
|
||||
api_key_id=principal.api_key.id if principal.api_key else None,
|
||||
)
|
||||
if deletion.changed:
|
||||
_record_connector_settings_change(
|
||||
session,
|
||||
collection=FILES_CONNECTOR_CREDENTIALS_COLLECTION,
|
||||
resource_type=FILES_CONNECTOR_CREDENTIAL_RESOURCE,
|
||||
resource_id=row.id,
|
||||
operation="deleted",
|
||||
principal=principal,
|
||||
tenant_id=row.tenant_id,
|
||||
payload={
|
||||
"scope_type": row.scope_type,
|
||||
"scope_id": row.scope_id,
|
||||
"provider": row.provider,
|
||||
},
|
||||
)
|
||||
for profile in deletion.affected_profiles:
|
||||
_record_connector_settings_change(
|
||||
session,
|
||||
collection=FILES_CONNECTOR_PROFILES_COLLECTION,
|
||||
resource_type=FILES_CONNECTOR_PROFILE_RESOURCE,
|
||||
resource_id=profile.id,
|
||||
operation="updated",
|
||||
principal=principal,
|
||||
tenant_id=profile.tenant_id,
|
||||
payload={
|
||||
"scope_type": profile.scope_type,
|
||||
"scope_id": profile.scope_id,
|
||||
"provider": profile.provider,
|
||||
"reason": "credential_deleted",
|
||||
},
|
||||
)
|
||||
session.commit()
|
||||
session.refresh(row)
|
||||
return _connector_credential_response(row)
|
||||
except FileStorageError as exc:
|
||||
session.rollback()
|
||||
raise _http_error(exc) from exc
|
||||
except Exception:
|
||||
session.rollback()
|
||||
raise
|
||||
|
||||
|
||||
@router.get("/connectors/profiles", response_model=FileConnectorProfilesResponse)
|
||||
def list_connector_profiles(
|
||||
provider: str | None = None,
|
||||
campaign_id: str | None = None,
|
||||
include_disabled: bool = False,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(
|
||||
require_any_scope(
|
||||
"files:file:read",
|
||||
"files:file:upload",
|
||||
"files:file:download",
|
||||
"files:file:admin",
|
||||
"system:settings:read",
|
||||
"admin:settings:read",
|
||||
)
|
||||
),
|
||||
):
|
||||
try:
|
||||
if campaign_id:
|
||||
_ensure_campaign_file_access(session, principal, campaign_id)
|
||||
profiles = _visible_connector_profiles(
|
||||
session,
|
||||
principal,
|
||||
provider=provider,
|
||||
campaign_id=campaign_id,
|
||||
include_disabled=include_disabled
|
||||
and _can_read_disabled_connector_profiles(principal),
|
||||
include_admin_scopes=_can_read_disabled_connector_profiles(principal),
|
||||
include_effective_policy=False,
|
||||
)
|
||||
except (OSError, ValueError, json.JSONDecodeError) as exc:
|
||||
raise _http_error(exc) from exc
|
||||
return FileConnectorProfilesResponse(
|
||||
profiles=[
|
||||
FileConnectorProfileResponse(**profile.to_response())
|
||||
for profile in profiles
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/connectors/profiles",
|
||||
response_model=FileConnectorProfileResponse,
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
)
|
||||
def create_connector_profile(
|
||||
payload: FileConnectorProfileCreateRequest,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(
|
||||
require_any_scope(
|
||||
"files:file:admin", "system:settings:write", "admin:settings:write"
|
||||
)
|
||||
),
|
||||
):
|
||||
_require_connector_profile_write(principal, payload.scope_type)
|
||||
credentials = payload.credentials
|
||||
try:
|
||||
credential_row = _credential_row_for_profile(
|
||||
session,
|
||||
principal,
|
||||
credential_profile_id=payload.credential_profile_id,
|
||||
provider=payload.provider,
|
||||
profile_id=payload.id,
|
||||
scope_type=payload.scope_type,
|
||||
scope_id=payload.scope_id,
|
||||
)
|
||||
_ensure_connector_configuration_allowed(
|
||||
session,
|
||||
principal,
|
||||
connector_id=payload.id,
|
||||
credential_id=payload.credential_profile_id,
|
||||
provider=payload.provider,
|
||||
endpoint_url=connector_effective_endpoint_url(
|
||||
provider=payload.provider,
|
||||
endpoint_url=payload.endpoint_url,
|
||||
metadata=payload.metadata,
|
||||
),
|
||||
base_path=payload.base_path,
|
||||
scope_type=payload.scope_type,
|
||||
scope_id=payload.scope_id,
|
||||
operation="configure",
|
||||
)
|
||||
_ensure_connector_local_policy_allowed(
|
||||
session,
|
||||
principal,
|
||||
scope_type=payload.scope_type,
|
||||
scope_id=payload.scope_id,
|
||||
policy=payload.policy,
|
||||
)
|
||||
row = create_connector_profile_row(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
user_id=principal.user.id,
|
||||
profile_id=payload.id,
|
||||
label=payload.label,
|
||||
provider=payload.provider,
|
||||
scope_type=payload.scope_type,
|
||||
scope_id=payload.scope_id,
|
||||
endpoint_url=payload.endpoint_url,
|
||||
base_path=payload.base_path,
|
||||
enabled=payload.enabled,
|
||||
credential_profile_id=payload.credential_profile_id,
|
||||
credential_mode=payload.credential_mode,
|
||||
username=credentials.username,
|
||||
password=credentials.password,
|
||||
token=credentials.token,
|
||||
password_env=credentials.password_env,
|
||||
token_env=credentials.token_env,
|
||||
secret_ref=credentials.secret_ref,
|
||||
capabilities=payload.capabilities,
|
||||
policy=payload.policy,
|
||||
metadata=payload.metadata,
|
||||
)
|
||||
_record_connector_settings_change(
|
||||
session,
|
||||
collection=FILES_CONNECTOR_PROFILES_COLLECTION,
|
||||
resource_type=FILES_CONNECTOR_PROFILE_RESOURCE,
|
||||
resource_id=row.id,
|
||||
operation="created",
|
||||
principal=principal,
|
||||
tenant_id=row.tenant_id,
|
||||
payload={
|
||||
"scope_type": row.scope_type,
|
||||
"scope_id": row.scope_id,
|
||||
"provider": row.provider,
|
||||
},
|
||||
)
|
||||
session.commit()
|
||||
session.refresh(row)
|
||||
return FileConnectorProfileResponse(
|
||||
**connector_profile_from_row(
|
||||
row, credential_row=credential_row
|
||||
).to_response()
|
||||
)
|
||||
except ConnectorPolicyDenied as exc:
|
||||
session.rollback()
|
||||
raise _connector_policy_error(exc) from exc
|
||||
except (FileStorageError, ValueError, json.JSONDecodeError) as exc:
|
||||
session.rollback()
|
||||
raise _http_error(exc) from exc
|
||||
@@ -0,0 +1,151 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Literal
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from govoplan_core.auth import ApiPrincipal, require_scope
|
||||
from govoplan_files.backend.schemas import (
|
||||
FileFolderCreateRequest,
|
||||
FileFolderDeleteRequest,
|
||||
FileFolderDeleteResponse,
|
||||
FileFolderResponse,
|
||||
FileFoldersResponse,
|
||||
)
|
||||
from govoplan_core.db.session import get_session
|
||||
from govoplan_files.backend.storage.paths import UnsafeFilePathError
|
||||
from govoplan_files.backend.storage.common import FileStorageError
|
||||
from govoplan_files.backend.storage.folders import (
|
||||
create_folder,
|
||||
list_folders_for_user,
|
||||
list_folders_for_user_window,
|
||||
soft_delete_folder,
|
||||
)
|
||||
|
||||
|
||||
from govoplan_files.backend.route_support import (
|
||||
_folder_response,
|
||||
_http_error,
|
||||
_is_admin,
|
||||
)
|
||||
from govoplan_files.backend.services.list_queries import (
|
||||
FOLDERS_LIST_CURSOR_SCOPE,
|
||||
_cursor_page_size,
|
||||
_files_delta_watermark,
|
||||
_folder_cursor_values,
|
||||
_folders_list_fingerprint,
|
||||
_next_folder_list_cursor,
|
||||
)
|
||||
|
||||
router = APIRouter(prefix="/files", tags=["files"])
|
||||
|
||||
|
||||
@router.get("/folders", response_model=FileFoldersResponse)
|
||||
def list_file_folders(
|
||||
owner_type: Literal["user", "group"],
|
||||
owner_id: str,
|
||||
page_size: int | None = Query(default=None, ge=1, le=1000),
|
||||
cursor: str | None = None,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(require_scope("files:file:read")),
|
||||
):
|
||||
try:
|
||||
watermark = _files_delta_watermark(session, principal.tenant_id)
|
||||
effective_page_size = _cursor_page_size(
|
||||
FOLDERS_LIST_CURSOR_SCOPE, cursor, page_size
|
||||
)
|
||||
if effective_page_size is None:
|
||||
folders = list_folders_for_user(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
user_id=principal.user.id,
|
||||
owner_type=owner_type,
|
||||
owner_id=owner_id,
|
||||
is_admin=_is_admin(principal),
|
||||
)
|
||||
return FileFoldersResponse(
|
||||
folders=[_folder_response(folder) for folder in folders],
|
||||
watermark=watermark,
|
||||
)
|
||||
fingerprint = _folders_list_fingerprint(
|
||||
principal,
|
||||
owner_type=owner_type,
|
||||
owner_id=owner_id,
|
||||
page_size=effective_page_size,
|
||||
)
|
||||
after_path, after_id = _folder_cursor_values(cursor, fingerprint=fingerprint)
|
||||
folders, has_more = list_folders_for_user_window(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
user_id=principal.user.id,
|
||||
owner_type=owner_type,
|
||||
owner_id=owner_id,
|
||||
is_admin=_is_admin(principal),
|
||||
page_size=effective_page_size,
|
||||
after_path=after_path,
|
||||
after_id=after_id,
|
||||
)
|
||||
return FileFoldersResponse(
|
||||
folders=[_folder_response(folder) for folder in folders],
|
||||
cursor=cursor,
|
||||
next_cursor=_next_folder_list_cursor(
|
||||
principal,
|
||||
folders,
|
||||
owner_type=owner_type,
|
||||
owner_id=owner_id,
|
||||
page_size=effective_page_size,
|
||||
has_more=has_more,
|
||||
),
|
||||
watermark=watermark,
|
||||
)
|
||||
except FileStorageError as exc:
|
||||
raise _http_error(exc) from exc
|
||||
|
||||
|
||||
@router.post("/folders", response_model=FileFolderResponse)
|
||||
def create_file_folder(
|
||||
payload: FileFolderCreateRequest,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(require_scope("files:file:organize")),
|
||||
):
|
||||
try:
|
||||
folder = create_folder(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
owner_type=payload.owner_type,
|
||||
owner_id=payload.owner_id,
|
||||
user_id=principal.user.id,
|
||||
path=payload.path,
|
||||
is_admin=_is_admin(principal),
|
||||
)
|
||||
session.commit()
|
||||
return _folder_response(folder)
|
||||
except (FileStorageError, UnsafeFilePathError, ValueError) as exc:
|
||||
session.rollback()
|
||||
raise _http_error(exc) from exc
|
||||
|
||||
|
||||
@router.post("/folders/delete", response_model=FileFolderDeleteResponse)
|
||||
def delete_file_folder(
|
||||
payload: FileFolderDeleteRequest,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(require_scope("files:file:delete")),
|
||||
):
|
||||
try:
|
||||
deleted_folders, deleted_files = soft_delete_folder(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
owner_type=payload.owner_type,
|
||||
owner_id=payload.owner_id,
|
||||
user_id=principal.user.id,
|
||||
path=payload.path,
|
||||
recursive=payload.recursive,
|
||||
is_admin=_is_admin(principal),
|
||||
)
|
||||
session.commit()
|
||||
return FileFolderDeleteResponse(
|
||||
deleted_folders=deleted_folders, deleted_files=deleted_files
|
||||
)
|
||||
except (FileStorageError, UnsafeFilePathError, ValueError) as exc:
|
||||
session.rollback()
|
||||
raise _http_error(exc) from exc
|
||||
@@ -0,0 +1,172 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
import hashlib
|
||||
|
||||
from fastapi import (
|
||||
APIRouter,
|
||||
Depends,
|
||||
File as FastAPIFile,
|
||||
Header,
|
||||
HTTPException,
|
||||
UploadFile,
|
||||
status,
|
||||
)
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from govoplan_core.core.events import (
|
||||
EventActorRef,
|
||||
EventObjectRef,
|
||||
EventTenantRef,
|
||||
PlatformEvent,
|
||||
emit_platform_event,
|
||||
)
|
||||
from govoplan_core.core.institutional import EvidenceReference
|
||||
from govoplan_core.db.session import get_session
|
||||
from govoplan_files.backend.db.models import FileFormEvidenceGrant
|
||||
from govoplan_files.backend.route_support import _read_limited_upload
|
||||
from govoplan_files.backend.storage.common import FileStorageError
|
||||
from govoplan_files.backend.storage.files import create_file_asset
|
||||
|
||||
|
||||
router = APIRouter(prefix="/files", tags=["files"])
|
||||
|
||||
|
||||
@router.post(
|
||||
"/form-evidence/upload",
|
||||
response_model=dict[str, object],
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
)
|
||||
def upload_form_evidence(
|
||||
file: UploadFile = FastAPIFile(...),
|
||||
x_form_evidence_token: str = Header(alias="X-Form-Evidence-Token"),
|
||||
session: Session = Depends(get_session),
|
||||
) -> dict[str, object]:
|
||||
now = datetime.now(UTC)
|
||||
try:
|
||||
grant = (
|
||||
session.query(FileFormEvidenceGrant)
|
||||
.filter(
|
||||
FileFormEvidenceGrant.token_sha256
|
||||
== _token_sha256(x_form_evidence_token)
|
||||
)
|
||||
.with_for_update()
|
||||
.one_or_none()
|
||||
)
|
||||
if grant is None or grant.status != "issued" or _aware(grant.expires_at) <= now:
|
||||
raise _upload_unavailable()
|
||||
content_type = (
|
||||
str(file.content_type or "application/octet-stream")
|
||||
.split(";", 1)[0]
|
||||
.strip()
|
||||
.casefold()
|
||||
)
|
||||
if grant.allowed_content_types and content_type not in set(
|
||||
grant.allowed_content_types
|
||||
):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_415_UNSUPPORTED_MEDIA_TYPE,
|
||||
detail="This Form does not accept the uploaded document type.",
|
||||
)
|
||||
data = _read_limited_upload(file, max_bytes=grant.max_size_bytes)
|
||||
stored = create_file_asset(
|
||||
session,
|
||||
tenant_id=grant.tenant_id,
|
||||
owner_type="user",
|
||||
owner_id=grant.custodian_user_id,
|
||||
user_id=grant.custodian_user_id,
|
||||
filename=file.filename or "form-attachment",
|
||||
data=data,
|
||||
folder=f"Form submissions/{grant.form_instance_id}",
|
||||
content_type=content_type,
|
||||
description="Managed attachment captured through Forms Runtime.",
|
||||
metadata={
|
||||
"form_evidence": {
|
||||
"grant_id": grant.id,
|
||||
"form_instance_id": grant.form_instance_id,
|
||||
"form_definition_id": grant.form_definition_id,
|
||||
"form_definition_revision": grant.form_definition_revision,
|
||||
"purpose": grant.purpose,
|
||||
},
|
||||
"source_provenance": {
|
||||
"source_type": "form_evidence",
|
||||
"connector_id": "forms_runtime.evidence.files",
|
||||
"provider": "forms_runtime",
|
||||
"external_id": grant.id,
|
||||
"revision": grant.form_definition_revision,
|
||||
},
|
||||
},
|
||||
conflict_strategy="rename",
|
||||
is_admin=True,
|
||||
)
|
||||
grant.status = "uploaded"
|
||||
grant.file_asset_id = stored.asset.id
|
||||
grant.file_version_id = stored.version.id
|
||||
grant.uploaded_at = now
|
||||
session.add(grant)
|
||||
evidence = EvidenceReference(
|
||||
kind="document",
|
||||
owner_module="files",
|
||||
evidence_id=stored.asset.id,
|
||||
tenant_id=grant.tenant_id,
|
||||
version=stored.version.id,
|
||||
checksum=stored.version.checksum_sha256,
|
||||
source_ref=(f"form_submission:{grant.form_instance_id}:grant:{grant.id}"),
|
||||
responsible_actor_ref=f"user:{grant.custodian_user_id}",
|
||||
captured_at=now,
|
||||
)
|
||||
emit_platform_event(
|
||||
session,
|
||||
PlatformEvent(
|
||||
type="files.form_evidence.uploaded",
|
||||
module_id="files",
|
||||
payload={
|
||||
"grant_id": grant.id,
|
||||
"form_instance_id": grant.form_instance_id,
|
||||
"file_asset_id": stored.asset.id,
|
||||
"file_version_id": stored.version.id,
|
||||
"checksum_sha256": stored.version.checksum_sha256,
|
||||
"size_bytes": stored.version.size_bytes,
|
||||
},
|
||||
occurred_at=now,
|
||||
actor=EventActorRef(type="user", id=grant.custodian_user_id),
|
||||
tenant=EventTenantRef(id=grant.tenant_id),
|
||||
resource=EventObjectRef(type="file", id=stored.asset.id),
|
||||
classification="confidential",
|
||||
),
|
||||
)
|
||||
session.commit()
|
||||
except HTTPException:
|
||||
session.rollback()
|
||||
raise
|
||||
except (FileStorageError, ValueError) as exc:
|
||||
session.rollback()
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
|
||||
detail=str(exc),
|
||||
) from exc
|
||||
return {
|
||||
"grant_id": grant.id,
|
||||
"evidence": evidence.to_dict(include_inspection=True),
|
||||
}
|
||||
|
||||
|
||||
def _token_sha256(token: str) -> str:
|
||||
clean = str(token or "").strip()
|
||||
if len(clean) < 32:
|
||||
raise _upload_unavailable()
|
||||
return hashlib.sha256(clean.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _upload_unavailable() -> HTTPException:
|
||||
return HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail="The Form evidence upload is unavailable.",
|
||||
)
|
||||
|
||||
|
||||
def _aware(value: datetime) -> datetime:
|
||||
return value if value.tzinfo is not None else value.replace(tzinfo=UTC)
|
||||
|
||||
|
||||
__all__ = ["router", "upload_form_evidence"]
|
||||
@@ -0,0 +1,387 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, status
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from govoplan_core.auth import ApiPrincipal, require_scope
|
||||
from govoplan_core.audit.logging import audit_from_principal
|
||||
from govoplan_core.db.session import get_session
|
||||
from govoplan_files.backend.db.models import (
|
||||
FileIntegrityFinding,
|
||||
FileIntegrityScan,
|
||||
)
|
||||
from govoplan_files.backend.schemas import (
|
||||
FileIntegrityActionRequest,
|
||||
FileIntegrityActionResponse,
|
||||
FileIntegrityFindingResponse,
|
||||
FileIntegrityFindingsResponse,
|
||||
FileIntegrityScanCreateRequest,
|
||||
FileIntegrityScanRunRequest,
|
||||
FileIntegrityScanResponse,
|
||||
FileIntegrityScansResponse,
|
||||
)
|
||||
from govoplan_files.backend.storage.backends import StorageBackendError
|
||||
from govoplan_files.backend.storage.common import FileStorageError
|
||||
from govoplan_files.backend.storage.integrity import (
|
||||
cleanup_orphan_finding,
|
||||
create_integrity_scan,
|
||||
mark_integrity_scan_failed,
|
||||
recheck_integrity_finding,
|
||||
run_integrity_scan_batch,
|
||||
)
|
||||
|
||||
|
||||
router = APIRouter(prefix="/files/integrity", tags=["files-integrity"])
|
||||
|
||||
|
||||
@router.get("/scans", response_model=FileIntegrityScansResponse)
|
||||
def list_integrity_scans(
|
||||
limit: int = Query(default=50, ge=1, le=200),
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(require_scope("files:file:admin")),
|
||||
) -> FileIntegrityScansResponse:
|
||||
rows = (
|
||||
session.query(FileIntegrityScan)
|
||||
.filter(FileIntegrityScan.tenant_id == principal.tenant_id)
|
||||
.order_by(FileIntegrityScan.created_at.desc(), FileIntegrityScan.id.desc())
|
||||
.limit(limit)
|
||||
.all()
|
||||
)
|
||||
return FileIntegrityScansResponse(
|
||||
scans=[_scan_response(row) for row in rows]
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/scans",
|
||||
response_model=FileIntegrityScanResponse,
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
)
|
||||
def create_scan(
|
||||
payload: FileIntegrityScanCreateRequest,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(require_scope("files:file:admin")),
|
||||
) -> FileIntegrityScanResponse:
|
||||
try:
|
||||
scan = create_integrity_scan(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
user_id=principal.user.id,
|
||||
verify_checksums=payload.verify_checksums,
|
||||
batch_size=payload.batch_size,
|
||||
)
|
||||
audit_from_principal(
|
||||
session,
|
||||
principal,
|
||||
action="files.integrity.scan_created",
|
||||
object_type="file_integrity_scan",
|
||||
object_id=scan.id,
|
||||
details={
|
||||
"storage_backend": scan.storage_backend,
|
||||
"verify_checksums": scan.verify_checksums,
|
||||
"batch_size": scan.batch_size,
|
||||
},
|
||||
)
|
||||
session.commit()
|
||||
return _scan_response(scan)
|
||||
except (FileStorageError, StorageBackendError) as exc:
|
||||
session.rollback()
|
||||
raise _integrity_http_error(exc) from exc
|
||||
|
||||
|
||||
@router.post("/scans/{scan_id}/run", response_model=FileIntegrityScanResponse)
|
||||
def run_scan_batch(
|
||||
scan_id: str,
|
||||
payload: FileIntegrityScanRunRequest,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(require_scope("files:file:admin")),
|
||||
) -> FileIntegrityScanResponse:
|
||||
scan = _scan_for_tenant(
|
||||
session,
|
||||
scan_id,
|
||||
principal.tenant_id,
|
||||
for_update=True,
|
||||
)
|
||||
_assert_expected_revision(scan.revision, payload.expected_revision)
|
||||
previous_status = scan.status
|
||||
try:
|
||||
run_integrity_scan_batch(session, scan)
|
||||
if scan.status == "completed" and previous_status != "completed":
|
||||
audit_from_principal(
|
||||
session,
|
||||
principal,
|
||||
action="files.integrity.scan_completed",
|
||||
object_type="file_integrity_scan",
|
||||
object_id=scan.id,
|
||||
details={
|
||||
"verified_blob_count": scan.verified_blob_count,
|
||||
"quarantined_blob_count": scan.quarantined_blob_count,
|
||||
"orphan_object_count": scan.orphan_object_count,
|
||||
},
|
||||
)
|
||||
session.commit()
|
||||
return _scan_response(scan)
|
||||
except (FileStorageError, StorageBackendError) as exc:
|
||||
session.rollback()
|
||||
scan = _scan_for_tenant(
|
||||
session,
|
||||
scan_id,
|
||||
principal.tenant_id,
|
||||
for_update=True,
|
||||
)
|
||||
mark_integrity_scan_failed(scan, error=exc)
|
||||
audit_from_principal(
|
||||
session,
|
||||
principal,
|
||||
action="files.integrity.scan_failed",
|
||||
object_type="file_integrity_scan",
|
||||
object_id=scan.id,
|
||||
details={"error_type": type(exc).__name__},
|
||||
)
|
||||
session.commit()
|
||||
raise _integrity_http_error(exc) from exc
|
||||
|
||||
|
||||
@router.get(
|
||||
"/scans/{scan_id}/findings",
|
||||
response_model=FileIntegrityFindingsResponse,
|
||||
)
|
||||
def list_integrity_findings(
|
||||
scan_id: str,
|
||||
state_filter: str | None = Query(default=None, alias="state"),
|
||||
limit: int = Query(default=500, ge=1, le=1000),
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(require_scope("files:file:admin")),
|
||||
) -> FileIntegrityFindingsResponse:
|
||||
scan = _scan_for_tenant(session, scan_id, principal.tenant_id)
|
||||
query = session.query(FileIntegrityFinding).filter(
|
||||
FileIntegrityFinding.scan_id == scan.id,
|
||||
FileIntegrityFinding.tenant_id == principal.tenant_id,
|
||||
)
|
||||
if state_filter:
|
||||
query = query.filter(FileIntegrityFinding.state == state_filter)
|
||||
rows = (
|
||||
query.order_by(
|
||||
FileIntegrityFinding.created_at.asc(),
|
||||
FileIntegrityFinding.id.asc(),
|
||||
)
|
||||
.limit(limit)
|
||||
.all()
|
||||
)
|
||||
return FileIntegrityFindingsResponse(
|
||||
findings=[_finding_response(row) for row in rows]
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/findings/{finding_id}/recheck",
|
||||
response_model=FileIntegrityActionResponse,
|
||||
)
|
||||
def recheck_finding(
|
||||
finding_id: str,
|
||||
payload: FileIntegrityActionRequest,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(require_scope("files:file:admin")),
|
||||
) -> FileIntegrityActionResponse:
|
||||
finding = _finding_for_tenant(
|
||||
session,
|
||||
finding_id,
|
||||
principal.tenant_id,
|
||||
for_update=True,
|
||||
)
|
||||
_assert_expected_revision(finding.revision, payload.expected_revision)
|
||||
try:
|
||||
result = recheck_integrity_finding(
|
||||
session,
|
||||
finding,
|
||||
user_id=principal.user.id,
|
||||
dry_run=payload.dry_run,
|
||||
)
|
||||
_audit_integrity_action(session, principal, result)
|
||||
session.commit()
|
||||
return _action_response(result)
|
||||
except (FileStorageError, StorageBackendError) as exc:
|
||||
session.rollback()
|
||||
raise _integrity_http_error(exc) from exc
|
||||
|
||||
|
||||
@router.post(
|
||||
"/findings/{finding_id}/cleanup",
|
||||
response_model=FileIntegrityActionResponse,
|
||||
)
|
||||
def cleanup_finding(
|
||||
finding_id: str,
|
||||
payload: FileIntegrityActionRequest,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(require_scope("files:file:admin")),
|
||||
) -> FileIntegrityActionResponse:
|
||||
finding = _finding_for_tenant(
|
||||
session,
|
||||
finding_id,
|
||||
principal.tenant_id,
|
||||
for_update=True,
|
||||
)
|
||||
_assert_expected_revision(finding.revision, payload.expected_revision)
|
||||
try:
|
||||
result = cleanup_orphan_finding(
|
||||
session,
|
||||
finding,
|
||||
user_id=principal.user.id,
|
||||
dry_run=payload.dry_run,
|
||||
)
|
||||
_audit_integrity_action(session, principal, result)
|
||||
session.commit()
|
||||
return _action_response(result)
|
||||
except (FileStorageError, StorageBackendError) as exc:
|
||||
session.rollback()
|
||||
raise _integrity_http_error(exc) from exc
|
||||
|
||||
|
||||
def _scan_for_tenant(
|
||||
session: Session,
|
||||
scan_id: str,
|
||||
tenant_id: str,
|
||||
*,
|
||||
for_update: bool = False,
|
||||
) -> FileIntegrityScan:
|
||||
query = session.query(FileIntegrityScan).filter(FileIntegrityScan.id == scan_id)
|
||||
if for_update:
|
||||
query = query.populate_existing().with_for_update()
|
||||
scan = query.one_or_none()
|
||||
if scan is None or scan.tenant_id != tenant_id:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail="Integrity scan not found",
|
||||
)
|
||||
return scan
|
||||
|
||||
|
||||
def _finding_for_tenant(
|
||||
session: Session,
|
||||
finding_id: str,
|
||||
tenant_id: str,
|
||||
*,
|
||||
for_update: bool = False,
|
||||
) -> FileIntegrityFinding:
|
||||
query = session.query(FileIntegrityFinding).filter(
|
||||
FileIntegrityFinding.id == finding_id
|
||||
)
|
||||
if for_update:
|
||||
query = query.populate_existing().with_for_update()
|
||||
finding = query.one_or_none()
|
||||
if finding is None or finding.tenant_id != tenant_id:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail="Integrity finding not found",
|
||||
)
|
||||
return finding
|
||||
|
||||
|
||||
def _assert_expected_revision(current: int, expected: int) -> None:
|
||||
if current != expected:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_409_CONFLICT,
|
||||
detail=(
|
||||
"The integrity record changed after it was loaded; reload before "
|
||||
"performing this action."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _audit_integrity_action(session, principal, result) -> None:
|
||||
audit_from_principal(
|
||||
session,
|
||||
principal,
|
||||
action=f"files.integrity.{result.action}",
|
||||
object_type="file_integrity_finding",
|
||||
object_id=result.finding.id,
|
||||
details={
|
||||
"scan_id": result.finding.scan_id,
|
||||
"kind": result.finding.kind,
|
||||
"dry_run": result.dry_run,
|
||||
"changed": result.changed,
|
||||
"storage_key_sha256": hashlib.sha256(
|
||||
result.finding.storage_key.encode("utf-8")
|
||||
).hexdigest(),
|
||||
"inspection": result.inspection.kind
|
||||
if result.inspection
|
||||
else None,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _scan_response(scan: FileIntegrityScan) -> FileIntegrityScanResponse:
|
||||
return FileIntegrityScanResponse(
|
||||
id=scan.id,
|
||||
tenant_id=scan.tenant_id,
|
||||
storage_backend=scan.storage_backend,
|
||||
storage_prefix=scan.storage_prefix,
|
||||
status=scan.status,
|
||||
revision=scan.revision,
|
||||
phase=scan.phase,
|
||||
verify_checksums=scan.verify_checksums,
|
||||
batch_size=scan.batch_size,
|
||||
scanned_blob_count=scan.scanned_blob_count,
|
||||
verified_blob_count=scan.verified_blob_count,
|
||||
quarantined_blob_count=scan.quarantined_blob_count,
|
||||
scanned_object_count=scan.scanned_object_count,
|
||||
orphan_object_count=scan.orphan_object_count,
|
||||
created_by_user_id=scan.created_by_user_id,
|
||||
started_at=scan.started_at.isoformat() if scan.started_at else None,
|
||||
completed_at=scan.completed_at.isoformat()
|
||||
if scan.completed_at
|
||||
else None,
|
||||
last_error=scan.last_error,
|
||||
created_at=scan.created_at.isoformat(),
|
||||
updated_at=scan.updated_at.isoformat(),
|
||||
)
|
||||
|
||||
|
||||
def _finding_response(
|
||||
finding: FileIntegrityFinding,
|
||||
) -> FileIntegrityFindingResponse:
|
||||
return FileIntegrityFindingResponse(
|
||||
id=finding.id,
|
||||
scan_id=finding.scan_id,
|
||||
tenant_id=finding.tenant_id,
|
||||
kind=finding.kind,
|
||||
state=finding.state,
|
||||
revision=finding.revision,
|
||||
blob_id=finding.blob_id,
|
||||
storage_key=finding.storage_key,
|
||||
expected_size_bytes=finding.expected_size_bytes,
|
||||
observed_size_bytes=finding.observed_size_bytes,
|
||||
expected_checksum_sha256=finding.expected_checksum_sha256,
|
||||
observed_checksum_sha256=finding.observed_checksum_sha256,
|
||||
resolved_at=finding.resolved_at.isoformat()
|
||||
if finding.resolved_at
|
||||
else None,
|
||||
resolved_by_user_id=finding.resolved_by_user_id,
|
||||
created_at=finding.created_at.isoformat(),
|
||||
updated_at=finding.updated_at.isoformat(),
|
||||
)
|
||||
|
||||
|
||||
def _action_response(result) -> FileIntegrityActionResponse:
|
||||
return FileIntegrityActionResponse(
|
||||
action=result.action,
|
||||
changed=result.changed,
|
||||
dry_run=result.dry_run,
|
||||
finding=_finding_response(result.finding),
|
||||
inspection_kind=result.inspection.kind if result.inspection else None,
|
||||
inspection_valid=result.inspection.valid if result.inspection else None,
|
||||
)
|
||||
|
||||
|
||||
def _integrity_http_error(exc: Exception) -> HTTPException:
|
||||
if isinstance(exc, FileStorageError):
|
||||
return HTTPException(
|
||||
status_code=status.HTTP_409_CONFLICT,
|
||||
detail=str(exc),
|
||||
)
|
||||
return HTTPException(
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
detail="The configured file storage backend could not complete the integrity operation",
|
||||
)
|
||||
@@ -0,0 +1,297 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from govoplan_core.auth import ApiPrincipal, require_scope
|
||||
from govoplan_core.audit.logging import audit_from_principal
|
||||
from govoplan_core.db.session import get_session
|
||||
from govoplan_files.backend.route_support import (
|
||||
_asset_response,
|
||||
_connector_space_response,
|
||||
_http_error,
|
||||
_is_admin,
|
||||
)
|
||||
from govoplan_files.backend.schemas import (
|
||||
FileAssetResponse,
|
||||
FileBlobGcRequest,
|
||||
FileBlobGcResponse,
|
||||
FileConnectorSpaceResponse,
|
||||
FileFolderDeleteRequest,
|
||||
FileFolderRestoreResponse,
|
||||
FileLifecycleUpdateRequest,
|
||||
FilePurgeExecuteRequest,
|
||||
FilePurgePreviewItem,
|
||||
FilePurgePreviewRequest,
|
||||
FilePurgePreviewResponse,
|
||||
FilePurgeResponse,
|
||||
FileRestoreResponse,
|
||||
)
|
||||
from govoplan_files.backend.storage.common import FileStorageError
|
||||
from govoplan_files.backend.storage.lifecycle import (
|
||||
execute_asset_purge,
|
||||
garbage_collect_unreferenced_blobs,
|
||||
get_asset_for_lifecycle,
|
||||
preview_asset_purge,
|
||||
restore_asset,
|
||||
restore_connector_space,
|
||||
restore_folder,
|
||||
set_asset_lifecycle,
|
||||
)
|
||||
|
||||
|
||||
router = APIRouter(prefix="/files", tags=["files-lifecycle"])
|
||||
|
||||
|
||||
@router.patch("/{file_id}/lifecycle", response_model=FileAssetResponse)
|
||||
def update_file_lifecycle(
|
||||
file_id: str,
|
||||
payload: FileLifecycleUpdateRequest,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(require_scope("files:file:retention")),
|
||||
):
|
||||
try:
|
||||
asset = get_asset_for_lifecycle(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
user_id=principal.user.id,
|
||||
asset_id=file_id,
|
||||
is_admin=_is_admin(principal),
|
||||
)
|
||||
set_asset_lifecycle(
|
||||
session,
|
||||
asset,
|
||||
retained_until=payload.retained_until,
|
||||
legal_hold=payload.legal_hold,
|
||||
reason=payload.reason,
|
||||
expected_revision=payload.expected_revision,
|
||||
)
|
||||
audit_from_principal(
|
||||
session,
|
||||
principal,
|
||||
action="files.lifecycle.updated",
|
||||
object_type="file",
|
||||
object_id=asset.id,
|
||||
details={
|
||||
"retained_until": (
|
||||
asset.retained_until.isoformat() if asset.retained_until else None
|
||||
),
|
||||
"legal_hold": asset.legal_hold,
|
||||
"lifecycle_revision": asset.lifecycle_revision,
|
||||
"reason": asset.lifecycle_reason,
|
||||
},
|
||||
)
|
||||
session.commit()
|
||||
return _asset_response(session, asset, include_shares=True)
|
||||
except FileStorageError as exc:
|
||||
session.rollback()
|
||||
raise _http_error(exc, not_found=True) from exc
|
||||
|
||||
|
||||
@router.post("/assets/{file_id}/restore", response_model=FileRestoreResponse)
|
||||
def restore_file(
|
||||
file_id: str,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(require_scope("files:file:restore")),
|
||||
):
|
||||
try:
|
||||
asset = get_asset_for_lifecycle(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
user_id=principal.user.id,
|
||||
asset_id=file_id,
|
||||
is_admin=_is_admin(principal),
|
||||
)
|
||||
changed = restore_asset(session, asset)
|
||||
if changed:
|
||||
audit_from_principal(
|
||||
session,
|
||||
principal,
|
||||
action="files.file.restored",
|
||||
object_type="file",
|
||||
object_id=asset.id,
|
||||
details={"display_path": asset.display_path},
|
||||
)
|
||||
session.commit()
|
||||
return FileRestoreResponse(restored_count=1 if changed else 0)
|
||||
except FileStorageError as exc:
|
||||
session.rollback()
|
||||
raise _http_error(exc, not_found=True) from exc
|
||||
|
||||
|
||||
@router.post("/folders/restore", response_model=FileFolderRestoreResponse)
|
||||
def restore_file_folder(
|
||||
payload: FileFolderDeleteRequest,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(require_scope("files:file:restore")),
|
||||
):
|
||||
try:
|
||||
restored_folders, restored_files = restore_folder(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
owner_type=payload.owner_type,
|
||||
owner_id=payload.owner_id,
|
||||
user_id=principal.user.id,
|
||||
path=payload.path,
|
||||
recursive=payload.recursive,
|
||||
is_admin=_is_admin(principal),
|
||||
)
|
||||
audit_from_principal(
|
||||
session,
|
||||
principal,
|
||||
action="files.folder.restored",
|
||||
object_type="file_folder",
|
||||
object_id=payload.path,
|
||||
details={
|
||||
"restored_folders": restored_folders,
|
||||
"restored_files": restored_files,
|
||||
},
|
||||
)
|
||||
session.commit()
|
||||
return FileFolderRestoreResponse(
|
||||
restored_folders=restored_folders, restored_files=restored_files
|
||||
)
|
||||
except FileStorageError as exc:
|
||||
session.rollback()
|
||||
raise _http_error(exc, not_found=True) from exc
|
||||
|
||||
|
||||
@router.post(
|
||||
"/connector-spaces/{space_id}/restore",
|
||||
response_model=FileConnectorSpaceResponse,
|
||||
)
|
||||
def restore_file_connector_space(
|
||||
space_id: str,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(require_scope("files:file:restore")),
|
||||
):
|
||||
try:
|
||||
space = restore_connector_space(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
user_id=principal.user.id,
|
||||
space_id=space_id,
|
||||
is_admin=_is_admin(principal),
|
||||
)
|
||||
audit_from_principal(
|
||||
session,
|
||||
principal,
|
||||
action="files.connector_space.restored",
|
||||
object_type="file_connector_space",
|
||||
object_id=space.id,
|
||||
details={"label": space.label},
|
||||
)
|
||||
session.commit()
|
||||
return _connector_space_response(space)
|
||||
except FileStorageError as exc:
|
||||
session.rollback()
|
||||
raise _http_error(exc, not_found=True) from exc
|
||||
|
||||
|
||||
@router.post("/purge/preview", response_model=FilePurgePreviewResponse)
|
||||
def preview_file_purge(
|
||||
payload: FilePurgePreviewRequest,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(require_scope("files:file:purge")),
|
||||
):
|
||||
try:
|
||||
preview = preview_asset_purge(
|
||||
session, tenant_id=principal.tenant_id, file_ids=payload.file_ids
|
||||
)
|
||||
return FilePurgePreviewResponse(
|
||||
preview_sha256=preview.preview_sha256,
|
||||
eligible=preview.eligible,
|
||||
items=[
|
||||
FilePurgePreviewItem(
|
||||
file_id=item.file_id,
|
||||
filename=item.filename,
|
||||
lifecycle_revision=item.lifecycle_revision,
|
||||
deleted_at=(item.deleted_at.isoformat() if item.deleted_at else None),
|
||||
retained_until=(
|
||||
item.retained_until.isoformat() if item.retained_until else None
|
||||
),
|
||||
legal_hold=item.legal_hold,
|
||||
blockers=list(item.blockers),
|
||||
blob_ids=list(item.blob_ids),
|
||||
)
|
||||
for item in preview.items
|
||||
],
|
||||
)
|
||||
except FileStorageError as exc:
|
||||
raise _http_error(exc) from exc
|
||||
|
||||
|
||||
@router.post("/purge/execute", response_model=FilePurgeResponse)
|
||||
def execute_file_purge(
|
||||
payload: FilePurgeExecuteRequest,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(require_scope("files:file:purge")),
|
||||
):
|
||||
try:
|
||||
result = execute_asset_purge(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
file_ids=payload.file_ids,
|
||||
preview_sha256=payload.preview_sha256,
|
||||
idempotency_key=payload.idempotency_key,
|
||||
approval_reference=payload.approval_reference,
|
||||
before_commit=lambda assets: audit_from_principal(
|
||||
session,
|
||||
principal,
|
||||
action="files.files.purged",
|
||||
object_type="file_batch",
|
||||
object_id=payload.preview_sha256,
|
||||
details={
|
||||
"file_ids": [asset.id for asset in assets],
|
||||
"approval_reference": payload.approval_reference,
|
||||
"preview_sha256": payload.preview_sha256,
|
||||
},
|
||||
),
|
||||
)
|
||||
return FilePurgeResponse(
|
||||
recovery_operation_id=result.recovery_operation_id,
|
||||
status=result.status,
|
||||
replayed=result.replayed,
|
||||
purged_files=result.purged_files,
|
||||
released_blobs=result.released_blobs,
|
||||
)
|
||||
except FileStorageError as exc:
|
||||
session.rollback()
|
||||
raise _http_error(exc) from exc
|
||||
|
||||
|
||||
@router.post("/purge/blobs", response_model=FileBlobGcResponse)
|
||||
def garbage_collect_file_blobs(
|
||||
payload: FileBlobGcRequest,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(require_scope("files:file:purge")),
|
||||
):
|
||||
try:
|
||||
result = garbage_collect_unreferenced_blobs(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
limit=payload.limit,
|
||||
approval_reference=payload.approval_reference,
|
||||
)
|
||||
audit_from_principal(
|
||||
session,
|
||||
principal,
|
||||
action="files.blobs.garbage_collected",
|
||||
object_type="file_blob_batch",
|
||||
object_id=principal.tenant_id,
|
||||
details={
|
||||
"inspected_blobs": result.inspected_blobs,
|
||||
"deleted_blobs": result.deleted_blobs,
|
||||
"unresolved_operation_ids": list(result.unresolved_operation_ids),
|
||||
"approval_reference": payload.approval_reference,
|
||||
},
|
||||
commit=True,
|
||||
)
|
||||
return FileBlobGcResponse(
|
||||
inspected_blobs=result.inspected_blobs,
|
||||
deleted_blobs=result.deleted_blobs,
|
||||
unresolved_operation_ids=list(result.unresolved_operation_ids),
|
||||
)
|
||||
except FileStorageError as exc:
|
||||
session.rollback()
|
||||
raise _http_error(exc) from exc
|
||||
@@ -0,0 +1,194 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Literal
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, status
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from govoplan_core.auth import ApiPrincipal, require_scope
|
||||
from govoplan_files.backend.schemas import (
|
||||
FileDeltaResponse,
|
||||
FileListResponse,
|
||||
)
|
||||
from govoplan_core.db.session import get_session
|
||||
from govoplan_files.backend.storage.files import (
|
||||
count_assets_for_user,
|
||||
list_assets_for_user,
|
||||
list_assets_for_user_window,
|
||||
list_recent_assets_for_user,
|
||||
)
|
||||
|
||||
|
||||
from govoplan_files.backend.route_support import (
|
||||
_asset_list_response,
|
||||
_ensure_campaign_file_access,
|
||||
_ensure_list_owner_access,
|
||||
_is_admin,
|
||||
)
|
||||
from govoplan_files.backend.services.list_queries import (
|
||||
FILES_LIST_CURSOR_SCOPE,
|
||||
_cursor_page_size,
|
||||
_file_cursor_values,
|
||||
_files_delta_response,
|
||||
_files_delta_watermark,
|
||||
_files_list_fingerprint,
|
||||
_full_file_delta_response,
|
||||
_next_file_list_cursor,
|
||||
)
|
||||
|
||||
router = APIRouter(prefix="/files", tags=["files"])
|
||||
|
||||
|
||||
@router.get("/delta", response_model=FileDeltaResponse)
|
||||
def files_delta(
|
||||
owner_type: Literal["user", "group"] | None = None,
|
||||
owner_id: str | None = None,
|
||||
campaign_id: str | None = None,
|
||||
path_prefix: str | None = None,
|
||||
since: str | None = None,
|
||||
limit: int = Query(default=500, ge=1, le=1000),
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(require_scope("files:file:read")),
|
||||
):
|
||||
_ensure_list_owner_access(session, principal, owner_type, owner_id)
|
||||
_ensure_campaign_file_access(session, principal, campaign_id)
|
||||
if since is None:
|
||||
return _full_file_delta_response(
|
||||
session,
|
||||
principal=principal,
|
||||
owner_type=owner_type,
|
||||
owner_id=owner_id,
|
||||
campaign_id=campaign_id,
|
||||
path_prefix=path_prefix,
|
||||
)
|
||||
return _files_delta_response(
|
||||
session,
|
||||
principal=principal,
|
||||
owner_type=owner_type,
|
||||
owner_id=owner_id,
|
||||
campaign_id=campaign_id,
|
||||
path_prefix=path_prefix,
|
||||
since=since,
|
||||
limit=limit,
|
||||
)
|
||||
|
||||
|
||||
@router.get("", response_model=FileListResponse)
|
||||
def list_files(
|
||||
owner_type: Literal["user", "group"] | None = None,
|
||||
owner_id: str | None = None,
|
||||
campaign_id: str | None = None,
|
||||
path_prefix: str | None = None,
|
||||
campaign_usage: Literal["linked", "unlinked"] | None = None,
|
||||
audit_relevant: bool | None = None,
|
||||
sort: Literal["path", "recent"] = "path",
|
||||
page_size: int | None = Query(default=None, ge=1, le=1000),
|
||||
cursor: str | None = None,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(require_scope("files:file:read")),
|
||||
):
|
||||
_ensure_list_owner_access(session, principal, owner_type, owner_id)
|
||||
_ensure_campaign_file_access(session, principal, campaign_id)
|
||||
watermark = _files_delta_watermark(session, principal.tenant_id)
|
||||
total = count_assets_for_user(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
user_id=principal.user.id,
|
||||
owner_type=owner_type,
|
||||
owner_id=owner_id,
|
||||
campaign_id=campaign_id,
|
||||
path_prefix=path_prefix,
|
||||
campaign_usage=campaign_usage,
|
||||
audit_relevant=audit_relevant,
|
||||
is_admin=_is_admin(principal),
|
||||
)
|
||||
if sort == "recent":
|
||||
if cursor:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
|
||||
detail="Recent file projections do not accept a path-order cursor.",
|
||||
)
|
||||
recent_limit = page_size or 25
|
||||
assets = list_recent_assets_for_user(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
user_id=principal.user.id,
|
||||
limit=recent_limit,
|
||||
owner_type=owner_type,
|
||||
owner_id=owner_id,
|
||||
campaign_id=campaign_id,
|
||||
path_prefix=path_prefix,
|
||||
campaign_usage=campaign_usage,
|
||||
audit_relevant=audit_relevant,
|
||||
is_admin=_is_admin(principal),
|
||||
)
|
||||
return FileListResponse(
|
||||
files=_asset_list_response(session, assets, include_shares=True),
|
||||
total=total,
|
||||
watermark=watermark,
|
||||
)
|
||||
effective_page_size = _cursor_page_size(FILES_LIST_CURSOR_SCOPE, cursor, page_size)
|
||||
if effective_page_size is not None:
|
||||
fingerprint = _files_list_fingerprint(
|
||||
principal,
|
||||
owner_type=owner_type,
|
||||
owner_id=owner_id,
|
||||
campaign_id=campaign_id,
|
||||
path_prefix=path_prefix,
|
||||
campaign_usage=campaign_usage,
|
||||
audit_relevant=audit_relevant,
|
||||
page_size=effective_page_size,
|
||||
)
|
||||
after_display_path, after_updated_at, after_id = _file_cursor_values(
|
||||
cursor, fingerprint=fingerprint
|
||||
)
|
||||
assets, has_more = list_assets_for_user_window(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
user_id=principal.user.id,
|
||||
owner_type=owner_type,
|
||||
owner_id=owner_id,
|
||||
campaign_id=campaign_id,
|
||||
path_prefix=path_prefix,
|
||||
campaign_usage=campaign_usage,
|
||||
audit_relevant=audit_relevant,
|
||||
is_admin=_is_admin(principal),
|
||||
page_size=effective_page_size,
|
||||
after_display_path=after_display_path,
|
||||
after_updated_at=after_updated_at,
|
||||
after_id=after_id,
|
||||
)
|
||||
return FileListResponse(
|
||||
files=_asset_list_response(session, assets, include_shares=True),
|
||||
total=total,
|
||||
cursor=cursor,
|
||||
next_cursor=_next_file_list_cursor(
|
||||
principal,
|
||||
assets,
|
||||
owner_type=owner_type,
|
||||
owner_id=owner_id,
|
||||
campaign_id=campaign_id,
|
||||
path_prefix=path_prefix,
|
||||
campaign_usage=campaign_usage,
|
||||
audit_relevant=audit_relevant,
|
||||
page_size=effective_page_size,
|
||||
has_more=has_more,
|
||||
),
|
||||
watermark=watermark,
|
||||
)
|
||||
assets = list_assets_for_user(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
user_id=principal.user.id,
|
||||
owner_type=owner_type,
|
||||
owner_id=owner_id,
|
||||
campaign_id=campaign_id,
|
||||
path_prefix=path_prefix,
|
||||
campaign_usage=campaign_usage,
|
||||
audit_relevant=audit_relevant,
|
||||
is_admin=_is_admin(principal),
|
||||
)
|
||||
return FileListResponse(
|
||||
files=_asset_list_response(session, assets, include_shares=True),
|
||||
total=total,
|
||||
watermark=watermark,
|
||||
)
|
||||
@@ -0,0 +1,330 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, status
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from govoplan_core.api.v1.schemas import (
|
||||
ReferenceOptionListResponse,
|
||||
ReferenceOptionResponse,
|
||||
)
|
||||
from govoplan_core.auth import ApiPrincipal, require_scope
|
||||
from govoplan_core.audit.logging import audit_from_principal
|
||||
from govoplan_core.core.references import (
|
||||
access_scope_reference_page,
|
||||
access_scope_reference_provider_available,
|
||||
)
|
||||
from govoplan_core.db.session import get_session
|
||||
from govoplan_files.backend.runtime import get_registry
|
||||
from govoplan_files.backend.schemas import (
|
||||
BulkFileShareRequest,
|
||||
BulkFileShareResponse,
|
||||
FileShareRequest,
|
||||
FileShareResponse,
|
||||
FileSharesResponse,
|
||||
)
|
||||
from govoplan_files.backend.storage.common import FileStorageError
|
||||
from govoplan_files.backend.storage.files import (
|
||||
current_file_share_for_target,
|
||||
get_asset_for_share_management,
|
||||
list_file_shares,
|
||||
revoke_file_share,
|
||||
share_file,
|
||||
share_files,
|
||||
)
|
||||
from govoplan_files.backend.route_support import (
|
||||
_file_share_response,
|
||||
_http_error,
|
||||
_is_admin,
|
||||
)
|
||||
|
||||
router = APIRouter(prefix="/files", tags=["files"])
|
||||
|
||||
|
||||
@router.get(
|
||||
"/{file_id}/share-target-options",
|
||||
response_model=ReferenceOptionListResponse,
|
||||
)
|
||||
def search_share_targets(
|
||||
file_id: str,
|
||||
target_type: str,
|
||||
q: str = "",
|
||||
selected: list[str] = Query(default=[]),
|
||||
limit: int = Query(default=50, ge=1, le=200),
|
||||
cursor: str | None = None,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(require_scope("files:file:share")),
|
||||
) -> ReferenceOptionListResponse:
|
||||
if target_type not in {"user", "group"}:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
|
||||
detail="Share target type must be user or group",
|
||||
)
|
||||
try:
|
||||
get_asset_for_share_management(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
user_id=principal.user.id,
|
||||
asset_id=file_id,
|
||||
is_admin=_is_admin(principal),
|
||||
)
|
||||
registry = get_registry()
|
||||
page = access_scope_reference_page(
|
||||
registry,
|
||||
principal,
|
||||
scope_type=target_type,
|
||||
reference_kind="membership" if target_type == "user" else "group",
|
||||
query=q,
|
||||
selected_values=selected,
|
||||
limit=limit,
|
||||
cursor=cursor,
|
||||
administrative=True,
|
||||
session=session,
|
||||
)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
|
||||
detail=str(exc),
|
||||
) from exc
|
||||
except FileStorageError as exc:
|
||||
raise _http_error(exc) from exc
|
||||
return ReferenceOptionListResponse(
|
||||
options=[
|
||||
ReferenceOptionResponse(**option.to_dict()) for option in page.options
|
||||
],
|
||||
provider_available=access_scope_reference_provider_available(registry),
|
||||
next_cursor=page.next_cursor,
|
||||
has_more=page.has_more,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{file_id}/shares", response_model=FileSharesResponse)
|
||||
def list_shares(
|
||||
file_id: str,
|
||||
include_inactive: bool = False,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(require_scope("files:file:share")),
|
||||
):
|
||||
try:
|
||||
asset = get_asset_for_share_management(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
user_id=principal.user.id,
|
||||
asset_id=file_id,
|
||||
is_admin=_is_admin(principal),
|
||||
)
|
||||
return FileSharesResponse(
|
||||
shares=[
|
||||
_file_share_response(share)
|
||||
for share in list_file_shares(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
asset_id=asset.id,
|
||||
include_inactive=include_inactive,
|
||||
)
|
||||
]
|
||||
)
|
||||
except FileStorageError as exc:
|
||||
raise _http_error(exc) from exc
|
||||
|
||||
|
||||
@router.post("/{file_id}/shares", response_model=FileShareResponse)
|
||||
def create_share(
|
||||
file_id: str,
|
||||
payload: FileShareRequest,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(require_scope("files:file:share")),
|
||||
):
|
||||
try:
|
||||
asset = get_asset_for_share_management(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
user_id=principal.user.id,
|
||||
asset_id=file_id,
|
||||
is_admin=_is_admin(principal),
|
||||
)
|
||||
previous = current_file_share_for_target(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
asset_id=asset.id,
|
||||
target_type=payload.target_type,
|
||||
target_id=payload.target_id,
|
||||
)
|
||||
previous_permission = previous.permission if previous else None
|
||||
previous_expiry = previous.expires_at if previous else None
|
||||
share = share_file(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
asset=asset,
|
||||
target_type=payload.target_type,
|
||||
target_id=payload.target_id,
|
||||
permission=payload.permission,
|
||||
user_id=principal.user.id,
|
||||
expires_at=payload.expires_at,
|
||||
)
|
||||
session.flush()
|
||||
action = _share_audit_action(
|
||||
existed=previous is not None,
|
||||
previous_permission=previous_permission,
|
||||
permission=share.permission,
|
||||
previous_expiry=previous_expiry,
|
||||
expiry=share.expires_at,
|
||||
)
|
||||
if action:
|
||||
_audit_share_change(session, principal, share, action=action)
|
||||
session.commit()
|
||||
return _file_share_response(share)
|
||||
except FileStorageError as exc:
|
||||
session.rollback()
|
||||
raise _http_error(exc) from exc
|
||||
|
||||
|
||||
@router.post("/bulk-shares", response_model=BulkFileShareResponse)
|
||||
def create_bulk_shares(
|
||||
payload: BulkFileShareRequest,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(require_scope("files:file:share")),
|
||||
):
|
||||
try:
|
||||
file_ids = list(dict.fromkeys(payload.file_ids))
|
||||
assets = [
|
||||
get_asset_for_share_management(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
user_id=principal.user.id,
|
||||
asset_id=file_id,
|
||||
is_admin=_is_admin(principal),
|
||||
)
|
||||
for file_id in file_ids
|
||||
]
|
||||
previous_by_asset = {
|
||||
asset.id: current_file_share_for_target(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
asset_id=asset.id,
|
||||
target_type=payload.target_type,
|
||||
target_id=payload.target_id,
|
||||
)
|
||||
for asset in assets
|
||||
}
|
||||
previous_values = {
|
||||
asset_id: (
|
||||
share.permission if share else None,
|
||||
share.expires_at if share else None,
|
||||
)
|
||||
for asset_id, share in previous_by_asset.items()
|
||||
}
|
||||
shares = share_files(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
assets=assets,
|
||||
target_type=payload.target_type,
|
||||
target_id=payload.target_id,
|
||||
permission=payload.permission,
|
||||
user_id=principal.user.id,
|
||||
expires_at=payload.expires_at,
|
||||
)
|
||||
session.flush()
|
||||
for share in shares:
|
||||
previous_permission, previous_expiry = previous_values[share.file_asset_id]
|
||||
action = _share_audit_action(
|
||||
existed=previous_by_asset[share.file_asset_id] is not None,
|
||||
previous_permission=previous_permission,
|
||||
permission=share.permission,
|
||||
previous_expiry=previous_expiry,
|
||||
expiry=share.expires_at,
|
||||
)
|
||||
if action:
|
||||
_audit_share_change(session, principal, share, action=action)
|
||||
session.commit()
|
||||
return BulkFileShareResponse(
|
||||
shared_count=len(shares),
|
||||
shares=[_file_share_response(share) for share in shares],
|
||||
)
|
||||
except FileStorageError as exc:
|
||||
session.rollback()
|
||||
raise _http_error(exc) from exc
|
||||
|
||||
|
||||
@router.delete("/{file_id}/shares/{share_id}", response_model=FileShareResponse)
|
||||
def revoke_share(
|
||||
file_id: str,
|
||||
share_id: str,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(require_scope("files:file:share")),
|
||||
):
|
||||
try:
|
||||
asset = get_asset_for_share_management(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
user_id=principal.user.id,
|
||||
asset_id=file_id,
|
||||
is_admin=_is_admin(principal),
|
||||
)
|
||||
share, changed = revoke_file_share(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
asset_id=asset.id,
|
||||
share_id=share_id,
|
||||
user_id=principal.user.id,
|
||||
)
|
||||
if changed:
|
||||
_audit_share_change(
|
||||
session, principal, share, action="files.share.revoked"
|
||||
)
|
||||
session.commit()
|
||||
return _file_share_response(share)
|
||||
except FileStorageError as exc:
|
||||
session.rollback()
|
||||
raise _http_error(exc) from exc
|
||||
|
||||
|
||||
def _share_audit_action(
|
||||
*,
|
||||
existed: bool,
|
||||
previous_permission: str | None,
|
||||
permission: str,
|
||||
previous_expiry: datetime | None,
|
||||
expiry: datetime | None,
|
||||
) -> str | None:
|
||||
if not existed:
|
||||
return "files.share.granted"
|
||||
permission_changed = previous_permission != permission
|
||||
expiry_changed = _normalized_expiry(previous_expiry) != _normalized_expiry(expiry)
|
||||
if permission_changed:
|
||||
return "files.share.changed"
|
||||
if expiry_changed:
|
||||
return "files.share.expiry_changed"
|
||||
return None
|
||||
|
||||
|
||||
def _audit_share_change(
|
||||
session: Session,
|
||||
principal: ApiPrincipal,
|
||||
share,
|
||||
*,
|
||||
action: str,
|
||||
) -> None:
|
||||
audit_from_principal(
|
||||
session,
|
||||
principal,
|
||||
action=action,
|
||||
object_type="file_share",
|
||||
object_id=share.id,
|
||||
details={
|
||||
"file_asset_id": share.file_asset_id,
|
||||
"target_type": share.target_type,
|
||||
"target_id": share.target_id,
|
||||
"permission": share.permission,
|
||||
"expires_at": _normalized_expiry(share.expires_at),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _normalized_expiry(value: datetime | None) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
if value.tzinfo is None:
|
||||
value = value.replace(tzinfo=timezone.utc)
|
||||
return value.astimezone(timezone.utc).isoformat()
|
||||
@@ -0,0 +1,308 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Literal
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from govoplan_core.auth import ApiPrincipal, require_scope
|
||||
from govoplan_files.backend.change_tracking import (
|
||||
FILES_CONNECTOR_SPACES_COLLECTION,
|
||||
)
|
||||
from govoplan_files.backend.schemas import (
|
||||
FileConnectorSpaceCreateRequest,
|
||||
FileConnectorSpaceResponse,
|
||||
FileConnectorSpacesResponse,
|
||||
FileConnectorSpaceUpdateRequest,
|
||||
FileSpaceResponse,
|
||||
FileSpacesResponse,
|
||||
)
|
||||
from govoplan_core.db.session import get_session
|
||||
from govoplan_files.backend.storage.access import group_refs_for_ids, user_group_ids
|
||||
from govoplan_files.backend.storage.common import FileStorageError
|
||||
from govoplan_files.backend.storage.connector_spaces import (
|
||||
connector_space_owner_id,
|
||||
create_connector_space,
|
||||
get_connector_space_for_user,
|
||||
list_connector_spaces_for_user,
|
||||
soft_delete_connector_space,
|
||||
update_connector_space,
|
||||
validate_connector_space_write_mode,
|
||||
)
|
||||
from govoplan_files.backend.storage.connector_policy import (
|
||||
ConnectorPolicyDenied,
|
||||
)
|
||||
|
||||
|
||||
from govoplan_files.backend.route_support import (
|
||||
FILES_CONNECTOR_SPACE_RESOURCE,
|
||||
_connector_policy_error,
|
||||
_connector_space_file_space_response,
|
||||
_connector_space_policy_decision,
|
||||
_connector_space_response,
|
||||
_http_error,
|
||||
_is_admin,
|
||||
_record_connector_settings_change,
|
||||
_visible_connector_profile,
|
||||
)
|
||||
|
||||
router = APIRouter(prefix="/files", tags=["files"])
|
||||
|
||||
|
||||
@router.get("/spaces", response_model=FileSpacesResponse)
|
||||
def list_file_spaces(
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(require_scope("files:file:read")),
|
||||
):
|
||||
spaces = [
|
||||
FileSpaceResponse(
|
||||
id=f"user:{principal.user.id}",
|
||||
label="My files",
|
||||
owner_type="user",
|
||||
owner_id=principal.user.id,
|
||||
description="Files owned by your user account.",
|
||||
)
|
||||
]
|
||||
group_ids = user_group_ids(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
user_id=principal.user.id,
|
||||
include_admin_groups=_is_admin(principal),
|
||||
)
|
||||
if group_ids:
|
||||
groups = group_refs_for_ids(tenant_id=principal.tenant_id, group_ids=group_ids)
|
||||
spaces.extend(
|
||||
FileSpaceResponse(
|
||||
id=f"group:{group.id}",
|
||||
label=f"{group.name} files",
|
||||
owner_type="group",
|
||||
owner_id=group.id,
|
||||
description="Files owned by this group.",
|
||||
)
|
||||
for group in groups
|
||||
)
|
||||
connector_spaces = list_connector_spaces_for_user(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
user_id=principal.user.id,
|
||||
is_admin=_is_admin(principal),
|
||||
)
|
||||
spaces.extend(
|
||||
_connector_space_file_space_response(space) for space in connector_spaces
|
||||
)
|
||||
return FileSpacesResponse(spaces=spaces)
|
||||
|
||||
|
||||
@router.get("/connector-spaces", response_model=FileConnectorSpacesResponse)
|
||||
def list_file_connector_spaces(
|
||||
owner_type: Literal["user", "group"] | None = None,
|
||||
owner_id: str | None = None,
|
||||
include_inactive: bool = False,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(require_scope("files:file:read")),
|
||||
):
|
||||
try:
|
||||
spaces = list_connector_spaces_for_user(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
user_id=principal.user.id,
|
||||
owner_type=owner_type,
|
||||
owner_id=owner_id,
|
||||
include_inactive=include_inactive and _is_admin(principal),
|
||||
is_admin=_is_admin(principal),
|
||||
)
|
||||
return FileConnectorSpacesResponse(
|
||||
spaces=[_connector_space_response(space) for space in spaces]
|
||||
)
|
||||
except FileStorageError as exc:
|
||||
raise _http_error(exc) from exc
|
||||
|
||||
|
||||
@router.post(
|
||||
"/connector-spaces",
|
||||
response_model=FileConnectorSpaceResponse,
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
)
|
||||
def create_file_connector_space(
|
||||
payload: FileConnectorSpaceCreateRequest,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(require_scope("files:file:organize")),
|
||||
):
|
||||
if payload.owner_type == "group" and not payload.owner_id:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="owner_id is required for group connector spaces",
|
||||
)
|
||||
target_owner = payload.owner_id or principal.user.id
|
||||
try:
|
||||
profile = _visible_connector_profile(
|
||||
session, principal, payload.connector_profile_id
|
||||
)
|
||||
decision = _connector_space_policy_decision(
|
||||
profile,
|
||||
library_id=payload.library_id,
|
||||
remote_path=payload.remote_path,
|
||||
operation="link",
|
||||
)
|
||||
if not decision.allowed:
|
||||
raise ConnectorPolicyDenied(decision)
|
||||
validate_connector_space_write_mode(profile, read_only=payload.read_only)
|
||||
space = create_connector_space(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
owner_type=payload.owner_type,
|
||||
owner_id=target_owner,
|
||||
user_id=principal.user.id,
|
||||
label=payload.label,
|
||||
profile=profile,
|
||||
library_id=payload.library_id,
|
||||
remote_path=payload.remote_path,
|
||||
sync_mode=payload.sync_mode,
|
||||
read_only=payload.read_only,
|
||||
metadata=payload.metadata,
|
||||
is_admin=_is_admin(principal),
|
||||
)
|
||||
_record_connector_settings_change(
|
||||
session,
|
||||
collection=FILES_CONNECTOR_SPACES_COLLECTION,
|
||||
resource_type=FILES_CONNECTOR_SPACE_RESOURCE,
|
||||
resource_id=space.id,
|
||||
operation="created",
|
||||
principal=principal,
|
||||
tenant_id=space.tenant_id,
|
||||
payload={
|
||||
"owner_type": space.owner_type,
|
||||
"owner_id": connector_space_owner_id(space),
|
||||
},
|
||||
)
|
||||
session.commit()
|
||||
return _connector_space_response(space)
|
||||
except ConnectorPolicyDenied as exc:
|
||||
session.rollback()
|
||||
raise _connector_policy_error(exc) from exc
|
||||
except (FileStorageError, ValueError, json.JSONDecodeError) as exc:
|
||||
session.rollback()
|
||||
raise _http_error(exc) from exc
|
||||
|
||||
|
||||
@router.patch("/connector-spaces/{space_id}", response_model=FileConnectorSpaceResponse)
|
||||
def update_file_connector_space(
|
||||
space_id: str,
|
||||
payload: FileConnectorSpaceUpdateRequest,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(require_scope("files:file:organize")),
|
||||
):
|
||||
try:
|
||||
space = get_connector_space_for_user(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
user_id=principal.user.id,
|
||||
space_id=space_id,
|
||||
include_inactive=_is_admin(principal),
|
||||
is_admin=_is_admin(principal),
|
||||
)
|
||||
except FileStorageError as exc:
|
||||
raise _http_error(exc, not_found=True) from exc
|
||||
try:
|
||||
if (
|
||||
payload.library_id is not None
|
||||
or payload.remote_path is not None
|
||||
or payload.read_only is not None
|
||||
):
|
||||
profile = _visible_connector_profile(
|
||||
session, principal, space.connector_profile_id
|
||||
)
|
||||
decision = _connector_space_policy_decision(
|
||||
profile,
|
||||
library_id=payload.library_id
|
||||
if payload.library_id is not None
|
||||
else space.library_id,
|
||||
remote_path=payload.remote_path
|
||||
if payload.remote_path is not None
|
||||
else space.remote_path,
|
||||
operation="link",
|
||||
)
|
||||
if not decision.allowed:
|
||||
raise ConnectorPolicyDenied(decision)
|
||||
validate_connector_space_write_mode(
|
||||
profile,
|
||||
read_only=(
|
||||
payload.read_only
|
||||
if payload.read_only is not None
|
||||
else space.read_only
|
||||
),
|
||||
)
|
||||
update_connector_space(
|
||||
session,
|
||||
space,
|
||||
user_id=principal.user.id,
|
||||
label=payload.label,
|
||||
library_id=payload.library_id,
|
||||
remote_path=payload.remote_path,
|
||||
sync_mode=payload.sync_mode,
|
||||
read_only=payload.read_only,
|
||||
is_active=payload.is_active,
|
||||
metadata=payload.metadata,
|
||||
is_admin=_is_admin(principal),
|
||||
)
|
||||
_record_connector_settings_change(
|
||||
session,
|
||||
collection=FILES_CONNECTOR_SPACES_COLLECTION,
|
||||
resource_type=FILES_CONNECTOR_SPACE_RESOURCE,
|
||||
resource_id=space.id,
|
||||
operation="updated",
|
||||
principal=principal,
|
||||
tenant_id=space.tenant_id,
|
||||
payload={
|
||||
"owner_type": space.owner_type,
|
||||
"owner_id": connector_space_owner_id(space),
|
||||
},
|
||||
)
|
||||
session.commit()
|
||||
return _connector_space_response(space)
|
||||
except ConnectorPolicyDenied as exc:
|
||||
session.rollback()
|
||||
raise _connector_policy_error(exc) from exc
|
||||
except (FileStorageError, ValueError, json.JSONDecodeError) as exc:
|
||||
session.rollback()
|
||||
raise _http_error(exc) from exc
|
||||
|
||||
|
||||
@router.delete(
|
||||
"/connector-spaces/{space_id}", response_model=FileConnectorSpaceResponse
|
||||
)
|
||||
def delete_file_connector_space(
|
||||
space_id: str,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(require_scope("files:file:organize")),
|
||||
):
|
||||
try:
|
||||
space = get_connector_space_for_user(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
user_id=principal.user.id,
|
||||
space_id=space_id,
|
||||
include_inactive=True,
|
||||
is_admin=_is_admin(principal),
|
||||
)
|
||||
soft_delete_connector_space(
|
||||
session, space, user_id=principal.user.id, is_admin=_is_admin(principal)
|
||||
)
|
||||
_record_connector_settings_change(
|
||||
session,
|
||||
collection=FILES_CONNECTOR_SPACES_COLLECTION,
|
||||
resource_type=FILES_CONNECTOR_SPACE_RESOURCE,
|
||||
resource_id=space.id,
|
||||
operation="deleted",
|
||||
principal=principal,
|
||||
tenant_id=space.tenant_id,
|
||||
payload={
|
||||
"owner_type": space.owner_type,
|
||||
"owner_id": connector_space_owner_id(space),
|
||||
},
|
||||
)
|
||||
session.commit()
|
||||
return _connector_space_response(space)
|
||||
except FileStorageError as exc:
|
||||
session.rollback()
|
||||
raise _http_error(exc, not_found=True) from exc
|
||||
@@ -0,0 +1,213 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
from fastapi import APIRouter, Depends
|
||||
from fastapi.responses import FileResponse
|
||||
from starlette.background import BackgroundTask
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from govoplan_core.auth import ApiPrincipal, require_scope
|
||||
from govoplan_files.backend.schemas import (
|
||||
ArchiveRequest,
|
||||
PatternMatchResponse,
|
||||
PatternResolveRequest,
|
||||
PatternResolveResponse,
|
||||
RenamePreviewItem,
|
||||
RenameRequest,
|
||||
RenameResponse,
|
||||
TransferRequest,
|
||||
TransferResponse,
|
||||
_conflict_resolutions,
|
||||
)
|
||||
from govoplan_core.db.session import get_session
|
||||
from govoplan_files.backend.storage.paths import (
|
||||
UnsafeFilePathError,
|
||||
filename_from_path,
|
||||
normalize_logical_path,
|
||||
)
|
||||
from govoplan_files.backend.storage.archives import create_zip_file
|
||||
from govoplan_files.backend.storage.common import FileStorageError
|
||||
from govoplan_files.backend.storage.files import (
|
||||
get_asset_for_user,
|
||||
list_assets_for_user,
|
||||
)
|
||||
from govoplan_files.backend.storage.search import resolve_patterns
|
||||
from govoplan_files.backend.storage.transfers import (
|
||||
rename_selection,
|
||||
transfer_selection,
|
||||
)
|
||||
|
||||
|
||||
from govoplan_files.backend.route_support import (
|
||||
_asset_response,
|
||||
_attachment_disposition,
|
||||
_audit_connector_access,
|
||||
_cleanup_temp_file,
|
||||
_ensure_campaign_file_access,
|
||||
_ensure_list_owner_access,
|
||||
_http_error,
|
||||
_is_admin,
|
||||
)
|
||||
|
||||
router = APIRouter(prefix="/files", tags=["files"])
|
||||
|
||||
|
||||
@router.post("/bulk-rename", response_model=RenameResponse)
|
||||
def bulk_rename(
|
||||
payload: RenameRequest,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(require_scope("files:file:organize")),
|
||||
):
|
||||
try:
|
||||
plan = rename_selection(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
user_id=principal.user.id,
|
||||
file_ids=payload.file_ids,
|
||||
folder_paths=payload.folder_paths,
|
||||
owner_type=payload.owner_type,
|
||||
owner_id=payload.owner_id,
|
||||
mode=payload.mode,
|
||||
new_name=payload.new_name,
|
||||
find=payload.find,
|
||||
replacement=payload.replacement,
|
||||
prefix=payload.prefix,
|
||||
suffix=payload.suffix,
|
||||
recursive=payload.recursive,
|
||||
dry_run=payload.dry_run,
|
||||
is_admin=_is_admin(principal),
|
||||
)
|
||||
if not payload.dry_run:
|
||||
session.commit()
|
||||
return RenameResponse(
|
||||
dry_run=payload.dry_run,
|
||||
items=[
|
||||
RenamePreviewItem(
|
||||
kind=item.kind,
|
||||
id=item.id,
|
||||
file_id=item.id if item.kind == "file" else None,
|
||||
folder_path=item.old_path if item.kind == "folder" else None,
|
||||
old_path=item.old_path,
|
||||
new_path=item.new_path,
|
||||
)
|
||||
for item in plan
|
||||
],
|
||||
)
|
||||
except (FileStorageError, UnsafeFilePathError, ValueError) as exc:
|
||||
session.rollback()
|
||||
raise _http_error(exc) from exc
|
||||
|
||||
|
||||
@router.post("/transfer", response_model=TransferResponse)
|
||||
def transfer_files(
|
||||
payload: TransferRequest,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(require_scope("files:file:organize")),
|
||||
):
|
||||
try:
|
||||
files, folders = transfer_selection(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
user_id=principal.user.id,
|
||||
operation=payload.operation,
|
||||
file_ids=payload.file_ids,
|
||||
folder_paths=payload.folder_paths,
|
||||
source_owner_type=payload.source_owner_type,
|
||||
source_owner_id=payload.source_owner_id,
|
||||
target_owner_type=payload.target_owner_type,
|
||||
target_owner_id=payload.target_owner_id,
|
||||
target_folder=payload.target_folder,
|
||||
conflict_strategy=payload.conflict_strategy,
|
||||
conflict_resolutions=_conflict_resolutions(payload.conflict_resolutions),
|
||||
is_admin=_is_admin(principal),
|
||||
)
|
||||
session.commit()
|
||||
return TransferResponse(
|
||||
operation=payload.operation, files=files, folders=folders
|
||||
)
|
||||
except (FileStorageError, UnsafeFilePathError, ValueError) as exc:
|
||||
session.rollback()
|
||||
raise _http_error(exc) from exc
|
||||
|
||||
|
||||
@router.post("/archive.zip")
|
||||
def download_archive(
|
||||
payload: ArchiveRequest,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(require_scope("files:file:download")),
|
||||
):
|
||||
try:
|
||||
assets = [
|
||||
get_asset_for_user(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
user_id=principal.user.id,
|
||||
asset_id=file_id,
|
||||
is_admin=_is_admin(principal),
|
||||
)
|
||||
for file_id in payload.file_ids
|
||||
]
|
||||
tmp = tempfile.NamedTemporaryFile(
|
||||
prefix="govoplan-files-", suffix=".zip", delete=False
|
||||
)
|
||||
tmp_path = tmp.name
|
||||
tmp.close()
|
||||
try:
|
||||
create_zip_file(session, assets, tmp_path)
|
||||
except Exception:
|
||||
_cleanup_temp_file(tmp_path)
|
||||
raise
|
||||
_audit_connector_access(session, principal, assets, operation="archive")
|
||||
except FileStorageError as exc:
|
||||
raise _http_error(exc) from exc
|
||||
filename = filename_from_path(
|
||||
normalize_logical_path(payload.filename, fallback_filename="files.zip")
|
||||
)
|
||||
headers = {"Content-Disposition": _attachment_disposition(filename)}
|
||||
return FileResponse(
|
||||
tmp_path,
|
||||
media_type="application/zip",
|
||||
headers=headers,
|
||||
background=BackgroundTask(_cleanup_temp_file, tmp_path),
|
||||
)
|
||||
|
||||
|
||||
@router.post("/resolve-patterns", response_model=PatternResolveResponse)
|
||||
def resolve_file_patterns(
|
||||
payload: PatternResolveRequest,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(require_scope("files:file:read")),
|
||||
):
|
||||
_ensure_list_owner_access(session, principal, payload.owner_type, payload.owner_id)
|
||||
_ensure_campaign_file_access(session, principal, payload.campaign_id)
|
||||
try:
|
||||
assets = list_assets_for_user(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
user_id=principal.user.id,
|
||||
owner_type=payload.owner_type,
|
||||
owner_id=payload.owner_id,
|
||||
campaign_id=payload.campaign_id,
|
||||
path_prefix=payload.path_prefix,
|
||||
is_admin=_is_admin(principal),
|
||||
)
|
||||
resolved, unmatched = resolve_patterns(
|
||||
assets,
|
||||
payload.patterns,
|
||||
base_path=payload.path_prefix,
|
||||
case_sensitive=payload.case_sensitive,
|
||||
)
|
||||
return PatternResolveResponse(
|
||||
patterns=[
|
||||
PatternMatchResponse(
|
||||
pattern=item.pattern,
|
||||
matches=[_asset_response(session, asset) for asset in item.matches],
|
||||
)
|
||||
for item in resolved
|
||||
],
|
||||
unmatched=[_asset_response(session, asset) for asset in unmatched]
|
||||
if payload.include_unmatched
|
||||
else [],
|
||||
)
|
||||
except (FileStorageError, UnsafeFilePathError, ValueError) as exc:
|
||||
raise _http_error(exc) from exc
|
||||
@@ -0,0 +1,487 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Literal
|
||||
from fastapi import APIRouter, Depends, File as FastAPIFile, Form, UploadFile
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from govoplan_core.auth import ApiPrincipal, require_scope
|
||||
from govoplan_core.security.secrets import (
|
||||
TransientPayloadError,
|
||||
open_transient_payload,
|
||||
seal_transient_payload,
|
||||
)
|
||||
from govoplan_files.backend.schemas import (
|
||||
ArchiveEntryResponse,
|
||||
ArchivePreviewResponse,
|
||||
ConflictResolutionRequest,
|
||||
FileUploadResponse,
|
||||
_conflict_resolutions,
|
||||
)
|
||||
from govoplan_files.backend.db.models import FileAsset
|
||||
from govoplan_core.db.session import get_session
|
||||
from govoplan_files.backend.runtime import settings
|
||||
from govoplan_files.backend.storage.paths import (
|
||||
UnsafeFilePathError,
|
||||
normalize_folder,
|
||||
)
|
||||
from govoplan_files.backend.storage.archives import (
|
||||
archive_format_for_filename,
|
||||
extract_archive_upload,
|
||||
extract_zip_upload,
|
||||
inspect_archive,
|
||||
)
|
||||
from govoplan_files.backend.storage.common import FileStorageError
|
||||
from govoplan_files.backend.storage.connector_policy import (
|
||||
ConnectorPolicyDenied,
|
||||
)
|
||||
from govoplan_files.backend.storage.files import (
|
||||
create_file_asset,
|
||||
)
|
||||
|
||||
|
||||
from govoplan_files.backend.route_support import (
|
||||
_asset_response,
|
||||
_audit_connector_imports,
|
||||
_cleanup_temp_file,
|
||||
_connector_policy_error,
|
||||
_enforce_connector_policy,
|
||||
_http_error,
|
||||
_is_admin,
|
||||
_read_limited_upload,
|
||||
_source_metadata_from_form,
|
||||
_spool_limited_upload_to_temp,
|
||||
)
|
||||
|
||||
router = APIRouter(prefix="/files", tags=["files"])
|
||||
_ARCHIVE_PREVIEW_PURPOSE = "files.archive-preview.v1"
|
||||
|
||||
|
||||
def _archive_suffix(filename: str) -> str:
|
||||
lowered = filename.casefold()
|
||||
for suffix in (".tar.bz2", ".tar.gz", ".tar.xz", ".tbz2", ".tgz", ".txz", ".tar", ".zip"):
|
||||
if lowered.endswith(suffix):
|
||||
return suffix
|
||||
return ".archive"
|
||||
|
||||
|
||||
def _archive_sha256(path: str) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with open(path, "rb") as source:
|
||||
while chunk := source.read(1024 * 1024):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _archive_selected_paths(value: str) -> list[str]:
|
||||
parsed = json.loads(value)
|
||||
if not isinstance(parsed, list) or not parsed:
|
||||
raise FileStorageError("Select at least one archive file or folder")
|
||||
if len(parsed) > settings.file_archive_max_entries:
|
||||
raise FileStorageError(
|
||||
"Archive selection exceeds the configured entry limit"
|
||||
)
|
||||
selected: list[str] = []
|
||||
for item in parsed:
|
||||
if not isinstance(item, str) or not item.strip():
|
||||
raise FileStorageError("Archive selection contains an invalid path")
|
||||
if len(item) > 4096:
|
||||
raise FileStorageError("Archive selection path is too long")
|
||||
selected.append(item)
|
||||
return selected
|
||||
|
||||
|
||||
def _validate_archive_preview_token(
|
||||
token: str,
|
||||
*,
|
||||
tenant_id: str,
|
||||
user_id: str,
|
||||
owner_type: str,
|
||||
owner_id: str,
|
||||
path: str,
|
||||
campaign_id: str | None,
|
||||
archive_format: str,
|
||||
archive_sha256: str,
|
||||
) -> None:
|
||||
payload = open_transient_payload(
|
||||
token,
|
||||
ttl_seconds=settings.file_archive_preview_ttl_seconds,
|
||||
)
|
||||
expected = {
|
||||
"purpose": _ARCHIVE_PREVIEW_PURPOSE,
|
||||
"tenant_id": tenant_id,
|
||||
"user_id": user_id,
|
||||
"owner_type": owner_type,
|
||||
"owner_id": owner_id,
|
||||
"path": path,
|
||||
"campaign_id": campaign_id or "",
|
||||
"archive_format": archive_format,
|
||||
}
|
||||
if any(payload.get(key) != value for key, value in expected.items()):
|
||||
raise FileStorageError(
|
||||
"Archive preview does not match this upload destination"
|
||||
)
|
||||
token_digest = payload.get("archive_sha256")
|
||||
if not isinstance(token_digest, str) or not hmac.compare_digest(
|
||||
token_digest, archive_sha256
|
||||
):
|
||||
raise FileStorageError(
|
||||
"Archive contents changed after preview; preview it again"
|
||||
)
|
||||
|
||||
|
||||
@router.post("/archive-preview", response_model=ArchivePreviewResponse)
|
||||
def preview_archive_upload(
|
||||
file: UploadFile = FastAPIFile(...),
|
||||
owner_type: Literal["user", "group"] = Form(default="user"),
|
||||
owner_id: str | None = Form(default=None),
|
||||
path: str = Form(default=""),
|
||||
campaign_id: str | None = Form(default=None),
|
||||
password: str | None = Form(default=None),
|
||||
principal: ApiPrincipal = Depends(require_scope("files:file:upload")),
|
||||
):
|
||||
target_owner = owner_id or principal.user.id
|
||||
archive_path: str | None = None
|
||||
filename = file.filename or "archive"
|
||||
try:
|
||||
archive_format = archive_format_for_filename(filename)
|
||||
archive_path = _spool_limited_upload_to_temp(
|
||||
file,
|
||||
max_bytes=settings.file_upload_zip_max_bytes,
|
||||
suffix=_archive_suffix(filename),
|
||||
)
|
||||
inspection = inspect_archive(
|
||||
archive_path,
|
||||
filename=filename,
|
||||
password=password,
|
||||
max_entries=settings.file_archive_max_entries,
|
||||
max_expanded_bytes=settings.file_archive_max_expanded_bytes,
|
||||
max_expansion_ratio=settings.file_archive_max_expansion_ratio,
|
||||
)
|
||||
digest = _archive_sha256(archive_path)
|
||||
normalized_path = normalize_folder(path)
|
||||
preview_token = seal_transient_payload(
|
||||
{
|
||||
"purpose": _ARCHIVE_PREVIEW_PURPOSE,
|
||||
"tenant_id": principal.tenant_id,
|
||||
"user_id": principal.user.id,
|
||||
"owner_type": owner_type,
|
||||
"owner_id": target_owner,
|
||||
"path": normalized_path,
|
||||
"campaign_id": campaign_id or "",
|
||||
"archive_format": archive_format,
|
||||
"archive_sha256": digest,
|
||||
}
|
||||
)
|
||||
expires_at = datetime.now(UTC) + timedelta(
|
||||
seconds=settings.file_archive_preview_ttl_seconds
|
||||
)
|
||||
return ArchivePreviewResponse(
|
||||
preview_token=preview_token,
|
||||
archive_format=inspection.archive_format,
|
||||
entries=[
|
||||
ArchiveEntryResponse(
|
||||
path=entry.path,
|
||||
kind=entry.kind,
|
||||
size_bytes=entry.size_bytes,
|
||||
compressed_size_bytes=entry.compressed_size_bytes,
|
||||
encrypted=entry.encrypted,
|
||||
)
|
||||
for entry in inspection.entries
|
||||
],
|
||||
file_count=inspection.file_count,
|
||||
directory_count=inspection.directory_count,
|
||||
expanded_size_bytes=inspection.expanded_size_bytes,
|
||||
compressed_size_bytes=inspection.compressed_size_bytes,
|
||||
requires_password=inspection.requires_password,
|
||||
password_verified=inspection.password_verified,
|
||||
expires_at=expires_at.isoformat(),
|
||||
)
|
||||
except (FileStorageError, UnsafeFilePathError, ValueError) as exc:
|
||||
raise _http_error(exc) from exc
|
||||
finally:
|
||||
if archive_path:
|
||||
_cleanup_temp_file(archive_path)
|
||||
|
||||
|
||||
@router.post("/archive-confirm", response_model=FileUploadResponse)
|
||||
def confirm_archive_upload(
|
||||
file: UploadFile = FastAPIFile(...),
|
||||
preview_token: str = Form(...),
|
||||
selected_paths_json: str = Form(...),
|
||||
owner_type: Literal["user", "group"] = Form(default="user"),
|
||||
owner_id: str | None = Form(default=None),
|
||||
path: str = Form(default=""),
|
||||
campaign_id: str | None = Form(default=None),
|
||||
password: str | None = Form(default=None),
|
||||
conflict_strategy: Literal["reject", "overwrite", "rename"] = Form(
|
||||
default="reject"
|
||||
),
|
||||
conflict_resolutions_json: str | None = Form(default=None),
|
||||
source_provenance_json: str | None = Form(default=None),
|
||||
source_revision: str | None = Form(default=None),
|
||||
connector_policy_json: str | None = Form(default=None),
|
||||
encryption_vault_id: str | None = Form(default=None),
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(require_scope("files:file:upload")),
|
||||
):
|
||||
target_owner = owner_id or principal.user.id
|
||||
archive_path: str | None = None
|
||||
filename = file.filename or "archive"
|
||||
try:
|
||||
raw_resolutions = (
|
||||
json.loads(conflict_resolutions_json)
|
||||
if conflict_resolutions_json
|
||||
else []
|
||||
)
|
||||
upload_resolutions = _conflict_resolutions(
|
||||
[ConflictResolutionRequest(**item) for item in raw_resolutions]
|
||||
)
|
||||
selected_paths = _archive_selected_paths(selected_paths_json)
|
||||
_enforce_connector_policy(
|
||||
source_provenance_json,
|
||||
connector_policy_json,
|
||||
operation="import",
|
||||
)
|
||||
metadata = _source_metadata_from_form(
|
||||
source_provenance_json, source_revision
|
||||
)
|
||||
archive_format = archive_format_for_filename(filename)
|
||||
normalized_path = normalize_folder(path)
|
||||
archive_path = _spool_limited_upload_to_temp(
|
||||
file,
|
||||
max_bytes=settings.file_upload_zip_max_bytes,
|
||||
suffix=_archive_suffix(filename),
|
||||
)
|
||||
digest = _archive_sha256(archive_path)
|
||||
_validate_archive_preview_token(
|
||||
preview_token,
|
||||
tenant_id=principal.tenant_id,
|
||||
user_id=principal.user.id,
|
||||
owner_type=owner_type,
|
||||
owner_id=target_owner,
|
||||
path=normalized_path,
|
||||
campaign_id=campaign_id,
|
||||
archive_format=archive_format,
|
||||
archive_sha256=digest,
|
||||
)
|
||||
extracted = extract_archive_upload(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
owner_type=owner_type,
|
||||
owner_id=target_owner,
|
||||
user_id=principal.user.id,
|
||||
archive_data=archive_path,
|
||||
filename=filename,
|
||||
folder=normalized_path,
|
||||
campaign_id=campaign_id,
|
||||
selected_paths=selected_paths,
|
||||
password=password,
|
||||
conflict_strategy=conflict_strategy,
|
||||
conflict_resolutions=upload_resolutions,
|
||||
metadata=metadata,
|
||||
is_admin=_is_admin(principal),
|
||||
encryption_vault_id=encryption_vault_id,
|
||||
max_entries=settings.file_archive_max_entries,
|
||||
max_file_bytes=settings.file_upload_max_bytes,
|
||||
max_expanded_bytes=settings.file_archive_max_expanded_bytes,
|
||||
max_expansion_ratio=settings.file_archive_max_expansion_ratio,
|
||||
)
|
||||
uploaded_assets = [item.asset for item in extracted]
|
||||
_audit_connector_imports(session, principal, uploaded_assets)
|
||||
session.commit()
|
||||
return FileUploadResponse(
|
||||
files=[
|
||||
_asset_response(session, asset, include_shares=True)
|
||||
for asset in uploaded_assets
|
||||
]
|
||||
)
|
||||
except ConnectorPolicyDenied as exc:
|
||||
session.rollback()
|
||||
raise _connector_policy_error(exc) from exc
|
||||
except (
|
||||
FileStorageError,
|
||||
TransientPayloadError,
|
||||
UnsafeFilePathError,
|
||||
ValueError,
|
||||
json.JSONDecodeError,
|
||||
) as exc:
|
||||
session.rollback()
|
||||
raise _http_error(exc) from exc
|
||||
finally:
|
||||
if archive_path:
|
||||
_cleanup_temp_file(archive_path)
|
||||
|
||||
|
||||
@router.post("/upload", response_model=FileUploadResponse)
|
||||
def upload_files(
|
||||
files: list[UploadFile] = FastAPIFile(...),
|
||||
owner_type: Literal["user", "group"] = Form(default="user"),
|
||||
owner_id: str | None = Form(default=None),
|
||||
path: str = Form(default=""),
|
||||
campaign_id: str | None = Form(default=None),
|
||||
unpack_zip: bool = Form(default=False),
|
||||
conflict_strategy: Literal["reject", "overwrite", "rename"] = Form(
|
||||
default="reject"
|
||||
),
|
||||
conflict_resolutions_json: str | None = Form(default=None),
|
||||
source_provenance_json: str | None = Form(default=None),
|
||||
source_revision: str | None = Form(default=None),
|
||||
connector_policy_json: str | None = Form(default=None),
|
||||
encryption_vault_id: str | None = Form(default=None),
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(require_scope("files:file:upload")),
|
||||
):
|
||||
target_owner = owner_id or principal.user.id
|
||||
uploaded_assets: list[FileAsset] = []
|
||||
try:
|
||||
raw_resolutions = (
|
||||
json.loads(conflict_resolutions_json) if conflict_resolutions_json else []
|
||||
)
|
||||
upload_resolutions = _conflict_resolutions(
|
||||
[ConflictResolutionRequest(**item) for item in raw_resolutions]
|
||||
)
|
||||
_enforce_connector_policy(
|
||||
source_provenance_json, connector_policy_json, operation="import"
|
||||
)
|
||||
metadata = _source_metadata_from_form(source_provenance_json, source_revision)
|
||||
for upload in files:
|
||||
filename = upload.filename or "file"
|
||||
content_type = upload.content_type or None
|
||||
upload_limit = (
|
||||
settings.file_upload_zip_max_bytes
|
||||
if unpack_zip and filename.lower().endswith(".zip")
|
||||
else settings.file_upload_max_bytes
|
||||
)
|
||||
if unpack_zip and filename.lower().endswith(".zip"):
|
||||
zip_path = _spool_limited_upload_to_temp(
|
||||
upload, max_bytes=upload_limit, suffix=".zip"
|
||||
)
|
||||
try:
|
||||
extracted = extract_zip_upload(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
owner_type=owner_type,
|
||||
owner_id=target_owner,
|
||||
user_id=principal.user.id,
|
||||
zip_data=zip_path,
|
||||
folder=path,
|
||||
campaign_id=campaign_id,
|
||||
conflict_strategy=conflict_strategy,
|
||||
conflict_resolutions=upload_resolutions,
|
||||
metadata=metadata,
|
||||
is_admin=_is_admin(principal),
|
||||
encryption_vault_id=encryption_vault_id,
|
||||
max_file_bytes=settings.file_upload_max_bytes,
|
||||
max_total_bytes=settings.file_upload_zip_max_bytes,
|
||||
)
|
||||
finally:
|
||||
_cleanup_temp_file(zip_path)
|
||||
uploaded_assets.extend(item.asset for item in extracted)
|
||||
continue
|
||||
data = _read_limited_upload(upload, max_bytes=upload_limit)
|
||||
stored = create_file_asset(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
owner_type=owner_type,
|
||||
owner_id=target_owner,
|
||||
user_id=principal.user.id,
|
||||
filename=filename,
|
||||
data=data,
|
||||
folder=path,
|
||||
content_type=content_type,
|
||||
campaign_id=campaign_id,
|
||||
conflict_strategy=conflict_strategy,
|
||||
conflict_resolutions=upload_resolutions,
|
||||
metadata=metadata,
|
||||
is_admin=_is_admin(principal),
|
||||
encryption_vault_id=encryption_vault_id,
|
||||
)
|
||||
uploaded_assets.append(stored.asset)
|
||||
_audit_connector_imports(session, principal, uploaded_assets)
|
||||
session.commit()
|
||||
except ConnectorPolicyDenied as exc:
|
||||
session.rollback()
|
||||
raise _connector_policy_error(exc) from exc
|
||||
except (FileStorageError, UnsafeFilePathError, ValueError) as exc:
|
||||
session.rollback()
|
||||
raise _http_error(exc) from exc
|
||||
return FileUploadResponse(
|
||||
files=[
|
||||
_asset_response(session, asset, include_shares=True)
|
||||
for asset in uploaded_assets
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@router.post("/upload-zip", response_model=FileUploadResponse)
|
||||
def upload_zip(
|
||||
file: UploadFile = FastAPIFile(...),
|
||||
owner_type: Literal["user", "group"] = Form(default="user"),
|
||||
owner_id: str | None = Form(default=None),
|
||||
path: str = Form(default=""),
|
||||
campaign_id: str | None = Form(default=None),
|
||||
conflict_strategy: Literal["reject", "overwrite", "rename"] = Form(
|
||||
default="reject"
|
||||
),
|
||||
conflict_resolutions_json: str | None = Form(default=None),
|
||||
source_provenance_json: str | None = Form(default=None),
|
||||
source_revision: str | None = Form(default=None),
|
||||
connector_policy_json: str | None = Form(default=None),
|
||||
encryption_vault_id: str | None = Form(default=None),
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(require_scope("files:file:upload")),
|
||||
):
|
||||
target_owner = owner_id or principal.user.id
|
||||
zip_path: str | None = None
|
||||
try:
|
||||
raw_resolutions = (
|
||||
json.loads(conflict_resolutions_json) if conflict_resolutions_json else []
|
||||
)
|
||||
upload_resolutions = _conflict_resolutions(
|
||||
[ConflictResolutionRequest(**item) for item in raw_resolutions]
|
||||
)
|
||||
_enforce_connector_policy(
|
||||
source_provenance_json, connector_policy_json, operation="import"
|
||||
)
|
||||
metadata = _source_metadata_from_form(source_provenance_json, source_revision)
|
||||
zip_path = _spool_limited_upload_to_temp(
|
||||
file, max_bytes=settings.file_upload_zip_max_bytes, suffix=".zip"
|
||||
)
|
||||
extracted = extract_zip_upload(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
owner_type=owner_type,
|
||||
owner_id=target_owner,
|
||||
user_id=principal.user.id,
|
||||
zip_data=zip_path,
|
||||
folder=path,
|
||||
campaign_id=campaign_id,
|
||||
conflict_strategy=conflict_strategy,
|
||||
conflict_resolutions=upload_resolutions,
|
||||
metadata=metadata,
|
||||
is_admin=_is_admin(principal),
|
||||
encryption_vault_id=encryption_vault_id,
|
||||
max_file_bytes=settings.file_upload_max_bytes,
|
||||
max_total_bytes=settings.file_upload_zip_max_bytes,
|
||||
)
|
||||
_audit_connector_imports(session, principal, [item.asset for item in extracted])
|
||||
session.commit()
|
||||
except ConnectorPolicyDenied as exc:
|
||||
session.rollback()
|
||||
raise _connector_policy_error(exc) from exc
|
||||
except (FileStorageError, UnsafeFilePathError, ValueError) as exc:
|
||||
session.rollback()
|
||||
raise _http_error(exc) from exc
|
||||
finally:
|
||||
if zip_path:
|
||||
_cleanup_temp_file(zip_path)
|
||||
return FileUploadResponse(
|
||||
files=[
|
||||
_asset_response(session, item.asset, include_shares=True)
|
||||
for item in extracted
|
||||
]
|
||||
)
|
||||
@@ -1,5 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
@@ -36,6 +37,7 @@ class FileConnectorSpaceCreateRequest(BaseModel):
|
||||
library_id: str | None = None
|
||||
remote_path: str = ""
|
||||
sync_mode: Literal["manual"] = "manual"
|
||||
read_only: bool = True
|
||||
metadata: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
@@ -44,6 +46,7 @@ class FileConnectorSpaceUpdateRequest(BaseModel):
|
||||
library_id: str | None = None
|
||||
remote_path: str | None = None
|
||||
sync_mode: Literal["manual"] | None = None
|
||||
read_only: bool | None = None
|
||||
is_active: bool | None = None
|
||||
metadata: dict[str, Any] | None = None
|
||||
|
||||
@@ -73,11 +76,93 @@ class FileConnectorSpacesResponse(BaseModel):
|
||||
|
||||
class FileShareResponse(BaseModel):
|
||||
id: str
|
||||
file_asset_id: str
|
||||
target_type: str
|
||||
target_id: str
|
||||
permission: str
|
||||
created_by_user_id: str | None = None
|
||||
created_at: str
|
||||
expires_at: str | None = None
|
||||
revoked_at: str | None = None
|
||||
revoked_by_user_id: str | None = None
|
||||
active: bool
|
||||
|
||||
|
||||
class FileSharesResponse(BaseModel):
|
||||
shares: list[FileShareResponse] = Field(default_factory=list)
|
||||
|
||||
|
||||
class FileIntegrityScanCreateRequest(BaseModel):
|
||||
verify_checksums: bool = True
|
||||
batch_size: int = Field(default=100, ge=1, le=1000)
|
||||
|
||||
|
||||
class FileIntegrityScanResponse(BaseModel):
|
||||
id: str
|
||||
tenant_id: str
|
||||
storage_backend: str
|
||||
storage_prefix: str
|
||||
status: str
|
||||
revision: int
|
||||
phase: str
|
||||
verify_checksums: bool
|
||||
batch_size: int
|
||||
scanned_blob_count: int
|
||||
verified_blob_count: int
|
||||
quarantined_blob_count: int
|
||||
scanned_object_count: int
|
||||
orphan_object_count: int
|
||||
created_by_user_id: str | None = None
|
||||
started_at: str | None = None
|
||||
completed_at: str | None = None
|
||||
last_error: str | None = None
|
||||
created_at: str
|
||||
updated_at: str
|
||||
|
||||
|
||||
class FileIntegrityScansResponse(BaseModel):
|
||||
scans: list[FileIntegrityScanResponse] = Field(default_factory=list)
|
||||
|
||||
|
||||
class FileIntegrityFindingResponse(BaseModel):
|
||||
id: str
|
||||
scan_id: str
|
||||
tenant_id: str
|
||||
kind: str
|
||||
state: str
|
||||
revision: int
|
||||
blob_id: str | None = None
|
||||
storage_key: str
|
||||
expected_size_bytes: int | None = None
|
||||
observed_size_bytes: int | None = None
|
||||
expected_checksum_sha256: str | None = None
|
||||
observed_checksum_sha256: str | None = None
|
||||
resolved_at: str | None = None
|
||||
resolved_by_user_id: str | None = None
|
||||
created_at: str
|
||||
updated_at: str
|
||||
|
||||
|
||||
class FileIntegrityFindingsResponse(BaseModel):
|
||||
findings: list[FileIntegrityFindingResponse] = Field(default_factory=list)
|
||||
|
||||
|
||||
class FileIntegrityActionRequest(BaseModel):
|
||||
dry_run: bool = True
|
||||
expected_revision: int = Field(ge=1)
|
||||
|
||||
|
||||
class FileIntegrityScanRunRequest(BaseModel):
|
||||
expected_revision: int = Field(ge=1)
|
||||
|
||||
|
||||
class FileIntegrityActionResponse(BaseModel):
|
||||
action: str
|
||||
changed: bool
|
||||
dry_run: bool
|
||||
finding: FileIntegrityFindingResponse
|
||||
inspection_kind: str | None = None
|
||||
inspection_valid: bool | None = None
|
||||
|
||||
|
||||
class FileSourceProvenance(BaseModel):
|
||||
@@ -109,6 +194,10 @@ class FileAssetResponse(BaseModel):
|
||||
created_at: str
|
||||
updated_at: str
|
||||
deleted_at: str | None = None
|
||||
retained_until: str | None = None
|
||||
legal_hold: bool = False
|
||||
lifecycle_revision: int = 1
|
||||
lifecycle_reason: str | None = None
|
||||
audit_relevant: bool = False
|
||||
metadata: dict[str, Any] | None = None
|
||||
source_provenance: FileSourceProvenance | None = None
|
||||
@@ -154,6 +243,7 @@ class FileFolderDeleteResponse(BaseModel):
|
||||
|
||||
class FileListResponse(BaseModel):
|
||||
files: list[FileAssetResponse]
|
||||
total: int
|
||||
cursor: str | None = None
|
||||
next_cursor: str | None = None
|
||||
watermark: str | None = None
|
||||
@@ -172,6 +262,27 @@ class FileUploadResponse(BaseModel):
|
||||
files: list[FileAssetResponse]
|
||||
|
||||
|
||||
class ArchiveEntryResponse(BaseModel):
|
||||
path: str
|
||||
kind: Literal["file", "directory"]
|
||||
size_bytes: int
|
||||
compressed_size_bytes: int | None = None
|
||||
encrypted: bool = False
|
||||
|
||||
|
||||
class ArchivePreviewResponse(BaseModel):
|
||||
preview_token: str
|
||||
archive_format: str
|
||||
entries: list[ArchiveEntryResponse] = Field(default_factory=list)
|
||||
file_count: int
|
||||
directory_count: int
|
||||
expanded_size_bytes: int
|
||||
compressed_size_bytes: int
|
||||
requires_password: bool
|
||||
password_verified: bool
|
||||
expires_at: str
|
||||
|
||||
|
||||
class FileConnectorPolicySource(BaseModel):
|
||||
scope_type: Literal["system", "tenant", "user", "group", "campaign"] = "system"
|
||||
scope_id: str | None = None
|
||||
@@ -372,6 +483,7 @@ class FileConnectorProviderResponse(BaseModel):
|
||||
installed: bool
|
||||
browse_supported: bool
|
||||
import_supported: bool
|
||||
write_supported: bool = False
|
||||
optional_dependency: str | None = None
|
||||
permission_model: str
|
||||
sync_strategy: str
|
||||
@@ -430,6 +542,24 @@ class FileConnectorSyncResponse(BaseModel):
|
||||
current_version_id: str
|
||||
|
||||
|
||||
class FileConnectorWriteRequest(BaseModel):
|
||||
file_id: str
|
||||
remote_path: str
|
||||
idempotency_key: str = Field(min_length=8, max_length=160)
|
||||
expected_revision: str | None = Field(default=None, max_length=500)
|
||||
|
||||
|
||||
class FileConnectorWriteResponse(BaseModel):
|
||||
recovery_operation_id: str
|
||||
status: str
|
||||
replayed: bool = False
|
||||
provider: str
|
||||
remote_path: str
|
||||
revision: str | None = None
|
||||
checksum_sha256: str
|
||||
size_bytes: int
|
||||
|
||||
|
||||
class BulkDeleteRequest(BaseModel):
|
||||
file_ids: list[str]
|
||||
|
||||
@@ -438,6 +568,69 @@ class BulkDeleteResponse(BaseModel):
|
||||
deleted_count: int
|
||||
|
||||
|
||||
class FileLifecycleUpdateRequest(BaseModel):
|
||||
retained_until: datetime | None = None
|
||||
legal_hold: bool = False
|
||||
reason: str = Field(min_length=1, max_length=500)
|
||||
expected_revision: int = Field(ge=1)
|
||||
|
||||
|
||||
class FileRestoreResponse(BaseModel):
|
||||
restored_count: int
|
||||
|
||||
|
||||
class FileFolderRestoreResponse(BaseModel):
|
||||
restored_folders: int
|
||||
restored_files: int
|
||||
|
||||
|
||||
class FilePurgePreviewRequest(BaseModel):
|
||||
file_ids: list[str] = Field(min_length=1, max_length=100)
|
||||
|
||||
|
||||
class FilePurgePreviewItem(BaseModel):
|
||||
file_id: str
|
||||
filename: str
|
||||
lifecycle_revision: int
|
||||
deleted_at: str | None = None
|
||||
retained_until: str | None = None
|
||||
legal_hold: bool = False
|
||||
blockers: list[str] = Field(default_factory=list)
|
||||
blob_ids: list[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
class FilePurgePreviewResponse(BaseModel):
|
||||
preview_sha256: str
|
||||
eligible: bool
|
||||
items: list[FilePurgePreviewItem]
|
||||
|
||||
|
||||
class FilePurgeExecuteRequest(FilePurgePreviewRequest):
|
||||
preview_sha256: str = Field(min_length=64, max_length=64)
|
||||
idempotency_key: str = Field(min_length=8, max_length=160)
|
||||
approval_reference: str = Field(min_length=3, max_length=1000)
|
||||
confirmation: Literal["PURGE"]
|
||||
|
||||
|
||||
class FilePurgeResponse(BaseModel):
|
||||
recovery_operation_id: str
|
||||
status: str
|
||||
replayed: bool = False
|
||||
purged_files: int = 0
|
||||
released_blobs: int = 0
|
||||
|
||||
|
||||
class FileBlobGcRequest(BaseModel):
|
||||
limit: int = Field(default=25, ge=1, le=100)
|
||||
approval_reference: str = Field(min_length=3, max_length=1000)
|
||||
|
||||
|
||||
class FileBlobGcResponse(BaseModel):
|
||||
inspected_blobs: int
|
||||
deleted_blobs: int
|
||||
unresolved_operation_ids: list[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
class ConflictResolutionRequest(BaseModel):
|
||||
target_path: str
|
||||
action: Literal["overwrite", "rename", "skip"]
|
||||
@@ -452,6 +645,7 @@ class FileShareRequest(BaseModel):
|
||||
target_type: Literal["user", "group", "campaign", "tenant"]
|
||||
target_id: str
|
||||
permission: Literal["read", "write", "manage"] = "read"
|
||||
expires_at: datetime | None = None
|
||||
|
||||
|
||||
class BulkFileShareRequest(BaseModel):
|
||||
@@ -459,6 +653,7 @@ class BulkFileShareRequest(BaseModel):
|
||||
target_type: Literal["user", "group", "campaign", "tenant"]
|
||||
target_id: str
|
||||
permission: Literal["read", "write", "manage"] = "read"
|
||||
expires_at: datetime | None = None
|
||||
|
||||
|
||||
class BulkFileShareResponse(BaseModel):
|
||||
|
||||
@@ -0,0 +1,329 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping, Sequence
|
||||
from urllib.parse import quote
|
||||
|
||||
from sqlalchemy import func, or_, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from govoplan_core.auth import ApiPrincipal
|
||||
from govoplan_core.core.events import PlatformEvent
|
||||
from govoplan_core.core.modules import ModuleContext
|
||||
from govoplan_core.core.search import (
|
||||
SearchAuthorizationRequest,
|
||||
SearchBackfillPage,
|
||||
SearchBackfillRequest,
|
||||
SearchDocument,
|
||||
SearchIndexChange,
|
||||
SearchResourceReference,
|
||||
SearchResourceType,
|
||||
)
|
||||
from govoplan_files.backend.db.models import FileAsset, FileFolder, FileShare
|
||||
from govoplan_files.backend.storage.share_state import effective_file_share_clause
|
||||
|
||||
|
||||
PROVIDER_ID = "files.objects"
|
||||
RESOURCE_MODELS = {
|
||||
"file": FileAsset,
|
||||
"folder": FileFolder,
|
||||
}
|
||||
READ_SCOPE = "files:file:read"
|
||||
ADMIN_SCOPE = "files:file:admin"
|
||||
|
||||
|
||||
class FilesSearchSource:
|
||||
def resource_types(self) -> Sequence[SearchResourceType]:
|
||||
return (
|
||||
SearchResourceType(
|
||||
provider_id=PROVIDER_ID,
|
||||
module_id="files",
|
||||
resource_type="file",
|
||||
label="Files",
|
||||
requires_authorization_recheck=True,
|
||||
),
|
||||
SearchResourceType(
|
||||
provider_id=PROVIDER_ID,
|
||||
module_id="files",
|
||||
resource_type="folder",
|
||||
label="File folders",
|
||||
requires_authorization_recheck=True,
|
||||
),
|
||||
)
|
||||
|
||||
def backfill(
|
||||
self,
|
||||
session: object,
|
||||
*,
|
||||
request: SearchBackfillRequest,
|
||||
) -> SearchBackfillPage:
|
||||
db = _session(session)
|
||||
model = _model(request.provider_id, request.resource_type)
|
||||
statement = select(model).where(
|
||||
model.tenant_id == request.tenant_id,
|
||||
model.deleted_at.is_(None),
|
||||
)
|
||||
if request.cursor:
|
||||
statement = statement.where(model.id > request.cursor)
|
||||
rows = list(
|
||||
db.scalars(
|
||||
statement.order_by(model.id).limit(request.limit + 1)
|
||||
)
|
||||
)
|
||||
has_more = len(rows) > request.limit
|
||||
selected = rows[: request.limit]
|
||||
shares = (
|
||||
_shares_by_asset(db, selected)
|
||||
if request.resource_type == "file"
|
||||
else {}
|
||||
)
|
||||
high_watermark = db.scalar(
|
||||
select(func.max(model.updated_at)).where(
|
||||
model.tenant_id == request.tenant_id,
|
||||
model.deleted_at.is_(None),
|
||||
)
|
||||
)
|
||||
return SearchBackfillPage(
|
||||
documents=tuple(
|
||||
_document(
|
||||
row,
|
||||
resource_type=request.resource_type,
|
||||
shares=shares.get(row.id, ()),
|
||||
)
|
||||
for row in selected
|
||||
),
|
||||
next_cursor=selected[-1].id if has_more and selected else None,
|
||||
complete=not has_more,
|
||||
high_watermark=(
|
||||
high_watermark.isoformat()
|
||||
if high_watermark is not None
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
||||
def authorize(
|
||||
self,
|
||||
session: object,
|
||||
principal: object,
|
||||
*,
|
||||
requests: Sequence[SearchAuthorizationRequest],
|
||||
) -> Mapping[str, bool]:
|
||||
decisions = {item.reference.key: False for item in requests}
|
||||
if not isinstance(principal, ApiPrincipal) or not (
|
||||
principal.has(READ_SCOPE) or principal.has(ADMIN_SCOPE)
|
||||
):
|
||||
return decisions
|
||||
db = _session(session)
|
||||
for request in requests:
|
||||
reference = request.reference
|
||||
if (
|
||||
reference.tenant_id != principal.tenant_id
|
||||
or reference.module_id != "files"
|
||||
or reference.resource_type not in RESOURCE_MODELS
|
||||
):
|
||||
continue
|
||||
decisions[reference.key] = _can_read(
|
||||
db,
|
||||
principal,
|
||||
reference=reference,
|
||||
)
|
||||
return decisions
|
||||
|
||||
def index_changes_for_event(
|
||||
self,
|
||||
session: object,
|
||||
*,
|
||||
event: PlatformEvent,
|
||||
delivery_key: str,
|
||||
) -> Sequence[SearchIndexChange]:
|
||||
if (
|
||||
event.module_id != "files"
|
||||
or event.tenant is None
|
||||
or event.resource is None
|
||||
or event.resource.id is None
|
||||
or event.resource.type not in RESOURCE_MODELS
|
||||
):
|
||||
return ()
|
||||
db = _session(session)
|
||||
reference = SearchResourceReference(
|
||||
tenant_id=event.tenant.id,
|
||||
module_id="files",
|
||||
resource_type=event.resource.type,
|
||||
resource_id=event.resource.id,
|
||||
)
|
||||
model = RESOURCE_MODELS[event.resource.type]
|
||||
row = db.get(model, event.resource.id)
|
||||
deleted = row is None or row.tenant_id != event.tenant.id or row.deleted_at is not None
|
||||
cursor = event.event_id
|
||||
document = None
|
||||
if not deleted:
|
||||
shares = (
|
||||
tuple(_active_shares(db, row.id))
|
||||
if event.resource.type == "file"
|
||||
else ()
|
||||
)
|
||||
document = _document(
|
||||
row,
|
||||
resource_type=event.resource.type,
|
||||
shares=shares,
|
||||
change_cursor=cursor,
|
||||
)
|
||||
return (
|
||||
SearchIndexChange(
|
||||
change_id=f"{delivery_key}:{PROVIDER_ID}:{event.resource.type}",
|
||||
provider_id=PROVIDER_ID,
|
||||
kind="delete" if deleted else "upsert",
|
||||
reference=reference,
|
||||
source_revision=(
|
||||
document.source_revision if document is not None else cursor
|
||||
),
|
||||
cursor=cursor,
|
||||
document=document,
|
||||
occurred_at=event.occurred_at,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def create_files_search_source(_context: ModuleContext) -> FilesSearchSource:
|
||||
return FilesSearchSource()
|
||||
|
||||
|
||||
def _model(provider_id: str, resource_type: str):
|
||||
if provider_id != PROVIDER_ID or resource_type not in RESOURCE_MODELS:
|
||||
raise ValueError("Unsupported Files search source.")
|
||||
return RESOURCE_MODELS[resource_type]
|
||||
|
||||
|
||||
def _document(
|
||||
row: FileAsset | FileFolder,
|
||||
*,
|
||||
resource_type: str,
|
||||
shares: Sequence[FileShare] = (),
|
||||
change_cursor: str | None = None,
|
||||
) -> SearchDocument:
|
||||
is_file = isinstance(row, FileAsset)
|
||||
title = row.filename if is_file else (row.path.rsplit("/", 1)[-1] or row.path)
|
||||
path = row.display_path if is_file else row.path
|
||||
owner_id = row.owner_user_id if row.owner_type == "user" else row.owner_group_id
|
||||
tokens = [f"scope:{READ_SCOPE}", f"scope:{ADMIN_SCOPE}"]
|
||||
if owner_id:
|
||||
tokens.append(
|
||||
f"membership:{owner_id}"
|
||||
if row.owner_type == "user"
|
||||
else f"group:{owner_id}"
|
||||
)
|
||||
for share in shares:
|
||||
prefix = "membership" if share.target_type == "user" else share.target_type
|
||||
if prefix in {"membership", "group", "tenant"}:
|
||||
tokens.append(f"{prefix}:{share.target_id}")
|
||||
updated_at = row.updated_at or row.created_at
|
||||
revision = (
|
||||
f"{row.current_version_id or 'none'}:{updated_at.isoformat()}"
|
||||
if is_file
|
||||
else updated_at.isoformat()
|
||||
)
|
||||
return SearchDocument(
|
||||
tenant_id=row.tenant_id,
|
||||
module_id="files",
|
||||
provider_id=PROVIDER_ID,
|
||||
resource_type=resource_type,
|
||||
resource_id=row.id,
|
||||
title=title,
|
||||
url=f"/files?{resource_type}Id={quote(row.id, safe='')}",
|
||||
summary=((row.description or "") if is_file else path)[:4000] or None,
|
||||
body=" ".join(
|
||||
value for value in (path, row.description if is_file else None) if value
|
||||
)[:200_000],
|
||||
keywords=(path[:200], row.owner_type[:200]),
|
||||
visibility="restricted",
|
||||
acl_tokens=tuple(dict.fromkeys(tokens)),
|
||||
metadata={
|
||||
"path": path,
|
||||
"owner_type": row.owner_type,
|
||||
"current_version_id": row.current_version_id if is_file else None,
|
||||
},
|
||||
source_revision=revision,
|
||||
change_cursor=change_cursor,
|
||||
source_updated_at=updated_at,
|
||||
requires_authorization_recheck=True,
|
||||
)
|
||||
|
||||
|
||||
def _can_read(
|
||||
session: Session,
|
||||
principal: ApiPrincipal,
|
||||
*,
|
||||
reference: SearchResourceReference,
|
||||
) -> bool:
|
||||
model = RESOURCE_MODELS[reference.resource_type]
|
||||
row = session.get(model, reference.resource_id)
|
||||
if row is None or row.tenant_id != principal.tenant_id or row.deleted_at is not None:
|
||||
return False
|
||||
if principal.has(ADMIN_SCOPE):
|
||||
return True
|
||||
user_id = str(getattr(principal.user, "id", "") or principal.membership_id or "")
|
||||
if row.owner_type == "user" and row.owner_user_id == user_id:
|
||||
return True
|
||||
if row.owner_type == "group" and row.owner_group_id in principal.group_ids:
|
||||
return True
|
||||
if reference.resource_type != "file":
|
||||
return False
|
||||
target_clauses = [
|
||||
(FileShare.target_type == "user") & (FileShare.target_id == user_id),
|
||||
(FileShare.target_type == "tenant")
|
||||
& (FileShare.target_id == principal.tenant_id),
|
||||
]
|
||||
if principal.group_ids:
|
||||
target_clauses.append(
|
||||
(FileShare.target_type == "group")
|
||||
& (FileShare.target_id.in_(tuple(principal.group_ids)))
|
||||
)
|
||||
return session.scalar(
|
||||
select(FileShare.id).where(
|
||||
FileShare.tenant_id == principal.tenant_id,
|
||||
FileShare.file_asset_id == row.id,
|
||||
effective_file_share_clause(),
|
||||
or_(*target_clauses),
|
||||
).limit(1)
|
||||
) is not None
|
||||
|
||||
|
||||
def _active_shares(session: Session, asset_id: str) -> Sequence[FileShare]:
|
||||
return tuple(
|
||||
session.scalars(
|
||||
select(FileShare).where(
|
||||
FileShare.file_asset_id == asset_id,
|
||||
effective_file_share_clause(),
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _shares_by_asset(
|
||||
session: Session,
|
||||
rows: Sequence[FileAsset | FileFolder],
|
||||
) -> dict[str, tuple[FileShare, ...]]:
|
||||
asset_ids = [row.id for row in rows if isinstance(row, FileAsset)]
|
||||
grouped: dict[str, list[FileShare]] = {asset_id: [] for asset_id in asset_ids}
|
||||
if not asset_ids:
|
||||
return {}
|
||||
for share in session.scalars(
|
||||
select(FileShare).where(
|
||||
FileShare.file_asset_id.in_(asset_ids),
|
||||
effective_file_share_clause(),
|
||||
)
|
||||
):
|
||||
grouped.setdefault(share.file_asset_id, []).append(share)
|
||||
return {key: tuple(value) for key, value in grouped.items()}
|
||||
|
||||
|
||||
def _session(value: object) -> Session:
|
||||
if not isinstance(value, Session):
|
||||
raise TypeError("Files search requires a SQLAlchemy session.")
|
||||
return value
|
||||
|
||||
|
||||
__all__ = [
|
||||
"FilesSearchSource",
|
||||
"PROVIDER_ID",
|
||||
"create_files_search_source",
|
||||
]
|
||||
@@ -0,0 +1 @@
|
||||
"""Query and response assembly services used by Files routes."""
|
||||
@@ -0,0 +1,292 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Literal
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from govoplan_core.auth import ApiPrincipal
|
||||
from govoplan_core.api.v1.schemas import DeltaDeletedItem
|
||||
from govoplan_core.core.change_sequence import (
|
||||
ChangeSequenceEntry,
|
||||
)
|
||||
from govoplan_files.backend.change_tracking import (
|
||||
FILES_CONNECTOR_CREDENTIALS_COLLECTION,
|
||||
FILES_CONNECTOR_POLICIES_COLLECTION,
|
||||
FILES_CONNECTOR_PROFILES_COLLECTION,
|
||||
FILES_CONNECTOR_SPACES_COLLECTION,
|
||||
)
|
||||
from govoplan_files.backend.schemas import (
|
||||
FileConnectorCredentialResponse,
|
||||
FileConnectorSettingsDeltaResponse,
|
||||
FileConnectorPolicyResponse,
|
||||
FileConnectorProfileResponse,
|
||||
)
|
||||
from govoplan_files.backend.db.models import FileConnectorSpace
|
||||
from govoplan_files.backend.storage.connector_credential_store import (
|
||||
ConnectorCredential,
|
||||
)
|
||||
from govoplan_files.backend.storage.connector_profiles import ConnectorProfile
|
||||
from govoplan_files.backend.storage.connector_policy_store import (
|
||||
connector_policy_response,
|
||||
)
|
||||
|
||||
|
||||
from govoplan_files.backend.route_support import (
|
||||
FILES_CONNECTOR_CREDENTIAL_RESOURCE,
|
||||
FILES_CONNECTOR_PROFILE_RESOURCE,
|
||||
FILES_CONNECTOR_SPACE_RESOURCE,
|
||||
_can_read_disabled_connector_profiles,
|
||||
_connector_deleted_item,
|
||||
_connector_space_response,
|
||||
_ensure_campaign_file_access,
|
||||
_file_connector_settings_response_watermark,
|
||||
_file_connector_settings_watermark,
|
||||
_visible_connector_credentials,
|
||||
_visible_connector_profiles,
|
||||
_visible_connector_spaces,
|
||||
)
|
||||
|
||||
|
||||
def _full_file_connector_settings_delta_response(
|
||||
session: Session,
|
||||
principal: ApiPrincipal,
|
||||
*,
|
||||
scope_type: str,
|
||||
scope_id: str | None,
|
||||
provider: str | None,
|
||||
campaign_id: str | None,
|
||||
include_disabled: bool,
|
||||
include_inactive: bool,
|
||||
owner_type: Literal["user", "group"] | None,
|
||||
owner_id: str | None,
|
||||
) -> FileConnectorSettingsDeltaResponse:
|
||||
if campaign_id:
|
||||
_ensure_campaign_file_access(session, principal, campaign_id)
|
||||
profiles = _visible_connector_profiles(
|
||||
session,
|
||||
principal,
|
||||
provider=provider,
|
||||
campaign_id=campaign_id,
|
||||
include_disabled=include_disabled
|
||||
and _can_read_disabled_connector_profiles(principal),
|
||||
include_admin_scopes=_can_read_disabled_connector_profiles(principal),
|
||||
include_effective_policy=False,
|
||||
)
|
||||
credentials = _visible_connector_credentials(
|
||||
session,
|
||||
principal,
|
||||
provider=provider,
|
||||
include_disabled=include_disabled,
|
||||
)
|
||||
spaces = _visible_connector_spaces(
|
||||
session,
|
||||
principal,
|
||||
owner_type=owner_type,
|
||||
owner_id=owner_id,
|
||||
include_inactive=include_inactive,
|
||||
)
|
||||
return FileConnectorSettingsDeltaResponse(
|
||||
profiles=[
|
||||
FileConnectorProfileResponse(**profile.to_response())
|
||||
for profile in profiles
|
||||
],
|
||||
credentials=[
|
||||
FileConnectorCredentialResponse(**credential.to_response())
|
||||
for credential in credentials
|
||||
],
|
||||
spaces=[_connector_space_response(space) for space in spaces],
|
||||
policy=FileConnectorPolicyResponse(
|
||||
**connector_policy_response(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
scope_type=scope_type,
|
||||
scope_id=scope_id,
|
||||
)
|
||||
),
|
||||
changed_sections=["profiles", "credentials", "spaces", "policy"],
|
||||
deleted=[],
|
||||
watermark=_file_connector_settings_watermark(
|
||||
session, tenant_id=principal.tenant_id
|
||||
),
|
||||
has_more=False,
|
||||
full=True,
|
||||
)
|
||||
|
||||
|
||||
def _changed_file_connector_setting_ids(
|
||||
entries: list[ChangeSequenceEntry],
|
||||
) -> tuple[set[str], set[str], set[str], bool]:
|
||||
changed_profile_ids = {
|
||||
entry.resource_id
|
||||
for entry in entries
|
||||
if entry.collection == FILES_CONNECTOR_PROFILES_COLLECTION
|
||||
and entry.resource_type == FILES_CONNECTOR_PROFILE_RESOURCE
|
||||
}
|
||||
changed_credential_ids = {
|
||||
entry.resource_id
|
||||
for entry in entries
|
||||
if entry.collection == FILES_CONNECTOR_CREDENTIALS_COLLECTION
|
||||
and entry.resource_type == FILES_CONNECTOR_CREDENTIAL_RESOURCE
|
||||
}
|
||||
changed_space_ids = {
|
||||
entry.resource_id
|
||||
for entry in entries
|
||||
if entry.collection == FILES_CONNECTOR_SPACES_COLLECTION
|
||||
and entry.resource_type == FILES_CONNECTOR_SPACE_RESOURCE
|
||||
}
|
||||
policy_changed = any(
|
||||
entry.collection == FILES_CONNECTOR_POLICIES_COLLECTION for entry in entries
|
||||
)
|
||||
return (
|
||||
changed_profile_ids,
|
||||
changed_credential_ids,
|
||||
changed_space_ids,
|
||||
policy_changed,
|
||||
)
|
||||
|
||||
|
||||
def _file_connector_settings_changed_sections(
|
||||
*,
|
||||
changed_profile_ids: set[str],
|
||||
changed_credential_ids: set[str],
|
||||
changed_space_ids: set[str],
|
||||
policy_changed: bool,
|
||||
profiles: list[ConnectorProfile],
|
||||
) -> list[str]:
|
||||
changed_sections = []
|
||||
if changed_profile_ids or any(
|
||||
profile.credential_profile_id
|
||||
and profile.credential_profile_id in changed_credential_ids
|
||||
for profile in profiles
|
||||
):
|
||||
changed_sections.append("profiles")
|
||||
if changed_credential_ids:
|
||||
changed_sections.append("credentials")
|
||||
if changed_space_ids:
|
||||
changed_sections.append("spaces")
|
||||
if policy_changed:
|
||||
changed_sections.append("policy")
|
||||
return changed_sections
|
||||
|
||||
|
||||
def _file_connector_settings_deleted_items(
|
||||
entries: list[ChangeSequenceEntry],
|
||||
*,
|
||||
visible_profiles: dict[str, ConnectorProfile],
|
||||
visible_credentials: dict[str, ConnectorCredential],
|
||||
visible_spaces: dict[str, FileConnectorSpace],
|
||||
) -> list[DeltaDeletedItem]:
|
||||
return [
|
||||
_connector_deleted_item(entry)
|
||||
for entry in entries
|
||||
if (
|
||||
entry.resource_type == FILES_CONNECTOR_PROFILE_RESOURCE
|
||||
and entry.resource_id not in visible_profiles
|
||||
)
|
||||
or (
|
||||
entry.resource_type == FILES_CONNECTOR_CREDENTIAL_RESOURCE
|
||||
and entry.resource_id not in visible_credentials
|
||||
)
|
||||
or (
|
||||
entry.resource_type == FILES_CONNECTOR_SPACE_RESOURCE
|
||||
and entry.resource_id not in visible_spaces
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def _incremental_file_connector_settings_delta_response(
|
||||
session: Session,
|
||||
principal: ApiPrincipal,
|
||||
*,
|
||||
entries: list[ChangeSequenceEntry],
|
||||
has_more: bool,
|
||||
scope_type: str,
|
||||
scope_id: str | None,
|
||||
provider: str | None,
|
||||
campaign_id: str | None,
|
||||
include_disabled: bool,
|
||||
include_inactive: bool,
|
||||
owner_type: Literal["user", "group"] | None,
|
||||
owner_id: str | None,
|
||||
) -> FileConnectorSettingsDeltaResponse:
|
||||
changed_profile_ids, changed_credential_ids, changed_space_ids, policy_changed = (
|
||||
_changed_file_connector_setting_ids(entries)
|
||||
)
|
||||
profiles = _visible_connector_profiles(
|
||||
session,
|
||||
principal,
|
||||
provider=provider,
|
||||
campaign_id=campaign_id,
|
||||
include_disabled=include_disabled
|
||||
and _can_read_disabled_connector_profiles(principal),
|
||||
include_admin_scopes=_can_read_disabled_connector_profiles(principal),
|
||||
include_effective_policy=False,
|
||||
)
|
||||
visible_profiles = {
|
||||
profile.id: profile
|
||||
for profile in profiles
|
||||
if profile.id in changed_profile_ids
|
||||
or (
|
||||
profile.credential_profile_id
|
||||
and profile.credential_profile_id in changed_credential_ids
|
||||
)
|
||||
}
|
||||
credentials = _visible_connector_credentials(
|
||||
session,
|
||||
principal,
|
||||
provider=provider,
|
||||
include_disabled=include_disabled,
|
||||
)
|
||||
visible_credentials = {
|
||||
credential.id: credential
|
||||
for credential in credentials
|
||||
if credential.id in changed_credential_ids
|
||||
}
|
||||
spaces = _visible_connector_spaces(
|
||||
session,
|
||||
principal,
|
||||
owner_type=owner_type,
|
||||
owner_id=owner_id,
|
||||
include_inactive=include_inactive,
|
||||
)
|
||||
visible_spaces = {
|
||||
space.id: space for space in spaces if space.id in changed_space_ids
|
||||
}
|
||||
return FileConnectorSettingsDeltaResponse(
|
||||
profiles=[
|
||||
FileConnectorProfileResponse(**profile.to_response())
|
||||
for profile in visible_profiles.values()
|
||||
],
|
||||
credentials=[
|
||||
FileConnectorCredentialResponse(**credential.to_response())
|
||||
for credential in visible_credentials.values()
|
||||
],
|
||||
spaces=[_connector_space_response(space) for space in visible_spaces.values()],
|
||||
policy=FileConnectorPolicyResponse(
|
||||
**connector_policy_response(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
scope_type=scope_type,
|
||||
scope_id=scope_id,
|
||||
)
|
||||
)
|
||||
if policy_changed
|
||||
else None,
|
||||
changed_sections=_file_connector_settings_changed_sections(
|
||||
changed_profile_ids=changed_profile_ids,
|
||||
changed_credential_ids=changed_credential_ids,
|
||||
changed_space_ids=changed_space_ids,
|
||||
policy_changed=policy_changed,
|
||||
profiles=profiles,
|
||||
),
|
||||
deleted=_file_connector_settings_deleted_items(
|
||||
entries,
|
||||
visible_profiles=visible_profiles,
|
||||
visible_credentials=visible_credentials,
|
||||
visible_spaces=visible_spaces,
|
||||
),
|
||||
watermark=_file_connector_settings_response_watermark(
|
||||
session, tenant_id=principal.tenant_id, entries=entries, has_more=has_more
|
||||
),
|
||||
has_more=has_more,
|
||||
full=False,
|
||||
)
|
||||
@@ -0,0 +1,620 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Literal
|
||||
from fastapi import HTTPException, status
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from govoplan_core.auth import ApiPrincipal
|
||||
from govoplan_core.api.v1.schemas import DeltaDeletedItem
|
||||
from govoplan_core.core.change_sequence import (
|
||||
ChangeSequenceEntry,
|
||||
decode_sequence_watermark,
|
||||
encode_sequence_watermark,
|
||||
max_sequence_id,
|
||||
sequence_entries_since,
|
||||
sequence_watermark_is_expired,
|
||||
)
|
||||
from govoplan_core.core.pagination import (
|
||||
KeysetCursorError,
|
||||
decode_keyset_cursor,
|
||||
encode_keyset_cursor,
|
||||
keyset_query_fingerprint,
|
||||
)
|
||||
from govoplan_files.backend.change_tracking import (
|
||||
FILES_ASSETS_COLLECTION,
|
||||
FILES_FOLDERS_COLLECTION,
|
||||
FILES_MODULE_ID,
|
||||
)
|
||||
from govoplan_files.backend.schemas import (
|
||||
FileDeltaResponse,
|
||||
)
|
||||
from govoplan_files.backend.db.models import FileAsset, FileFolder, FileShare
|
||||
from govoplan_files.backend.storage.access import user_group_ids
|
||||
from govoplan_files.backend.storage.files import (
|
||||
list_assets_for_user,
|
||||
)
|
||||
from govoplan_files.backend.storage.folders import list_folders_for_user
|
||||
from govoplan_files.backend.storage.share_state import effective_file_share_clause
|
||||
|
||||
|
||||
from govoplan_files.backend.route_support import (
|
||||
_asset_list_response,
|
||||
_folder_response,
|
||||
_is_admin,
|
||||
)
|
||||
|
||||
_FILES_DELTA_COLLECTIONS = (FILES_ASSETS_COLLECTION, FILES_FOLDERS_COLLECTION)
|
||||
FILES_LIST_CURSOR_SCOPE = "files.list.v1"
|
||||
FOLDERS_LIST_CURSOR_SCOPE = "files.folders.list.v1"
|
||||
DEFAULT_FILE_LIST_PAGE_SIZE = 500
|
||||
|
||||
|
||||
def _cursor_http_error(exc: Exception) -> HTTPException:
|
||||
return HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc))
|
||||
|
||||
|
||||
def _cursor_page_size(
|
||||
scope: str, cursor: str | None, explicit_page_size: int | None
|
||||
) -> int | None:
|
||||
if explicit_page_size is not None:
|
||||
return explicit_page_size
|
||||
if not cursor:
|
||||
return DEFAULT_FILE_LIST_PAGE_SIZE
|
||||
try:
|
||||
values = decode_keyset_cursor(scope, cursor)
|
||||
except KeysetCursorError as exc:
|
||||
raise _cursor_http_error(exc) from exc
|
||||
raw_page_size = values.get("page_size")
|
||||
if not isinstance(raw_page_size, int) or raw_page_size < 1 or raw_page_size > 1000:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid pagination cursor"
|
||||
)
|
||||
return raw_page_size
|
||||
|
||||
|
||||
def _files_list_fingerprint(
|
||||
principal: ApiPrincipal,
|
||||
*,
|
||||
owner_type: Literal["user", "group"] | None,
|
||||
owner_id: str | None,
|
||||
campaign_id: str | None,
|
||||
path_prefix: str | None,
|
||||
campaign_usage: Literal["linked", "unlinked"] | None,
|
||||
audit_relevant: bool | None,
|
||||
page_size: int,
|
||||
) -> str:
|
||||
return keyset_query_fingerprint(
|
||||
FILES_LIST_CURSOR_SCOPE,
|
||||
{
|
||||
"tenant_id": principal.tenant_id,
|
||||
"actor": "admin" if _is_admin(principal) else principal.user.id,
|
||||
"owner_type": owner_type,
|
||||
"owner_id": owner_id,
|
||||
"campaign_id": campaign_id,
|
||||
"path_prefix": path_prefix or "",
|
||||
"campaign_usage": campaign_usage or "",
|
||||
"audit_relevant": audit_relevant,
|
||||
"sort": "display_path.asc,updated_at.desc,id.asc",
|
||||
"page_size": page_size,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _folders_list_fingerprint(
|
||||
principal: ApiPrincipal,
|
||||
*,
|
||||
owner_type: Literal["user", "group"],
|
||||
owner_id: str,
|
||||
page_size: int,
|
||||
) -> str:
|
||||
return keyset_query_fingerprint(
|
||||
FOLDERS_LIST_CURSOR_SCOPE,
|
||||
{
|
||||
"tenant_id": principal.tenant_id,
|
||||
"actor": "admin" if _is_admin(principal) else principal.user.id,
|
||||
"owner_type": owner_type,
|
||||
"owner_id": owner_id,
|
||||
"sort": "path.asc,id.asc",
|
||||
"page_size": page_size,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _file_cursor_values(
|
||||
cursor: str | None, *, fingerprint: str
|
||||
) -> tuple[str | None, datetime | None, str | None]:
|
||||
try:
|
||||
values = decode_keyset_cursor(
|
||||
FILES_LIST_CURSOR_SCOPE, cursor, fingerprint=fingerprint
|
||||
)
|
||||
except KeysetCursorError as exc:
|
||||
raise _cursor_http_error(exc) from exc
|
||||
if values is None:
|
||||
return None, None, None
|
||||
display_path = values.get("display_path")
|
||||
updated_at = values.get("updated_at")
|
||||
asset_id = values.get("id")
|
||||
if (
|
||||
not isinstance(display_path, str)
|
||||
or not isinstance(updated_at, str)
|
||||
or not isinstance(asset_id, str)
|
||||
):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid pagination cursor"
|
||||
)
|
||||
try:
|
||||
parsed_updated_at = datetime.fromisoformat(updated_at)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid pagination cursor"
|
||||
) from exc
|
||||
return display_path, parsed_updated_at, asset_id
|
||||
|
||||
|
||||
def _folder_cursor_values(
|
||||
cursor: str | None, *, fingerprint: str
|
||||
) -> tuple[str | None, str | None]:
|
||||
try:
|
||||
values = decode_keyset_cursor(
|
||||
FOLDERS_LIST_CURSOR_SCOPE, cursor, fingerprint=fingerprint
|
||||
)
|
||||
except KeysetCursorError as exc:
|
||||
raise _cursor_http_error(exc) from exc
|
||||
if values is None:
|
||||
return None, None
|
||||
folder_path = values.get("path")
|
||||
folder_id = values.get("id")
|
||||
if not isinstance(folder_path, str) or not isinstance(folder_id, str):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid pagination cursor"
|
||||
)
|
||||
return folder_path, folder_id
|
||||
|
||||
|
||||
def _next_file_list_cursor(
|
||||
principal: ApiPrincipal,
|
||||
assets: list[FileAsset],
|
||||
*,
|
||||
owner_type: Literal["user", "group"] | None,
|
||||
owner_id: str | None,
|
||||
campaign_id: str | None,
|
||||
path_prefix: str | None,
|
||||
campaign_usage: Literal["linked", "unlinked"] | None,
|
||||
audit_relevant: bool | None,
|
||||
page_size: int,
|
||||
has_more: bool,
|
||||
) -> str | None:
|
||||
if not has_more or not assets:
|
||||
return None
|
||||
last = assets[-1]
|
||||
return encode_keyset_cursor(
|
||||
FILES_LIST_CURSOR_SCOPE,
|
||||
fingerprint=_files_list_fingerprint(
|
||||
principal,
|
||||
owner_type=owner_type,
|
||||
owner_id=owner_id,
|
||||
campaign_id=campaign_id,
|
||||
path_prefix=path_prefix,
|
||||
campaign_usage=campaign_usage,
|
||||
audit_relevant=audit_relevant,
|
||||
page_size=page_size,
|
||||
),
|
||||
values={
|
||||
"display_path": last.display_path,
|
||||
"updated_at": last.updated_at,
|
||||
"id": last.id,
|
||||
"page_size": page_size,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _next_folder_list_cursor(
|
||||
principal: ApiPrincipal,
|
||||
folders: list[FileFolder],
|
||||
*,
|
||||
owner_type: Literal["user", "group"],
|
||||
owner_id: str,
|
||||
page_size: int,
|
||||
has_more: bool,
|
||||
) -> str | None:
|
||||
if not has_more or not folders:
|
||||
return None
|
||||
last = folders[-1]
|
||||
return encode_keyset_cursor(
|
||||
FOLDERS_LIST_CURSOR_SCOPE,
|
||||
fingerprint=_folders_list_fingerprint(
|
||||
principal, owner_type=owner_type, owner_id=owner_id, page_size=page_size
|
||||
),
|
||||
values={"path": last.path, "id": last.id, "page_size": page_size},
|
||||
)
|
||||
|
||||
|
||||
def _files_delta_watermark(session: Session, tenant_id: str) -> str:
|
||||
return encode_sequence_watermark(
|
||||
max_sequence_id(
|
||||
session,
|
||||
tenant_id=tenant_id,
|
||||
module_id=FILES_MODULE_ID,
|
||||
collections=_FILES_DELTA_COLLECTIONS,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _full_file_delta_response(
|
||||
session: Session,
|
||||
*,
|
||||
principal: ApiPrincipal,
|
||||
owner_type: Literal["user", "group"] | None,
|
||||
owner_id: str | None,
|
||||
campaign_id: str | None,
|
||||
path_prefix: str | None,
|
||||
) -> FileDeltaResponse:
|
||||
assets = list_assets_for_user(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
user_id=principal.user.id,
|
||||
owner_type=owner_type,
|
||||
owner_id=owner_id,
|
||||
campaign_id=campaign_id,
|
||||
path_prefix=path_prefix,
|
||||
is_admin=_is_admin(principal),
|
||||
)
|
||||
folders = _visible_folders_for_delta(
|
||||
session,
|
||||
principal=principal,
|
||||
owner_type=owner_type,
|
||||
owner_id=owner_id,
|
||||
path_prefix=path_prefix,
|
||||
)
|
||||
return FileDeltaResponse(
|
||||
files=_asset_list_response(session, assets, include_shares=True),
|
||||
folders=[_folder_response(folder) for folder in folders],
|
||||
deleted=[],
|
||||
watermark=_files_delta_watermark(session, principal.tenant_id),
|
||||
has_more=False,
|
||||
full=True,
|
||||
)
|
||||
|
||||
|
||||
def _visible_folders_for_delta(
|
||||
session: Session,
|
||||
*,
|
||||
principal: ApiPrincipal,
|
||||
owner_type: Literal["user", "group"] | None,
|
||||
owner_id: str | None,
|
||||
path_prefix: str | None,
|
||||
) -> list[FileFolder]:
|
||||
if not owner_type or not owner_id:
|
||||
return []
|
||||
folders = list_folders_for_user(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
user_id=principal.user.id,
|
||||
owner_type=owner_type,
|
||||
owner_id=owner_id,
|
||||
is_admin=_is_admin(principal),
|
||||
)
|
||||
if not path_prefix:
|
||||
return folders
|
||||
normalized = path_prefix.strip().strip("/")
|
||||
if not normalized:
|
||||
return folders
|
||||
return [
|
||||
folder
|
||||
for folder in folders
|
||||
if folder.path == normalized or folder.path.startswith(f"{normalized}/")
|
||||
]
|
||||
|
||||
|
||||
def _entry_path_matches(
|
||||
entry_payload: dict[str, object], path_prefix: str | None
|
||||
) -> bool:
|
||||
if not path_prefix:
|
||||
return True
|
||||
normalized = path_prefix.strip().strip("/")
|
||||
if not normalized:
|
||||
return True
|
||||
for key in ("path", "previous_path"):
|
||||
value = entry_payload.get(key)
|
||||
if isinstance(value, str) and (
|
||||
value == normalized or value.startswith(f"{normalized}/")
|
||||
):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _entry_owner_matches(
|
||||
entry_payload: dict[str, object],
|
||||
owner_type: Literal["user", "group"] | None,
|
||||
owner_id: str | None,
|
||||
) -> bool:
|
||||
if not owner_type or not owner_id:
|
||||
return True
|
||||
return (
|
||||
entry_payload.get("owner_type") == owner_type
|
||||
and entry_payload.get("owner_id") == owner_id
|
||||
)
|
||||
|
||||
|
||||
def _entry_campaign_matches(session: Session, entry, campaign_id: str | None) -> bool:
|
||||
if not campaign_id:
|
||||
return True
|
||||
payload = entry.payload or {}
|
||||
if (
|
||||
payload.get("share_target_type") == "campaign"
|
||||
and payload.get("share_target_id") == campaign_id
|
||||
):
|
||||
return True
|
||||
if entry.resource_type != "file":
|
||||
return False
|
||||
return (
|
||||
session.query(FileShare)
|
||||
.filter(
|
||||
FileShare.tenant_id == entry.tenant_id,
|
||||
FileShare.file_asset_id == entry.resource_id,
|
||||
FileShare.target_type == "campaign",
|
||||
FileShare.target_id == campaign_id,
|
||||
effective_file_share_clause(),
|
||||
)
|
||||
.first()
|
||||
is not None
|
||||
)
|
||||
|
||||
|
||||
def _principal_group_ids_for_delta(
|
||||
session: Session, principal: ApiPrincipal
|
||||
) -> set[str]:
|
||||
cache = session.info.setdefault("files_delta_group_ids", {})
|
||||
key = (principal.tenant_id, principal.user.id)
|
||||
if key not in cache:
|
||||
cache[key] = set(
|
||||
user_group_ids(
|
||||
session, tenant_id=principal.tenant_id, user_id=principal.user.id
|
||||
)
|
||||
)
|
||||
return cache[key]
|
||||
|
||||
|
||||
def _entry_subject_matches_principal(
|
||||
session: Session, principal: ApiPrincipal, entry_payload: dict[str, object]
|
||||
) -> bool:
|
||||
if _is_admin(principal):
|
||||
return True
|
||||
owner_type = entry_payload.get("owner_type")
|
||||
owner_id = entry_payload.get("owner_id")
|
||||
if owner_type == "user" and owner_id == principal.user.id:
|
||||
return True
|
||||
if owner_type == "group" and isinstance(owner_id, str):
|
||||
if owner_id in _principal_group_ids_for_delta(session, principal):
|
||||
return True
|
||||
share_target_type = entry_payload.get("share_target_type")
|
||||
share_target_id = entry_payload.get("share_target_id")
|
||||
if share_target_type == "user" and share_target_id == principal.user.id:
|
||||
return True
|
||||
if share_target_type == "tenant" and share_target_id == principal.tenant_id:
|
||||
return True
|
||||
if share_target_type == "group" and isinstance(share_target_id, str):
|
||||
if share_target_id in _principal_group_ids_for_delta(session, principal):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _entry_matches_delta_scope(
|
||||
session: Session,
|
||||
principal: ApiPrincipal,
|
||||
entry,
|
||||
*,
|
||||
owner_type: Literal["user", "group"] | None,
|
||||
owner_id: str | None,
|
||||
campaign_id: str | None,
|
||||
path_prefix: str | None,
|
||||
) -> bool:
|
||||
payload = entry.payload or {}
|
||||
if (
|
||||
not owner_type
|
||||
and not campaign_id
|
||||
and not _entry_subject_matches_principal(session, principal, payload)
|
||||
):
|
||||
return False
|
||||
return (
|
||||
_entry_owner_matches(payload, owner_type, owner_id)
|
||||
and _entry_path_matches(payload, path_prefix)
|
||||
and _entry_campaign_matches(session, entry, campaign_id)
|
||||
)
|
||||
|
||||
|
||||
def _decode_files_delta_watermark(since: str) -> int:
|
||||
try:
|
||||
return decode_sequence_watermark(since)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)
|
||||
) from exc
|
||||
|
||||
|
||||
def _changed_delta_resource_ids(
|
||||
entries: list[ChangeSequenceEntry],
|
||||
) -> tuple[list[str], list[str]]:
|
||||
file_ids = list(
|
||||
dict.fromkeys(
|
||||
entry.resource_id for entry in entries if entry.resource_type == "file"
|
||||
)
|
||||
)
|
||||
folder_ids = list(
|
||||
dict.fromkeys(
|
||||
entry.resource_id for entry in entries if entry.resource_type == "folder"
|
||||
)
|
||||
)
|
||||
return file_ids, folder_ids
|
||||
|
||||
|
||||
def _visible_assets_for_delta(
|
||||
session: Session,
|
||||
*,
|
||||
principal: ApiPrincipal,
|
||||
owner_type: Literal["user", "group"] | None,
|
||||
owner_id: str | None,
|
||||
campaign_id: str | None,
|
||||
path_prefix: str | None,
|
||||
changed_file_ids: list[str],
|
||||
) -> dict[str, FileAsset]:
|
||||
return {
|
||||
asset.id: asset
|
||||
for asset in list_assets_for_user(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
user_id=principal.user.id,
|
||||
owner_type=owner_type,
|
||||
owner_id=owner_id,
|
||||
campaign_id=campaign_id,
|
||||
path_prefix=path_prefix,
|
||||
is_admin=_is_admin(principal),
|
||||
)
|
||||
if asset.id in changed_file_ids
|
||||
}
|
||||
|
||||
|
||||
def _changed_visible_folders_for_delta(
|
||||
session: Session,
|
||||
*,
|
||||
principal: ApiPrincipal,
|
||||
owner_type: Literal["user", "group"] | None,
|
||||
owner_id: str | None,
|
||||
path_prefix: str | None,
|
||||
changed_folder_ids: list[str],
|
||||
) -> dict[str, FileFolder]:
|
||||
return {
|
||||
folder.id: folder
|
||||
for folder in _visible_folders_for_delta(
|
||||
session,
|
||||
principal=principal,
|
||||
owner_type=owner_type,
|
||||
owner_id=owner_id,
|
||||
path_prefix=path_prefix,
|
||||
)
|
||||
if folder.id in changed_folder_ids
|
||||
}
|
||||
|
||||
|
||||
def _deleted_delta_items(
|
||||
session: Session,
|
||||
*,
|
||||
principal: ApiPrincipal,
|
||||
entries: list[ChangeSequenceEntry],
|
||||
visible_assets: dict[str, FileAsset],
|
||||
visible_folders: dict[str, FileFolder],
|
||||
owner_type: Literal["user", "group"] | None,
|
||||
owner_id: str | None,
|
||||
campaign_id: str | None,
|
||||
path_prefix: str | None,
|
||||
) -> list[DeltaDeletedItem]:
|
||||
deleted: dict[tuple[str, str], DeltaDeletedItem] = {}
|
||||
for entry in entries:
|
||||
if entry.resource_type == "file" and entry.resource_id in visible_assets:
|
||||
continue
|
||||
if entry.resource_type == "folder" and entry.resource_id in visible_folders:
|
||||
continue
|
||||
if not _entry_matches_delta_scope(
|
||||
session,
|
||||
principal,
|
||||
entry,
|
||||
owner_type=owner_type,
|
||||
owner_id=owner_id,
|
||||
campaign_id=campaign_id,
|
||||
path_prefix=path_prefix,
|
||||
):
|
||||
continue
|
||||
deleted[(entry.resource_type, entry.resource_id)] = DeltaDeletedItem(
|
||||
id=entry.resource_id,
|
||||
resource_type=entry.resource_type,
|
||||
revision=encode_sequence_watermark(entry.id),
|
||||
deleted_at=entry.created_at if entry.operation == "deleted" else None,
|
||||
)
|
||||
return list(deleted.values())
|
||||
|
||||
|
||||
def _files_delta_response(
|
||||
session: Session,
|
||||
*,
|
||||
principal: ApiPrincipal,
|
||||
owner_type: Literal["user", "group"] | None,
|
||||
owner_id: str | None,
|
||||
campaign_id: str | None,
|
||||
path_prefix: str | None,
|
||||
since: str,
|
||||
limit: int,
|
||||
) -> FileDeltaResponse:
|
||||
since_sequence = _decode_files_delta_watermark(since)
|
||||
if sequence_watermark_is_expired(
|
||||
session,
|
||||
since=since_sequence,
|
||||
tenant_id=principal.tenant_id,
|
||||
module_id=FILES_MODULE_ID,
|
||||
collections=_FILES_DELTA_COLLECTIONS,
|
||||
):
|
||||
return _full_file_delta_response(
|
||||
session,
|
||||
principal=principal,
|
||||
owner_type=owner_type,
|
||||
owner_id=owner_id,
|
||||
campaign_id=campaign_id,
|
||||
path_prefix=path_prefix,
|
||||
)
|
||||
|
||||
entries_plus_one = sequence_entries_since(
|
||||
session,
|
||||
since=since_sequence,
|
||||
tenant_id=principal.tenant_id,
|
||||
module_id=FILES_MODULE_ID,
|
||||
collections=_FILES_DELTA_COLLECTIONS,
|
||||
limit=limit + 1,
|
||||
)
|
||||
has_more = len(entries_plus_one) > limit
|
||||
entries = entries_plus_one[:limit]
|
||||
|
||||
changed_file_ids, changed_folder_ids = _changed_delta_resource_ids(entries)
|
||||
visible_assets = _visible_assets_for_delta(
|
||||
session,
|
||||
principal=principal,
|
||||
owner_type=owner_type,
|
||||
owner_id=owner_id,
|
||||
campaign_id=campaign_id,
|
||||
path_prefix=path_prefix,
|
||||
changed_file_ids=changed_file_ids,
|
||||
)
|
||||
visible_folders = _changed_visible_folders_for_delta(
|
||||
session,
|
||||
principal=principal,
|
||||
owner_type=owner_type,
|
||||
owner_id=owner_id,
|
||||
path_prefix=path_prefix,
|
||||
changed_folder_ids=changed_folder_ids,
|
||||
)
|
||||
deleted = _deleted_delta_items(
|
||||
session,
|
||||
principal=principal,
|
||||
entries=entries,
|
||||
visible_assets=visible_assets,
|
||||
visible_folders=visible_folders,
|
||||
owner_type=owner_type,
|
||||
owner_id=owner_id,
|
||||
campaign_id=campaign_id,
|
||||
path_prefix=path_prefix,
|
||||
)
|
||||
|
||||
watermark = (
|
||||
encode_sequence_watermark(entries[-1].id)
|
||||
if has_more and entries
|
||||
else _files_delta_watermark(session, principal.tenant_id)
|
||||
)
|
||||
return FileDeltaResponse(
|
||||
files=_asset_list_response(
|
||||
session, list(visible_assets.values()), include_shares=True
|
||||
),
|
||||
folders=[_folder_response(folder) for folder in visible_folders.values()],
|
||||
deleted=deleted,
|
||||
watermark=watermark,
|
||||
has_more=has_more,
|
||||
full=False,
|
||||
)
|
||||
@@ -1,55 +1,91 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import mimetypes
|
||||
import re
|
||||
import stat
|
||||
import tarfile
|
||||
import zipfile
|
||||
from dataclasses import dataclass
|
||||
from io import BytesIO
|
||||
from os import PathLike
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Any, BinaryIO, Iterable, Literal
|
||||
|
||||
import pyzipper
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from govoplan_files.backend.db.models import FileAsset
|
||||
from govoplan_files.backend.storage.common import FileConflictResolution, FileStorageError, UploadedStoredFile
|
||||
from govoplan_files.backend.storage.backends import StorageBackendError, get_storage_backend
|
||||
from govoplan_files.backend.storage.files import create_file_asset, current_versions_and_blobs
|
||||
from govoplan_files.backend.storage.paths import filename_from_path, normalize_folder, normalize_logical_path
|
||||
from govoplan_files.backend.storage.backends import (
|
||||
StorageBackendError,
|
||||
get_storage_backend,
|
||||
)
|
||||
from govoplan_files.backend.storage.common import (
|
||||
FileConflictResolution,
|
||||
FileStorageError,
|
||||
UploadedStoredFile,
|
||||
)
|
||||
from govoplan_files.backend.storage.files import (
|
||||
create_file_asset,
|
||||
current_versions_and_blobs,
|
||||
)
|
||||
from govoplan_files.backend.storage.paths import (
|
||||
filename_from_path,
|
||||
normalize_folder,
|
||||
normalize_logical_path,
|
||||
)
|
||||
|
||||
|
||||
_ZIP_READ_CHUNK_SIZE = 1024 * 1024
|
||||
ZIP_UPLOAD_MAX_FILES = 1000
|
||||
_ARCHIVE_READ_CHUNK_SIZE = 1024 * 1024
|
||||
_WINDOWS_DRIVE_RE = re.compile(r"^[A-Za-z]:")
|
||||
ARCHIVE_UPLOAD_MAX_ENTRIES = 10_000
|
||||
# Kept for callers and documentation using the former ZIP-specific name.
|
||||
ZIP_UPLOAD_MAX_FILES = ARCHIVE_UPLOAD_MAX_ENTRIES
|
||||
SUPPORTED_ARCHIVE_SUFFIXES = (
|
||||
".tar.bz2",
|
||||
".tar.gz",
|
||||
".tar.xz",
|
||||
".tbz2",
|
||||
".tgz",
|
||||
".txz",
|
||||
".tar",
|
||||
".zip",
|
||||
)
|
||||
|
||||
|
||||
def _read_zip_member(
|
||||
archive: zipfile.ZipFile,
|
||||
info: zipfile.ZipInfo,
|
||||
*,
|
||||
max_file_bytes: int,
|
||||
max_total_bytes: int,
|
||||
current_total: int,
|
||||
) -> tuple[bytes, int]:
|
||||
parts: list[bytes] = []
|
||||
actual_size = 0
|
||||
with archive.open(info) as source:
|
||||
while True:
|
||||
read_size = min(_ZIP_READ_CHUNK_SIZE, max_file_bytes + 1 - actual_size)
|
||||
chunk = source.read(read_size)
|
||||
if not chunk:
|
||||
break
|
||||
actual_size += len(chunk)
|
||||
if actual_size > max_file_bytes:
|
||||
raise FileStorageError(f"ZIP member {info.filename!r} exceeds per-file limit")
|
||||
if current_total + actual_size > max_total_bytes:
|
||||
raise FileStorageError("ZIP is too large after extraction")
|
||||
parts.append(chunk)
|
||||
return b"".join(parts), current_total + actual_size
|
||||
class ArchivePasswordError(FileStorageError):
|
||||
pass
|
||||
|
||||
|
||||
def create_zip_file(session: Session, assets: Iterable[FileAsset], output_path: str | Path) -> None:
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ArchiveEntry:
|
||||
path: str
|
||||
kind: Literal["file", "directory"]
|
||||
size_bytes: int
|
||||
compressed_size_bytes: int | None = None
|
||||
encrypted: bool = False
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ArchiveInspection:
|
||||
archive_format: str
|
||||
entries: tuple[ArchiveEntry, ...]
|
||||
file_count: int
|
||||
directory_count: int
|
||||
expanded_size_bytes: int
|
||||
compressed_size_bytes: int
|
||||
requires_password: bool
|
||||
password_verified: bool
|
||||
|
||||
|
||||
def create_zip_file(
|
||||
session: Session, assets: Iterable[FileAsset], output_path: str | Path
|
||||
) -> None:
|
||||
backend = get_storage_backend()
|
||||
asset_list = list(assets)
|
||||
version_blobs = current_versions_and_blobs(session, asset_list)
|
||||
with zipfile.ZipFile(output_path, mode="w", compression=zipfile.ZIP_DEFLATED) as archive:
|
||||
with zipfile.ZipFile(
|
||||
output_path, mode="w", compression=zipfile.ZIP_DEFLATED
|
||||
) as archive:
|
||||
for asset in asset_list:
|
||||
_version, blob = version_blobs[asset.id]
|
||||
info = zipfile.ZipInfo(asset.display_path)
|
||||
@@ -63,6 +99,147 @@ def create_zip_file(session: Session, assets: Iterable[FileAsset], output_path:
|
||||
raise FileStorageError(str(exc)) from exc
|
||||
|
||||
|
||||
def archive_format_for_filename(filename: str) -> str:
|
||||
lowered = filename.strip().casefold()
|
||||
if lowered.endswith(".zip"):
|
||||
return "zip"
|
||||
if lowered.endswith((".tar.gz", ".tgz")):
|
||||
return "tar.gz"
|
||||
if lowered.endswith((".tar.bz2", ".tbz2")):
|
||||
return "tar.bz2"
|
||||
if lowered.endswith((".tar.xz", ".txz")):
|
||||
return "tar.xz"
|
||||
if lowered.endswith(".tar"):
|
||||
return "tar"
|
||||
raise FileStorageError(
|
||||
"Unsupported archive format. Use ZIP, TAR, TAR.GZ, TAR.BZ2, or TAR.XZ."
|
||||
)
|
||||
|
||||
|
||||
def is_supported_archive_filename(filename: str) -> bool:
|
||||
try:
|
||||
archive_format_for_filename(filename)
|
||||
except FileStorageError:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def inspect_archive(
|
||||
archive_data: bytes | str | PathLike[str],
|
||||
*,
|
||||
filename: str,
|
||||
password: str | None = None,
|
||||
max_entries: int = ARCHIVE_UPLOAD_MAX_ENTRIES,
|
||||
max_expanded_bytes: int = 2 * 1024 * 1024 * 1024,
|
||||
max_expansion_ratio: int = 100,
|
||||
) -> ArchiveInspection:
|
||||
archive_format = archive_format_for_filename(filename)
|
||||
compressed_size = _archive_size(archive_data)
|
||||
if archive_format == "zip":
|
||||
entries, requires_password, password_verified = _inspect_zip(
|
||||
archive_data,
|
||||
password=password,
|
||||
)
|
||||
else:
|
||||
entries = _inspect_tar(archive_data)
|
||||
requires_password = False
|
||||
password_verified = True
|
||||
_validate_archive_limits(
|
||||
entries,
|
||||
compressed_size=compressed_size,
|
||||
max_entries=max_entries,
|
||||
max_expanded_bytes=max_expanded_bytes,
|
||||
max_expansion_ratio=max_expansion_ratio,
|
||||
)
|
||||
complete_entries = _with_derived_directories(entries)
|
||||
return ArchiveInspection(
|
||||
archive_format=archive_format,
|
||||
entries=tuple(complete_entries),
|
||||
file_count=sum(entry.kind == "file" for entry in complete_entries),
|
||||
directory_count=sum(
|
||||
entry.kind == "directory" for entry in complete_entries
|
||||
),
|
||||
expanded_size_bytes=sum(
|
||||
entry.size_bytes for entry in entries if entry.kind == "file"
|
||||
),
|
||||
compressed_size_bytes=compressed_size,
|
||||
requires_password=requires_password,
|
||||
password_verified=password_verified,
|
||||
)
|
||||
|
||||
|
||||
def extract_archive_upload(
|
||||
session: Session,
|
||||
*,
|
||||
tenant_id: str,
|
||||
owner_type: str,
|
||||
owner_id: str,
|
||||
user_id: str,
|
||||
archive_data: bytes | str | PathLike[str],
|
||||
filename: str,
|
||||
folder: str | None,
|
||||
campaign_id: str | None,
|
||||
selected_paths: Iterable[str] | None = None,
|
||||
password: str | None = None,
|
||||
conflict_strategy: str = "reject",
|
||||
conflict_resolutions: Iterable[FileConflictResolution] | None = None,
|
||||
metadata: dict[str, Any] | None = None,
|
||||
is_admin: bool = False,
|
||||
encryption_vault_id: str | None = None,
|
||||
max_entries: int = ARCHIVE_UPLOAD_MAX_ENTRIES,
|
||||
max_file_bytes: int = 50 * 1024 * 1024,
|
||||
max_expanded_bytes: int = 2 * 1024 * 1024 * 1024,
|
||||
max_expansion_ratio: int = 100,
|
||||
) -> list[UploadedStoredFile]:
|
||||
inspection = inspect_archive(
|
||||
archive_data,
|
||||
filename=filename,
|
||||
password=password,
|
||||
max_entries=max_entries,
|
||||
max_expanded_bytes=max_expanded_bytes,
|
||||
max_expansion_ratio=max_expansion_ratio,
|
||||
)
|
||||
if inspection.requires_password and not inspection.password_verified:
|
||||
raise ArchivePasswordError("Archive password is required")
|
||||
selected_files = _selected_file_paths(inspection.entries, selected_paths)
|
||||
if not selected_files:
|
||||
raise FileStorageError("Select at least one archive file to import")
|
||||
actual_total_limit = min(
|
||||
max_expanded_bytes,
|
||||
inspection.compressed_size_bytes * max_expansion_ratio,
|
||||
)
|
||||
if inspection.archive_format == "zip":
|
||||
members = _read_selected_zip_members(
|
||||
archive_data,
|
||||
selected_files=selected_files,
|
||||
password=password,
|
||||
max_file_bytes=max_file_bytes,
|
||||
max_total_bytes=actual_total_limit,
|
||||
)
|
||||
else:
|
||||
members = _read_selected_tar_members(
|
||||
archive_data,
|
||||
selected_files=selected_files,
|
||||
max_file_bytes=max_file_bytes,
|
||||
max_total_bytes=actual_total_limit,
|
||||
)
|
||||
return _store_archive_members(
|
||||
session,
|
||||
members=members,
|
||||
tenant_id=tenant_id,
|
||||
owner_type=owner_type,
|
||||
owner_id=owner_id,
|
||||
user_id=user_id,
|
||||
folder=folder,
|
||||
campaign_id=campaign_id,
|
||||
conflict_strategy=conflict_strategy,
|
||||
conflict_resolutions=conflict_resolutions,
|
||||
metadata=metadata,
|
||||
is_admin=is_admin,
|
||||
encryption_vault_id=encryption_vault_id,
|
||||
)
|
||||
|
||||
|
||||
def extract_zip_upload(
|
||||
session: Session,
|
||||
*,
|
||||
@@ -77,55 +254,409 @@ def extract_zip_upload(
|
||||
conflict_resolutions: Iterable[FileConflictResolution] | None = None,
|
||||
metadata: dict[str, Any] | None = None,
|
||||
is_admin: bool = False,
|
||||
encryption_vault_id: str | None = None,
|
||||
max_files: int = ZIP_UPLOAD_MAX_FILES,
|
||||
max_file_bytes: int = 50 * 1024 * 1024,
|
||||
max_total_bytes: int = 250 * 1024 * 1024,
|
||||
) -> list[UploadedStoredFile]:
|
||||
uploaded: list[UploadedStoredFile] = []
|
||||
total = 0
|
||||
base_folder = normalize_folder(folder)
|
||||
"""Backward-compatible wrapper for the original immediate ZIP endpoint."""
|
||||
|
||||
return extract_archive_upload(
|
||||
session,
|
||||
tenant_id=tenant_id,
|
||||
owner_type=owner_type,
|
||||
owner_id=owner_id,
|
||||
user_id=user_id,
|
||||
archive_data=zip_data,
|
||||
filename="archive.zip",
|
||||
folder=folder,
|
||||
campaign_id=campaign_id,
|
||||
conflict_strategy=conflict_strategy,
|
||||
conflict_resolutions=conflict_resolutions,
|
||||
metadata=metadata,
|
||||
is_admin=is_admin,
|
||||
encryption_vault_id=encryption_vault_id,
|
||||
max_entries=max_files,
|
||||
max_file_bytes=max_file_bytes,
|
||||
max_expanded_bytes=max_total_bytes,
|
||||
)
|
||||
|
||||
|
||||
def _inspect_zip(
|
||||
archive_data: bytes | str | PathLike[str],
|
||||
*,
|
||||
password: str | None,
|
||||
) -> tuple[list[ArchiveEntry], bool, bool]:
|
||||
try:
|
||||
source = BytesIO(zip_data) if isinstance(zip_data, bytes) else zip_data
|
||||
with zipfile.ZipFile(source) as archive:
|
||||
infos = [info for info in archive.infolist() if not info.is_dir()]
|
||||
if len(infos) > max_files:
|
||||
raise FileStorageError(f"ZIP contains too many files (limit {max_files})")
|
||||
for info in infos:
|
||||
if info.flag_bits & 0x1:
|
||||
raise FileStorageError("Encrypted ZIP uploads are not supported")
|
||||
if info.file_size < 0:
|
||||
raise FileStorageError("Invalid ZIP member")
|
||||
if info.file_size > max_file_bytes:
|
||||
raise FileStorageError(f"ZIP member {info.filename!r} exceeds per-file limit")
|
||||
if total + info.file_size > max_total_bytes:
|
||||
raise FileStorageError("ZIP is too large after extraction")
|
||||
inner_path = normalize_logical_path(info.filename)
|
||||
target_path = f"{base_folder}/{inner_path}" if base_folder else inner_path
|
||||
data, total = _read_zip_member(
|
||||
archive,
|
||||
info,
|
||||
max_file_bytes=max_file_bytes,
|
||||
max_total_bytes=max_total_bytes,
|
||||
current_total=total,
|
||||
)
|
||||
uploaded.append(
|
||||
create_file_asset(
|
||||
session,
|
||||
tenant_id=tenant_id,
|
||||
owner_type=owner_type,
|
||||
owner_id=owner_id,
|
||||
user_id=user_id,
|
||||
filename=filename_from_path(inner_path),
|
||||
data=data,
|
||||
display_path=target_path,
|
||||
content_type=mimetypes.guess_type(inner_path)[0] or "application/octet-stream",
|
||||
metadata=metadata,
|
||||
campaign_id=campaign_id,
|
||||
conflict_strategy=conflict_strategy,
|
||||
conflict_resolutions=conflict_resolutions,
|
||||
is_admin=is_admin,
|
||||
with pyzipper.AESZipFile(_archive_source(archive_data)) as archive:
|
||||
infos = archive.infolist()
|
||||
entries = [_zip_entry(info) for info in infos]
|
||||
encrypted_info = next(
|
||||
(
|
||||
info
|
||||
for info in infos
|
||||
if not info.is_dir() and bool(info.flag_bits & 0x1)
|
||||
),
|
||||
None,
|
||||
)
|
||||
if encrypted_info is None:
|
||||
return entries, False, True
|
||||
if not password:
|
||||
return entries, True, False
|
||||
try:
|
||||
with archive.open(
|
||||
encrypted_info, pwd=password.encode("utf-8")
|
||||
) as member:
|
||||
member.read(1)
|
||||
except (RuntimeError, ValueError, zipfile.BadZipFile) as exc:
|
||||
raise ArchivePasswordError("Archive password is incorrect") from exc
|
||||
return entries, True, True
|
||||
except ArchivePasswordError:
|
||||
raise
|
||||
except (OSError, ValueError, zipfile.BadZipFile) as exc:
|
||||
raise FileStorageError("Invalid ZIP upload") from exc
|
||||
|
||||
|
||||
def _inspect_tar(
|
||||
archive_data: bytes | str | PathLike[str],
|
||||
) -> list[ArchiveEntry]:
|
||||
try:
|
||||
with _open_tar(archive_data) as archive:
|
||||
entries: list[ArchiveEntry] = []
|
||||
for member in archive.getmembers():
|
||||
path = _safe_member_path(member.name)
|
||||
if member.isdir():
|
||||
entries.append(
|
||||
ArchiveEntry(path=path, kind="directory", size_bytes=0)
|
||||
)
|
||||
continue
|
||||
if not member.isfile():
|
||||
raise FileStorageError(
|
||||
f"Archive member {member.name!r} is not a regular file or directory"
|
||||
)
|
||||
entries.append(
|
||||
ArchiveEntry(
|
||||
path=path,
|
||||
kind="file",
|
||||
size_bytes=max(0, int(member.size)),
|
||||
)
|
||||
)
|
||||
except zipfile.BadZipFile as exc:
|
||||
raise FileStorageError("Invalid ZIP upload") from exc
|
||||
return entries
|
||||
except FileStorageError:
|
||||
raise
|
||||
except (OSError, tarfile.TarError) as exc:
|
||||
raise FileStorageError("Invalid TAR upload") from exc
|
||||
|
||||
|
||||
def _zip_entry(info: zipfile.ZipInfo) -> ArchiveEntry:
|
||||
path = _safe_member_path(info.filename)
|
||||
unix_mode = (info.external_attr >> 16) & 0xFFFF
|
||||
file_type = stat.S_IFMT(unix_mode)
|
||||
if file_type and not (
|
||||
stat.S_ISREG(unix_mode) or stat.S_ISDIR(unix_mode)
|
||||
):
|
||||
raise FileStorageError(
|
||||
f"Archive member {info.filename!r} is not a regular file or directory"
|
||||
)
|
||||
if info.file_size < 0 or info.compress_size < 0:
|
||||
raise FileStorageError(f"Archive member {info.filename!r} has invalid size metadata")
|
||||
return ArchiveEntry(
|
||||
path=path,
|
||||
kind="directory" if info.is_dir() else "file",
|
||||
size_bytes=0 if info.is_dir() else int(info.file_size),
|
||||
compressed_size_bytes=(
|
||||
None if info.is_dir() else int(info.compress_size)
|
||||
),
|
||||
encrypted=bool(info.flag_bits & 0x1),
|
||||
)
|
||||
|
||||
|
||||
def _validate_archive_limits(
|
||||
entries: list[ArchiveEntry],
|
||||
*,
|
||||
compressed_size: int,
|
||||
max_entries: int,
|
||||
max_expanded_bytes: int,
|
||||
max_expansion_ratio: int,
|
||||
) -> None:
|
||||
if len(entries) > max_entries:
|
||||
raise FileStorageError(
|
||||
f"Archive contains too many entries (limit {max_entries})"
|
||||
)
|
||||
seen: dict[str, str] = {}
|
||||
expanded_size = 0
|
||||
for entry in entries:
|
||||
previous_kind = seen.get(entry.path)
|
||||
if previous_kind is not None:
|
||||
raise FileStorageError(
|
||||
f"Archive contains duplicate path {entry.path!r}"
|
||||
)
|
||||
seen[entry.path] = entry.kind
|
||||
if entry.kind == "file":
|
||||
expanded_size += entry.size_bytes
|
||||
if expanded_size > max_expanded_bytes:
|
||||
raise FileStorageError(
|
||||
"Archive is too large after extraction "
|
||||
f"(limit {max_expanded_bytes} bytes)"
|
||||
)
|
||||
if expanded_size and (
|
||||
compressed_size <= 0
|
||||
or expanded_size > compressed_size * max_expansion_ratio
|
||||
):
|
||||
raise FileStorageError(
|
||||
"Archive expansion ratio exceeds "
|
||||
f"{max_expansion_ratio}:1"
|
||||
)
|
||||
|
||||
|
||||
def _with_derived_directories(
|
||||
entries: list[ArchiveEntry],
|
||||
) -> list[ArchiveEntry]:
|
||||
by_path = {entry.path: entry for entry in entries}
|
||||
for entry in entries:
|
||||
parent = PurePosixPath(entry.path).parent
|
||||
while str(parent) not in {"", "."}:
|
||||
path = str(parent)
|
||||
existing = by_path.get(path)
|
||||
if existing and existing.kind != "directory":
|
||||
raise FileStorageError(
|
||||
f"Archive path {path!r} is both a file and a directory"
|
||||
)
|
||||
by_path.setdefault(
|
||||
path,
|
||||
ArchiveEntry(path=path, kind="directory", size_bytes=0),
|
||||
)
|
||||
parent = parent.parent
|
||||
return sorted(
|
||||
by_path.values(),
|
||||
key=lambda entry: (
|
||||
tuple(entry.path.casefold().split("/")),
|
||||
entry.kind != "directory",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _selected_file_paths(
|
||||
entries: tuple[ArchiveEntry, ...],
|
||||
selected_paths: Iterable[str] | None,
|
||||
) -> set[str]:
|
||||
file_paths = {entry.path for entry in entries if entry.kind == "file"}
|
||||
if selected_paths is None:
|
||||
return file_paths
|
||||
known = {entry.path: entry for entry in entries}
|
||||
normalized = {_safe_member_path(path) for path in selected_paths}
|
||||
unknown = normalized - known.keys()
|
||||
if unknown:
|
||||
raise FileStorageError(
|
||||
f"Archive selection contains unknown path {sorted(unknown)[0]!r}"
|
||||
)
|
||||
selected_files: set[str] = set()
|
||||
for path in normalized:
|
||||
entry = known[path]
|
||||
if entry.kind == "file":
|
||||
selected_files.add(path)
|
||||
continue
|
||||
prefix = f"{path}/"
|
||||
selected_files.update(
|
||||
file_path
|
||||
for file_path in file_paths
|
||||
if file_path.startswith(prefix)
|
||||
)
|
||||
return selected_files
|
||||
|
||||
|
||||
def _read_selected_zip_members(
|
||||
archive_data: bytes | str | PathLike[str],
|
||||
*,
|
||||
selected_files: set[str],
|
||||
password: str | None,
|
||||
max_file_bytes: int,
|
||||
max_total_bytes: int,
|
||||
) -> list[tuple[str, bytes]]:
|
||||
result: list[tuple[str, bytes]] = []
|
||||
total = 0
|
||||
try:
|
||||
with pyzipper.AESZipFile(_archive_source(archive_data)) as archive:
|
||||
for info in archive.infolist():
|
||||
if info.is_dir():
|
||||
continue
|
||||
path = _safe_member_path(info.filename)
|
||||
if path not in selected_files:
|
||||
continue
|
||||
pwd = password.encode("utf-8") if password else None
|
||||
try:
|
||||
with archive.open(info, pwd=pwd) as source:
|
||||
data, total = _read_member(
|
||||
source,
|
||||
path=path,
|
||||
max_file_bytes=max_file_bytes,
|
||||
max_total_bytes=max_total_bytes,
|
||||
current_total=total,
|
||||
)
|
||||
except (RuntimeError, ValueError, zipfile.BadZipFile) as exc:
|
||||
if info.flag_bits & 0x1:
|
||||
raise ArchivePasswordError(
|
||||
"Archive password is incorrect"
|
||||
) from exc
|
||||
raise
|
||||
result.append((path, data))
|
||||
except (FileStorageError, ArchivePasswordError):
|
||||
raise
|
||||
except (OSError, ValueError, zipfile.BadZipFile) as exc:
|
||||
raise FileStorageError("ZIP extraction failed") from exc
|
||||
return result
|
||||
|
||||
|
||||
def _read_selected_tar_members(
|
||||
archive_data: bytes | str | PathLike[str],
|
||||
*,
|
||||
selected_files: set[str],
|
||||
max_file_bytes: int,
|
||||
max_total_bytes: int,
|
||||
) -> list[tuple[str, bytes]]:
|
||||
result: list[tuple[str, bytes]] = []
|
||||
total = 0
|
||||
try:
|
||||
with _open_tar(archive_data) as archive:
|
||||
for member in archive.getmembers():
|
||||
if not member.isfile():
|
||||
continue
|
||||
path = _safe_member_path(member.name)
|
||||
if path not in selected_files:
|
||||
continue
|
||||
source = archive.extractfile(member)
|
||||
if source is None:
|
||||
raise FileStorageError(
|
||||
f"Archive member {path!r} could not be read"
|
||||
)
|
||||
with source:
|
||||
data, total = _read_member(
|
||||
source,
|
||||
path=path,
|
||||
max_file_bytes=max_file_bytes,
|
||||
max_total_bytes=max_total_bytes,
|
||||
current_total=total,
|
||||
)
|
||||
result.append((path, data))
|
||||
except FileStorageError:
|
||||
raise
|
||||
except (OSError, tarfile.TarError) as exc:
|
||||
raise FileStorageError("TAR extraction failed") from exc
|
||||
return result
|
||||
|
||||
|
||||
def _read_member(
|
||||
source: BinaryIO,
|
||||
*,
|
||||
path: str,
|
||||
max_file_bytes: int,
|
||||
max_total_bytes: int,
|
||||
current_total: int,
|
||||
) -> tuple[bytes, int]:
|
||||
parts: list[bytes] = []
|
||||
actual_size = 0
|
||||
while True:
|
||||
read_size = min(
|
||||
_ARCHIVE_READ_CHUNK_SIZE,
|
||||
max_file_bytes + 1 - actual_size,
|
||||
)
|
||||
chunk = source.read(read_size)
|
||||
if not chunk:
|
||||
break
|
||||
actual_size += len(chunk)
|
||||
if actual_size > max_file_bytes:
|
||||
raise FileStorageError(
|
||||
f"Archive member {path!r} exceeds per-file limit"
|
||||
)
|
||||
if current_total + actual_size > max_total_bytes:
|
||||
raise FileStorageError("Archive is too large after extraction")
|
||||
parts.append(chunk)
|
||||
return b"".join(parts), current_total + actual_size
|
||||
|
||||
|
||||
def _store_archive_members(
|
||||
session: Session,
|
||||
*,
|
||||
members: Iterable[tuple[str, bytes]],
|
||||
tenant_id: str,
|
||||
owner_type: str,
|
||||
owner_id: str,
|
||||
user_id: str,
|
||||
folder: str | None,
|
||||
campaign_id: str | None,
|
||||
conflict_strategy: str,
|
||||
conflict_resolutions: Iterable[FileConflictResolution] | None,
|
||||
metadata: dict[str, Any] | None,
|
||||
is_admin: bool,
|
||||
encryption_vault_id: str | None,
|
||||
) -> list[UploadedStoredFile]:
|
||||
uploaded: list[UploadedStoredFile] = []
|
||||
base_folder = normalize_folder(folder)
|
||||
for inner_path, data in members:
|
||||
target_path = (
|
||||
f"{base_folder}/{inner_path}" if base_folder else inner_path
|
||||
)
|
||||
uploaded.append(
|
||||
create_file_asset(
|
||||
session,
|
||||
tenant_id=tenant_id,
|
||||
owner_type=owner_type,
|
||||
owner_id=owner_id,
|
||||
user_id=user_id,
|
||||
filename=filename_from_path(inner_path),
|
||||
data=data,
|
||||
display_path=target_path,
|
||||
content_type=mimetypes.guess_type(inner_path)[0]
|
||||
or "application/octet-stream",
|
||||
metadata=metadata,
|
||||
campaign_id=campaign_id,
|
||||
conflict_strategy=conflict_strategy,
|
||||
conflict_resolutions=conflict_resolutions,
|
||||
is_admin=is_admin,
|
||||
encryption_vault_id=encryption_vault_id,
|
||||
)
|
||||
)
|
||||
return uploaded
|
||||
|
||||
|
||||
def _safe_member_path(value: str) -> str:
|
||||
raw = str(value or "").replace("\\", "/").strip()
|
||||
if (
|
||||
not raw
|
||||
or "\x00" in raw
|
||||
or raw.startswith("/")
|
||||
or _WINDOWS_DRIVE_RE.match(raw)
|
||||
):
|
||||
raise FileStorageError(f"Unsafe archive member path {value!r}")
|
||||
if any(part == ".." for part in raw.split("/")):
|
||||
raise FileStorageError(f"Unsafe archive member path {value!r}")
|
||||
try:
|
||||
return normalize_logical_path(raw.rstrip("/"))
|
||||
except ValueError as exc:
|
||||
raise FileStorageError(f"Unsafe archive member path {value!r}") from exc
|
||||
|
||||
|
||||
def _archive_size(archive_data: bytes | str | PathLike[str]) -> int:
|
||||
if isinstance(archive_data, bytes):
|
||||
return len(archive_data)
|
||||
try:
|
||||
return Path(archive_data).stat().st_size
|
||||
except OSError as exc:
|
||||
raise FileStorageError("Archive upload could not be read") from exc
|
||||
|
||||
|
||||
def _archive_source(
|
||||
archive_data: bytes | str | PathLike[str],
|
||||
) -> BytesIO | str | PathLike[str]:
|
||||
return BytesIO(archive_data) if isinstance(archive_data, bytes) else archive_data
|
||||
|
||||
|
||||
def _open_tar(
|
||||
archive_data: bytes | str | PathLike[str],
|
||||
) -> tarfile.TarFile:
|
||||
if isinstance(archive_data, bytes):
|
||||
return tarfile.open(fileobj=BytesIO(archive_data), mode="r:*")
|
||||
try:
|
||||
return tarfile.open(name=archive_data, mode="r:*")
|
||||
except OSError as exc:
|
||||
raise FileStorageError("Archive upload could not be read") from exc
|
||||
|
||||
@@ -1,207 +1,32 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Iterable, Protocol
|
||||
|
||||
from govoplan_core.security.outbound_http import (
|
||||
OutboundHttpError,
|
||||
response_limit,
|
||||
validate_unpinned_sdk_http_url,
|
||||
from govoplan_core.core.object_storage import (
|
||||
LocalFilesystemStorageBackend,
|
||||
S3StorageBackend,
|
||||
StorageBackend,
|
||||
StorageBackendError,
|
||||
StorageObjectInfo,
|
||||
StorageObjectMissing,
|
||||
StorageObjectPage,
|
||||
configured_storage_backend,
|
||||
)
|
||||
|
||||
from govoplan_files.backend.runtime import settings
|
||||
|
||||
|
||||
class StorageBackendError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
class StorageBackend(Protocol):
|
||||
name: str
|
||||
|
||||
def put_bytes(self, key: str, data: bytes, *, content_type: str | None = None) -> None: ...
|
||||
def get_bytes(self, key: str) -> bytes: ...
|
||||
def iter_bytes(self, key: str, *, chunk_size: int = 1024 * 1024) -> Iterable[bytes]: ...
|
||||
def delete(self, key: str) -> None: ...
|
||||
def exists(self, key: str) -> bool: ...
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class LocalFilesystemStorageBackend:
|
||||
root: Path
|
||||
fallback_roots: tuple[Path, ...] = field(default_factory=tuple)
|
||||
name: str = "local"
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
self.root = self.root.expanduser().resolve()
|
||||
self.fallback_roots = tuple(root.expanduser().resolve() for root in self.fallback_roots if root)
|
||||
self.root.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def _path_for_root(self, root: Path, key: str) -> Path:
|
||||
path = (root / key).resolve()
|
||||
if not path.is_relative_to(root):
|
||||
raise StorageBackendError("Storage key escapes local storage root")
|
||||
return path
|
||||
|
||||
def _path(self, key: str) -> Path:
|
||||
return self._path_for_root(self.root, key)
|
||||
|
||||
def _readable_path(self, key: str) -> Path:
|
||||
primary = self._path(key)
|
||||
if primary.exists() and primary.is_file():
|
||||
return primary
|
||||
for root in self.fallback_roots:
|
||||
candidate = self._path_for_root(root, key)
|
||||
if candidate.exists() and candidate.is_file():
|
||||
return candidate
|
||||
raise StorageBackendError("Stored object does not exist")
|
||||
|
||||
def put_bytes(self, key: str, data: bytes, *, content_type: str | None = None) -> None:
|
||||
path = self._path(key)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_bytes(data)
|
||||
|
||||
def get_bytes(self, key: str) -> bytes:
|
||||
return self._readable_path(key).read_bytes()
|
||||
|
||||
def iter_bytes(self, key: str, *, chunk_size: int = 1024 * 1024) -> Iterable[bytes]:
|
||||
path = self._readable_path(key)
|
||||
with path.open("rb") as handle:
|
||||
while True:
|
||||
chunk = handle.read(chunk_size)
|
||||
if not chunk:
|
||||
break
|
||||
yield chunk
|
||||
|
||||
def delete(self, key: str) -> None:
|
||||
path = self._path(key)
|
||||
if path.exists() and path.is_file():
|
||||
path.unlink()
|
||||
|
||||
def exists(self, key: str) -> bool:
|
||||
try:
|
||||
self._readable_path(key)
|
||||
except StorageBackendError:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class S3StorageBackend:
|
||||
bucket: str
|
||||
endpoint_url: str
|
||||
region_name: str
|
||||
access_key_id: str
|
||||
secret_access_key: str
|
||||
name: str = "s3"
|
||||
|
||||
@property
|
||||
def client(self):
|
||||
try:
|
||||
endpoint_url = validate_unpinned_sdk_http_url(
|
||||
self.endpoint_url,
|
||||
label="File storage S3 endpoint",
|
||||
)
|
||||
except OutboundHttpError as exc:
|
||||
raise StorageBackendError(str(exc)) from exc
|
||||
try:
|
||||
import boto3
|
||||
except ModuleNotFoundError as exc:
|
||||
raise StorageBackendError("boto3 is required for the S3 storage backend") from exc
|
||||
return boto3.client(
|
||||
"s3",
|
||||
endpoint_url=endpoint_url,
|
||||
region_name=self.region_name,
|
||||
aws_access_key_id=self.access_key_id,
|
||||
aws_secret_access_key=self.secret_access_key,
|
||||
)
|
||||
|
||||
def put_bytes(self, key: str, data: bytes, *, content_type: str | None = None) -> None:
|
||||
max_bytes = response_limit("file")
|
||||
if len(data) > max_bytes:
|
||||
raise StorageBackendError(f"Stored object exceeds the deployment limit of {max_bytes} bytes")
|
||||
kwargs = {"Bucket": self.bucket, "Key": key, "Body": data}
|
||||
if content_type:
|
||||
kwargs["ContentType"] = content_type
|
||||
self.client.put_object(**kwargs)
|
||||
|
||||
def get_bytes(self, key: str) -> bytes:
|
||||
try:
|
||||
obj = self.client.get_object(Bucket=self.bucket, Key=key)
|
||||
max_bytes = response_limit("file")
|
||||
body = obj["Body"]
|
||||
try:
|
||||
_reject_declared_object_size(obj, max_bytes=max_bytes)
|
||||
data = body.read(max_bytes + 1)
|
||||
if len(data) > max_bytes:
|
||||
raise StorageBackendError(f"Stored object exceeds the deployment limit of {max_bytes} bytes")
|
||||
return data
|
||||
finally:
|
||||
if hasattr(body, "close"):
|
||||
body.close()
|
||||
except Exception as exc: # pragma: no cover - depends on S3 backend
|
||||
raise StorageBackendError(str(exc)) from exc
|
||||
|
||||
def iter_bytes(self, key: str, *, chunk_size: int = 1024 * 1024) -> Iterable[bytes]:
|
||||
try:
|
||||
obj = self.client.get_object(Bucket=self.bucket, Key=key)
|
||||
max_bytes = response_limit("file")
|
||||
body = obj["Body"]
|
||||
try:
|
||||
_reject_declared_object_size(obj, max_bytes=max_bytes)
|
||||
total = 0
|
||||
while True:
|
||||
chunk = body.read(chunk_size)
|
||||
if not chunk:
|
||||
break
|
||||
total += len(chunk)
|
||||
if total > max_bytes:
|
||||
raise StorageBackendError(f"Stored object exceeds the deployment limit of {max_bytes} bytes")
|
||||
yield chunk
|
||||
finally:
|
||||
if hasattr(body, "close"):
|
||||
body.close()
|
||||
except Exception as exc: # pragma: no cover - depends on S3 backend
|
||||
raise StorageBackendError(str(exc)) from exc
|
||||
|
||||
def delete(self, key: str) -> None:
|
||||
self.client.delete_object(Bucket=self.bucket, Key=key)
|
||||
|
||||
def exists(self, key: str) -> bool:
|
||||
try:
|
||||
self.client.head_object(Bucket=self.bucket, Key=key)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _reject_declared_object_size(obj: object, *, max_bytes: int) -> None:
|
||||
if not isinstance(obj, dict):
|
||||
return
|
||||
try:
|
||||
declared_size = int(obj.get("ContentLength"))
|
||||
except (TypeError, ValueError):
|
||||
return
|
||||
if declared_size > max_bytes:
|
||||
raise StorageBackendError(f"Stored object exceeds the deployment limit of {max_bytes} bytes")
|
||||
|
||||
|
||||
def _fallback_roots() -> tuple[Path, ...]:
|
||||
raw = getattr(settings, "file_storage_local_fallback_roots", "") or ""
|
||||
return tuple(Path(item.strip()) for item in str(raw).split(",") if item.strip())
|
||||
|
||||
|
||||
def get_storage_backend() -> StorageBackend:
|
||||
backend = settings.file_storage_backend.lower().strip()
|
||||
if backend in {"local", "filesystem", "fs"}:
|
||||
return LocalFilesystemStorageBackend(Path(settings.file_storage_local_root), fallback_roots=_fallback_roots())
|
||||
if backend in {"s3", "garage"}:
|
||||
return S3StorageBackend(
|
||||
bucket=settings.file_storage_s3_bucket or settings.s3_bucket,
|
||||
endpoint_url=settings.file_storage_s3_endpoint_url or settings.s3_endpoint_url,
|
||||
region_name=settings.file_storage_s3_region or settings.s3_region,
|
||||
access_key_id=settings.file_storage_s3_access_key_id or settings.s3_access_key_id,
|
||||
secret_access_key=settings.file_storage_s3_secret_access_key or settings.s3_secret_access_key,
|
||||
)
|
||||
raise StorageBackendError(f"Unsupported file storage backend: {settings.file_storage_backend}")
|
||||
"""Return the deployment-owned backend for Files-managed objects."""
|
||||
|
||||
return configured_storage_backend(settings)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"LocalFilesystemStorageBackend",
|
||||
"S3StorageBackend",
|
||||
"StorageBackend",
|
||||
"StorageBackendError",
|
||||
"StorageObjectInfo",
|
||||
"StorageObjectMissing",
|
||||
"StorageObjectPage",
|
||||
"get_storage_backend",
|
||||
]
|
||||
|
||||
@@ -13,12 +13,16 @@ from typing import Any, Iterator
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from govoplan_files.backend.db.models import FileAsset, FileBlob, FileShare, FileVersion
|
||||
from govoplan_files.backend.storage.backends import StorageBackend, StorageBackendError, get_storage_backend
|
||||
from govoplan_files.backend.storage.common import FileStorageError
|
||||
from govoplan_files.backend.storage.backends import StorageBackend, get_storage_backend
|
||||
from govoplan_files.backend.storage.files import current_versions_and_blobs, get_asset_for_user, list_assets_for_user, share_files
|
||||
from govoplan_files.backend.storage.access import ensure_owner_access
|
||||
from govoplan_files.backend.storage.paths import normalize_folder, normalize_logical_path, safe_storage_component
|
||||
from govoplan_files.backend.storage.provenance import source_provenance_from_metadata, source_revision_from_metadata
|
||||
from govoplan_files.backend.storage.share_state import effective_file_share_clause
|
||||
from govoplan_files.backend.storage.integrity import (
|
||||
ensure_blob_is_readable,
|
||||
read_verified_blob_bytes,
|
||||
)
|
||||
|
||||
|
||||
MANAGED_SOURCE_PREFIX = "managed:"
|
||||
@@ -165,7 +169,7 @@ def _campaign_linked_asset_ids(
|
||||
FileShare.file_asset_id.in_(chunk),
|
||||
FileShare.target_type == "campaign",
|
||||
FileShare.target_id == campaign_id,
|
||||
FileShare.revoked_at.is_(None),
|
||||
effective_file_share_clause(),
|
||||
)
|
||||
.all()
|
||||
)
|
||||
@@ -301,11 +305,9 @@ def _materialize_managed_asset(
|
||||
target = _safe_local_target(local_root, relative_path)
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
version, blob = version_blobs[asset.id]
|
||||
ensure_blob_is_readable(blob)
|
||||
if include_bytes:
|
||||
try:
|
||||
data = backend.get_bytes(blob.storage_key) if backend else b""
|
||||
except StorageBackendError as exc:
|
||||
raise FileStorageError(str(exc)) from exc
|
||||
data = read_verified_blob_bytes(blob, backend=backend) if backend else b""
|
||||
target.write_bytes(data)
|
||||
else:
|
||||
target.touch()
|
||||
|
||||
@@ -15,8 +15,8 @@ from defusedxml import ElementTree as SafeElementTree
|
||||
|
||||
from govoplan_core.security.outbound_http import (
|
||||
OutboundHttpError,
|
||||
validate_unpinned_sdk_host,
|
||||
validate_unpinned_sdk_http_url,
|
||||
validate_outbound_host,
|
||||
validate_outbound_http_url,
|
||||
)
|
||||
|
||||
from govoplan_files.backend.storage.http_client import ConnectorHttpError, request_connector_bytes
|
||||
@@ -27,6 +27,12 @@ from govoplan_files.backend.storage.connector_deployment import (
|
||||
validate_connector_tls_metadata,
|
||||
)
|
||||
from govoplan_files.backend.storage.connector_profiles import ConnectorProfile
|
||||
from govoplan_files.backend.storage.sdk_peer_pinning import (
|
||||
SdkPeerPinningError,
|
||||
create_pinned_s3_client,
|
||||
install_pinned_smb_transport,
|
||||
pinned_smb_connection_cache,
|
||||
)
|
||||
|
||||
|
||||
class ConnectorBrowseError(RuntimeError):
|
||||
@@ -246,6 +252,28 @@ def _browse_smb(profile: ConnectorProfile, *, path: str) -> list[ConnectorBrowse
|
||||
|
||||
def _browse_s3(profile: ConnectorProfile, *, path: str, library_id: str | None, continuation_token: str | None) -> list[ConnectorBrowseItem]:
|
||||
client = _s3_client(profile)
|
||||
try:
|
||||
return _browse_s3_with_client(
|
||||
client,
|
||||
profile=profile,
|
||||
path=path,
|
||||
library_id=library_id,
|
||||
continuation_token=continuation_token,
|
||||
)
|
||||
finally:
|
||||
close = getattr(client, "close", None)
|
||||
if callable(close):
|
||||
close()
|
||||
|
||||
|
||||
def _browse_s3_with_client(
|
||||
client: Any,
|
||||
*,
|
||||
profile: ConnectorProfile,
|
||||
path: str,
|
||||
library_id: str | None,
|
||||
continuation_token: str | None,
|
||||
) -> list[ConnectorBrowseItem]:
|
||||
bucket = _s3_bucket(profile, library_id)
|
||||
if not bucket:
|
||||
try:
|
||||
@@ -313,23 +341,22 @@ def _s3_client(profile: ConnectorProfile) -> Any:
|
||||
raise ConnectorBrowseError("Secret-ref S3 credentials need a runtime secret resolver before live browsing")
|
||||
if profile.endpoint_url:
|
||||
try:
|
||||
endpoint_url = validate_unpinned_sdk_http_url(
|
||||
endpoint_url = validate_outbound_http_url(
|
||||
profile.endpoint_url,
|
||||
label="S3 connector endpoint",
|
||||
)
|
||||
except OutboundHttpError as exc:
|
||||
raise ConnectorBrowseError(str(exc)) from exc
|
||||
else:
|
||||
raise ConnectorBrowseError(
|
||||
"S3 connector endpoint discovery uses an SDK transport that cannot guarantee connection-time DNS/IP "
|
||||
"pinning; live S3 access is disabled until that transport supports pinning"
|
||||
)
|
||||
endpoint_url = None
|
||||
try:
|
||||
boto3 = import_module("boto3")
|
||||
config_module = import_module("botocore.config")
|
||||
unsigned = import_module("botocore").UNSIGNED
|
||||
except ImportError as exc:
|
||||
raise ConnectorBrowseUnsupported("S3 connector browsing requires the optional boto3 dependency") from exc
|
||||
kwargs: dict[str, object] = {"endpoint_url": endpoint_url}
|
||||
kwargs: dict[str, object] = {}
|
||||
if endpoint_url:
|
||||
kwargs["endpoint_url"] = endpoint_url
|
||||
region = _metadata_string(profile, "region") or _metadata_string(profile, "aws_region")
|
||||
if region:
|
||||
kwargs["region_name"] = region
|
||||
@@ -346,10 +373,21 @@ def _s3_client(profile: ConnectorProfile) -> Any:
|
||||
if verify is not None:
|
||||
kwargs["verify"] = verify
|
||||
addressing_style = _s3_addressing_style(profile)
|
||||
config_values: dict[str, object] = {
|
||||
"proxies": {},
|
||||
"retries": {"mode": "standard", "max_attempts": 4},
|
||||
}
|
||||
if addressing_style:
|
||||
kwargs["config"] = config_module.Config(s3={"addressing_style": addressing_style})
|
||||
config_values["s3"] = {"addressing_style": addressing_style}
|
||||
if bool(access_key) != bool(secret_key):
|
||||
raise ConnectorBrowseError("S3 connectors require both an access key and a secret key")
|
||||
if not access_key:
|
||||
config_values["signature_version"] = unsigned
|
||||
kwargs["config"] = config_module.Config(**config_values)
|
||||
try:
|
||||
return boto3.client("s3", **kwargs)
|
||||
return create_pinned_s3_client(**kwargs)
|
||||
except SdkPeerPinningError as exc:
|
||||
raise ConnectorBrowseError(str(exc)) from exc
|
||||
except Exception as exc: # pragma: no cover - concrete exception types are dependency-version specific
|
||||
raise ConnectorBrowseError(f"S3 connector could not be initialized: {exc}") from exc
|
||||
|
||||
@@ -783,7 +821,7 @@ def _smb_location(profile: ConnectorProfile) -> _SmbLocation:
|
||||
raise ConnectorBrowseError("SMB connector endpoint_url must include a server")
|
||||
port = parsed.port or _int(profile.metadata.get("port")) or 445
|
||||
try:
|
||||
validate_unpinned_sdk_host(server, port=port, label="SMB connector endpoint")
|
||||
validate_outbound_host(server, port=port, label="SMB connector endpoint")
|
||||
except OutboundHttpError as exc:
|
||||
raise ConnectorBrowseError(str(exc)) from exc
|
||||
path_parts = [part for part in unquote(parsed.path or "").strip("/").split("/") if part]
|
||||
@@ -810,6 +848,7 @@ def _smb_unc_path(location: _SmbLocation, path: str) -> str:
|
||||
def _smb_client_kwargs(profile: ConnectorProfile, location: _SmbLocation) -> dict[str, object]:
|
||||
kwargs: dict[str, object] = {
|
||||
"port": location.port,
|
||||
"connection_cache": pinned_smb_connection_cache(),
|
||||
"require_signing": _metadata_bool(profile, "require_signing", default=True),
|
||||
"auth_protocol": _metadata_string(profile, "auth_protocol") or "ntlm",
|
||||
}
|
||||
@@ -845,9 +884,11 @@ def _profile_token(profile: ConnectorProfile) -> str | None:
|
||||
|
||||
def _smbclient_module() -> Any:
|
||||
try:
|
||||
return import_module("smbclient")
|
||||
return install_pinned_smb_transport(import_module("smbclient"))
|
||||
except ImportError as exc:
|
||||
raise ConnectorBrowseUnsupported("SMB connector browsing requires the optional smbprotocol dependency") from exc
|
||||
except SdkPeerPinningError as exc:
|
||||
raise ConnectorBrowseUnsupported(str(exc)) from exc
|
||||
|
||||
|
||||
def _smb_entry_stat(entry: object) -> object | None:
|
||||
|
||||
@@ -5,8 +5,17 @@ from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy import inspect
|
||||
|
||||
from govoplan_core.core.policy import normalize_policy_scope_type, policy_source_path
|
||||
from govoplan_core.security.credential_envelopes import (
|
||||
CredentialAccessContext,
|
||||
CredentialEnvelope,
|
||||
CredentialEnvelopeError,
|
||||
ResolvedCredentialEnvelope,
|
||||
list_credential_envelopes,
|
||||
resolve_credential_envelope,
|
||||
)
|
||||
from govoplan_core.security.secrets import decrypt_secret, encrypt_secret
|
||||
from govoplan_files.backend.db.models import FileConnectorCredential
|
||||
from govoplan_files.backend.storage.common import FileStorageError
|
||||
@@ -14,6 +23,8 @@ from govoplan_files.backend.storage.connector_deployment import reject_api_contr
|
||||
from govoplan_files.backend.storage.connector_policy import ConnectorPolicySource, connector_policy_sources_from_payload
|
||||
from govoplan_files.backend.storage.connector_profiles import supported_connector_providers
|
||||
|
||||
CORE_CREDENTIAL_ENVELOPE_PREFIX = "credential-envelope:"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ConnectorCredential:
|
||||
@@ -33,6 +44,7 @@ class ConnectorCredential:
|
||||
policy_sources: tuple[ConnectorPolicySource, ...] = ()
|
||||
metadata: Mapping[str, Any] | None = None
|
||||
source_kind: str = "database"
|
||||
has_secret: bool = False
|
||||
|
||||
@property
|
||||
def source_path(self) -> str:
|
||||
@@ -58,7 +70,7 @@ class ConnectorCredential:
|
||||
return True
|
||||
# Environment references are deliberately unavailable to API-managed
|
||||
# credential rows. Legacy rows remain visible but fail closed.
|
||||
return bool(self.secret_ref or self.password_value or self.token_value)
|
||||
return bool(self.has_secret or self.secret_ref or self.password_value or self.token_value)
|
||||
|
||||
def to_response(self) -> dict[str, Any]:
|
||||
return {
|
||||
@@ -85,7 +97,159 @@ def list_database_connector_credentials(
|
||||
tenant_id: str,
|
||||
include_disabled: bool = False,
|
||||
) -> list[ConnectorCredential]:
|
||||
return [connector_credential_from_row(row) for row in list_connector_credential_rows(session, tenant_id=tenant_id, include_disabled=include_disabled)]
|
||||
local = [
|
||||
connector_credential_from_row(row)
|
||||
for row in list_connector_credential_rows(
|
||||
session,
|
||||
tenant_id=tenant_id,
|
||||
include_disabled=include_disabled,
|
||||
)
|
||||
]
|
||||
return [
|
||||
*local,
|
||||
*list_reusable_connector_credentials(
|
||||
session,
|
||||
tenant_id=tenant_id,
|
||||
include_disabled=include_disabled,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def reusable_credential_reference(credential_id: str) -> str:
|
||||
return f"{CORE_CREDENTIAL_ENVELOPE_PREFIX}{credential_id}"
|
||||
|
||||
|
||||
def reusable_credential_id(credential_ref: str | None) -> str | None:
|
||||
if not credential_ref or not credential_ref.startswith(CORE_CREDENTIAL_ENVELOPE_PREFIX):
|
||||
return None
|
||||
value = credential_ref.removeprefix(CORE_CREDENTIAL_ENVELOPE_PREFIX).strip()
|
||||
return value or None
|
||||
|
||||
|
||||
def file_credential_context(
|
||||
*,
|
||||
tenant_id: str,
|
||||
profile_id: str | None = None,
|
||||
scope_type: str = "tenant",
|
||||
scope_id: str | None = None,
|
||||
administrative: bool = False,
|
||||
) -> CredentialAccessContext:
|
||||
return CredentialAccessContext(
|
||||
tenant_id=tenant_id,
|
||||
user_id=scope_id if scope_type == "user" else None,
|
||||
group_ids=frozenset({scope_id}) if scope_type == "group" and scope_id else frozenset(),
|
||||
target_scope_type=scope_type,
|
||||
target_scope_id=scope_id or tenant_id,
|
||||
module_id="files",
|
||||
server_ref=f"files:{profile_id}" if profile_id else None,
|
||||
administrative=administrative,
|
||||
)
|
||||
|
||||
|
||||
def list_reusable_connector_credentials(
|
||||
session: Session,
|
||||
*,
|
||||
tenant_id: str,
|
||||
include_disabled: bool = False,
|
||||
) -> list[ConnectorCredential]:
|
||||
if not inspect(session.get_bind()).has_table(CredentialEnvelope.__tablename__):
|
||||
return []
|
||||
context = file_credential_context(tenant_id=tenant_id, administrative=True)
|
||||
return [
|
||||
connector_credential_from_envelope(row)
|
||||
for row in list_credential_envelopes(
|
||||
session,
|
||||
context=context,
|
||||
include_inactive=include_disabled,
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def resolve_reusable_connector_credential(
|
||||
session: Session,
|
||||
*,
|
||||
tenant_id: str,
|
||||
credential_ref: str,
|
||||
profile_id: str,
|
||||
scope_type: str,
|
||||
scope_id: str | None,
|
||||
) -> ConnectorCredential:
|
||||
credential_id = reusable_credential_id(credential_ref)
|
||||
if credential_id is None:
|
||||
raise FileStorageError("Reusable credential reference is invalid")
|
||||
try:
|
||||
resolved = resolve_credential_envelope(
|
||||
session,
|
||||
credential_id=credential_id,
|
||||
context=file_credential_context(
|
||||
tenant_id=tenant_id,
|
||||
profile_id=profile_id,
|
||||
scope_type=scope_type,
|
||||
scope_id=scope_id,
|
||||
),
|
||||
)
|
||||
except CredentialEnvelopeError as exc:
|
||||
raise FileStorageError("Reusable credential is unavailable to this file connection") from exc
|
||||
return connector_credential_from_resolved_envelope(
|
||||
resolved,
|
||||
scope_type=scope_type,
|
||||
scope_id=scope_id,
|
||||
)
|
||||
|
||||
|
||||
def connector_credential_from_envelope(row: CredentialEnvelope) -> ConnectorCredential:
|
||||
return ConnectorCredential(
|
||||
id=reusable_credential_reference(row.id),
|
||||
label=row.name,
|
||||
scope_type=row.scope_type,
|
||||
scope_id=row.scope_id,
|
||||
enabled=row.is_active,
|
||||
credential_mode=_envelope_credential_mode(row.credential_kind, row.secret_keys),
|
||||
username=_clean_public_value(row.public_data, "username"),
|
||||
source_kind="credential_envelope",
|
||||
has_secret=bool(row.secret_data_encrypted),
|
||||
metadata={
|
||||
"credential_envelope_id": row.id,
|
||||
"credential_kind": row.credential_kind,
|
||||
"allowed_modules": list(row.allowed_modules or []),
|
||||
"allowed_server_refs": list(row.allowed_server_refs or []),
|
||||
"inherit_to_lower_scopes": bool(row.inherit_to_lower_scopes),
|
||||
"revision": row.revision,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def connector_credential_from_resolved_envelope(
|
||||
row: ResolvedCredentialEnvelope,
|
||||
*,
|
||||
scope_type: str,
|
||||
scope_id: str | None,
|
||||
) -> ConnectorCredential:
|
||||
return ConnectorCredential(
|
||||
id=reusable_credential_reference(row.id),
|
||||
label=row.name,
|
||||
scope_type=scope_type,
|
||||
scope_id=scope_id,
|
||||
enabled=True,
|
||||
credential_mode=_envelope_credential_mode(row.credential_kind, row.secret_data),
|
||||
username=_clean_public_value(row.public_data, "username"),
|
||||
password_value=_first_secret(row.secret_data, "password", "secret"),
|
||||
token_value=_first_secret(
|
||||
row.secret_data,
|
||||
"access_token",
|
||||
"bearer_token",
|
||||
"token",
|
||||
"api_key",
|
||||
),
|
||||
source_kind="credential_envelope",
|
||||
has_secret=bool(row.secret_data),
|
||||
metadata={
|
||||
"credential_envelope_id": row.id,
|
||||
"credential_kind": row.credential_kind,
|
||||
"inherit_to_lower_scopes": True,
|
||||
"revision": row.revision,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def list_connector_credential_rows(
|
||||
@@ -348,6 +512,27 @@ def _policy_source_response(source: ConnectorPolicySource) -> dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
def _envelope_credential_mode(kind: str, secret_values: Mapping[str, Any] | list[str]) -> str:
|
||||
keys = {str(key) for key in secret_values}
|
||||
if kind in {"token", "oauth2", "api_key"} or keys.intersection(
|
||||
{"access_token", "bearer_token", "token", "api_key"}
|
||||
):
|
||||
return "token"
|
||||
return "basic"
|
||||
|
||||
|
||||
def _clean_public_value(values: Mapping[str, Any] | None, key: str) -> str | None:
|
||||
return _clean((values or {}).get(key))
|
||||
|
||||
|
||||
def _first_secret(values: Mapping[str, Any], *keys: str) -> str | None:
|
||||
for key in keys:
|
||||
value = _clean(values.get(key))
|
||||
if value is not None:
|
||||
return value
|
||||
return None
|
||||
|
||||
|
||||
def _clean(value: object) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
|
||||
@@ -314,15 +314,20 @@ def _read_s3_file(profile: ConnectorProfile, *, library_id: str, path: str, max_
|
||||
if not key:
|
||||
raise ConnectorImportError("S3 import requires an object key")
|
||||
client = _s3_import_client(profile)
|
||||
detail = _s3_object_detail(client, bucket=bucket, key=key, max_bytes=max_bytes)
|
||||
version_id = _clean(detail.get("VersionId"))
|
||||
response, data = _download_s3_object(
|
||||
client,
|
||||
bucket=bucket,
|
||||
key=key,
|
||||
version_id=version_id,
|
||||
max_bytes=max_bytes,
|
||||
)
|
||||
try:
|
||||
detail = _s3_object_detail(client, bucket=bucket, key=key, max_bytes=max_bytes)
|
||||
version_id = _clean(detail.get("VersionId"))
|
||||
response, data = _download_s3_object(
|
||||
client,
|
||||
bucket=bucket,
|
||||
key=key,
|
||||
version_id=version_id,
|
||||
max_bytes=max_bytes,
|
||||
)
|
||||
finally:
|
||||
close = getattr(client, "close", None)
|
||||
if callable(close):
|
||||
close()
|
||||
content_type = _clean(response.get("ContentType") if isinstance(response, dict) else None) or _clean(detail.get("ContentType")) or mimetypes.guess_type(key)[0]
|
||||
etag = _clean(response.get("ETag") if isinstance(response, dict) else None) or _clean(detail.get("ETag"))
|
||||
filename = filename_from_path(key)
|
||||
|
||||
@@ -246,11 +246,11 @@ def _applied_fields(policy: Mapping[str, Any]) -> tuple[str, ...]:
|
||||
|
||||
def _matches_field(request: ConnectorAccessRequest, field: str, patterns: list[str]) -> bool:
|
||||
if field == "connectors":
|
||||
return _matches_exact(request.connector_id, patterns)
|
||||
return _matches_reference(request.connector_id, patterns)
|
||||
if field == "credentials":
|
||||
return _matches_exact(request.credential_id, patterns)
|
||||
return _matches_reference(request.credential_id, patterns)
|
||||
if field == "providers":
|
||||
return _matches_exact(request.provider, patterns)
|
||||
return _matches_reference(request.provider, patterns)
|
||||
if field == "external_ids":
|
||||
return _matches_glob(request.external_id, patterns)
|
||||
if field == "external_paths":
|
||||
@@ -260,11 +260,11 @@ def _matches_field(request: ConnectorAccessRequest, field: str, patterns: list[s
|
||||
return False
|
||||
|
||||
|
||||
def _matches_exact(value: str | None, patterns: list[str]) -> bool:
|
||||
def _matches_reference(value: str | None, patterns: list[str]) -> bool:
|
||||
if value is None:
|
||||
return False
|
||||
clean = value.casefold()
|
||||
return any(pattern == "*" or clean == pattern.casefold() for pattern in patterns)
|
||||
return any(fnmatchcase(clean, pattern.casefold()) for pattern in patterns)
|
||||
|
||||
|
||||
def _matches_glob(value: str | None, patterns: list[str]) -> bool:
|
||||
|
||||
@@ -10,7 +10,13 @@ from govoplan_core.security.secrets import decrypt_secret, encrypt_secret
|
||||
from govoplan_files.backend.db.models import FileConnectorCredential, FileConnectorProfile
|
||||
from govoplan_files.backend.storage.common import FileStorageError
|
||||
from govoplan_files.backend.storage.connector_deployment import reject_api_controlled_deployment_references
|
||||
from govoplan_files.backend.storage.connector_credential_store import connector_credential_from_row, credential_rows_by_id
|
||||
from govoplan_files.backend.storage.connector_credential_store import (
|
||||
ConnectorCredential,
|
||||
connector_credential_from_row,
|
||||
credential_rows_by_id,
|
||||
resolve_reusable_connector_credential,
|
||||
reusable_credential_id,
|
||||
)
|
||||
from govoplan_files.backend.storage.connector_policy import connector_policy_sources_from_payload
|
||||
from govoplan_files.backend.storage.connector_profiles import ConnectorProfile, supported_connector_providers
|
||||
|
||||
@@ -50,10 +56,28 @@ def select_database_connector_profiles(
|
||||
profile_ids = {row.id for row in rows}
|
||||
if row_visible is not None:
|
||||
rows = [row for row in rows if row_visible(row)]
|
||||
credential_ids = {_clean(row.credential_profile_id) for row in rows if _clean(row.credential_profile_id)}
|
||||
credential_ids = {
|
||||
_clean(row.credential_profile_id)
|
||||
for row in rows
|
||||
if _clean(row.credential_profile_id) and not reusable_credential_id(row.credential_profile_id)
|
||||
}
|
||||
credentials = credential_rows_by_id(session, tenant_id=tenant_id, credential_ids={item for item in credential_ids if item}, include_disabled=include_disabled)
|
||||
return (
|
||||
[connector_profile_from_row(row, credential_row=credentials.get(row.credential_profile_id or "")) for row in rows],
|
||||
[
|
||||
connector_profile_from_row(
|
||||
row,
|
||||
credential_row=(
|
||||
_resolve_profile_reusable_credential(
|
||||
session,
|
||||
tenant_id=tenant_id,
|
||||
row=row,
|
||||
)
|
||||
if reusable_credential_id(row.credential_profile_id)
|
||||
else credentials.get(row.credential_profile_id or "")
|
||||
),
|
||||
)
|
||||
for row in rows
|
||||
],
|
||||
profile_ids,
|
||||
)
|
||||
|
||||
@@ -73,7 +97,10 @@ def list_connector_profile_rows(
|
||||
return query.order_by(FileConnectorProfile.scope_type.asc(), FileConnectorProfile.label.asc()).all()
|
||||
|
||||
|
||||
def connector_profile_from_row(row: FileConnectorProfile, credential_row: FileConnectorCredential | None = None) -> ConnectorProfile:
|
||||
def connector_profile_from_row(
|
||||
row: FileConnectorProfile,
|
||||
credential_row: FileConnectorCredential | ConnectorCredential | None = None,
|
||||
) -> ConnectorProfile:
|
||||
policy_sources = []
|
||||
if row.policy:
|
||||
policy_sources = connector_policy_sources_from_payload({
|
||||
@@ -82,9 +109,18 @@ def connector_profile_from_row(row: FileConnectorProfile, credential_row: FileCo
|
||||
"label": row.label,
|
||||
"policy": row.policy,
|
||||
})
|
||||
credential = connector_credential_from_row(credential_row) if credential_row is not None else None
|
||||
credential = (
|
||||
credential_row
|
||||
if isinstance(credential_row, ConnectorCredential)
|
||||
else connector_credential_from_row(credential_row)
|
||||
if credential_row is not None
|
||||
else None
|
||||
)
|
||||
if credential:
|
||||
policy_sources.extend(credential.policy_sources)
|
||||
metadata = dict(row.metadata_ or {})
|
||||
if reusable_credential_id(row.credential_profile_id) and credential is None:
|
||||
metadata["credential_unavailable"] = True
|
||||
return ConnectorProfile(
|
||||
id=row.id,
|
||||
label=row.label,
|
||||
@@ -105,11 +141,30 @@ def connector_profile_from_row(row: FileConnectorProfile, credential_row: FileCo
|
||||
token_value=credential.token_value if credential else decrypt_secret(row.token_encrypted),
|
||||
capabilities=tuple(_string_list(row.capabilities)),
|
||||
policy_sources=tuple(policy_sources),
|
||||
metadata=dict(row.metadata_ or {}),
|
||||
metadata=metadata,
|
||||
source_kind="database",
|
||||
)
|
||||
|
||||
|
||||
def _resolve_profile_reusable_credential(
|
||||
session: Session,
|
||||
*,
|
||||
tenant_id: str,
|
||||
row: FileConnectorProfile,
|
||||
) -> ConnectorCredential | None:
|
||||
try:
|
||||
return resolve_reusable_connector_credential(
|
||||
session,
|
||||
tenant_id=tenant_id,
|
||||
credential_ref=row.credential_profile_id or "",
|
||||
profile_id=row.id,
|
||||
scope_type=row.scope_type,
|
||||
scope_id=row.scope_id,
|
||||
)
|
||||
except FileStorageError:
|
||||
return None
|
||||
|
||||
|
||||
def get_connector_profile_row(
|
||||
session: Session,
|
||||
*,
|
||||
|
||||
@@ -32,6 +32,7 @@ class ConnectorProviderDescriptor:
|
||||
"installed": self.installed,
|
||||
"browse_supported": self.browse_supported,
|
||||
"import_supported": self.import_supported,
|
||||
"write_supported": self.provider == "s3",
|
||||
"optional_dependency": self.optional_dependency,
|
||||
"permission_model": self.permission_model,
|
||||
"sync_strategy": self.sync_strategy,
|
||||
@@ -110,7 +111,7 @@ def connector_provider_descriptors() -> tuple[ConnectorProviderDescriptor, ...]:
|
||||
conflict_strategy="Managed import/sync uses existing files conflict_strategy handling after download.",
|
||||
preview_strategy="Previews are generated from the frozen managed file after import or sync, not directly from the share.",
|
||||
audit_events=("files.connector.imported", "files.connector.synced", "files.connector.accessed"),
|
||||
notes="Browse/import logic is implemented, but live smbprotocol access fails closed until the SDK supports connection-time DNS/IP pinning for initial connections and DFS referrals.",
|
||||
notes="Initial sessions, reconnects, aliases, and DFS referral targets use the Files-owned pinned smbprotocol transport and the deployment-wide private-network policy.",
|
||||
),
|
||||
ConnectorProviderDescriptor(
|
||||
provider="s3",
|
||||
@@ -125,7 +126,7 @@ def connector_provider_descriptors() -> tuple[ConnectorProviderDescriptor, ...]:
|
||||
conflict_strategy="Managed import/sync uses existing files conflict_strategy handling after object download.",
|
||||
preview_strategy="Previews are generated from the frozen managed file after import or sync, not directly from the bucket.",
|
||||
audit_events=("files.connector.imported", "files.connector.synced", "files.connector.accessed"),
|
||||
notes="Browse/import logic is implemented, but live boto3 access fails closed until its HTTP transport supports connection-time DNS/IP pinning and redirect revalidation.",
|
||||
notes="Every botocore connection pool uses the Files-owned pinned transport, including retries, redirects, endpoint discovery, and virtual-host aliases; outbound proxies and ambient credential discovery are disabled.",
|
||||
),
|
||||
ConnectorProviderDescriptor(
|
||||
provider="sharepoint",
|
||||
|
||||
@@ -14,6 +14,7 @@ from govoplan_files.backend.storage.connector_profiles import ConnectorProfile
|
||||
|
||||
|
||||
SYNC_MODES = {"manual"}
|
||||
WRITABLE_PROVIDERS = {"s3"}
|
||||
|
||||
|
||||
def connector_space_owner_id(space: FileConnectorSpace) -> str:
|
||||
@@ -40,6 +41,7 @@ def create_connector_space(
|
||||
ensure_owner_access(session, tenant_id=tenant_id, owner_type=owner_type, owner_id=owner_id, user_id=user_id, is_admin=is_admin)
|
||||
label = _normalize_label(label)
|
||||
sync_mode = _normalize_sync_mode(sync_mode)
|
||||
validate_connector_space_write_mode(profile, read_only=read_only)
|
||||
remote_path = normalize_connector_browse_path(remote_path)
|
||||
library_id = _clean_optional(library_id)
|
||||
|
||||
@@ -145,6 +147,7 @@ def update_connector_space(
|
||||
library_id: str | None = None,
|
||||
remote_path: str | None = None,
|
||||
sync_mode: str | None = None,
|
||||
read_only: bool | None = None,
|
||||
is_active: bool | None = None,
|
||||
metadata: Mapping[str, Any] | None = None,
|
||||
is_admin: bool = False,
|
||||
@@ -178,6 +181,8 @@ def update_connector_space(
|
||||
space.remote_path = normalize_connector_browse_path(remote_path)
|
||||
if sync_mode is not None:
|
||||
space.sync_mode = _normalize_sync_mode(sync_mode)
|
||||
if read_only is not None:
|
||||
space.read_only = bool(read_only)
|
||||
if is_active is not None:
|
||||
space.is_active = bool(is_active)
|
||||
if metadata is not None:
|
||||
@@ -187,6 +192,17 @@ def update_connector_space(
|
||||
return space
|
||||
|
||||
|
||||
def validate_connector_space_write_mode(
|
||||
profile: ConnectorProfile, *, read_only: bool
|
||||
) -> None:
|
||||
if read_only:
|
||||
return
|
||||
if profile.provider not in WRITABLE_PROVIDERS or "write" not in profile.capabilities:
|
||||
raise FileStorageError(
|
||||
"Two-way mode requires an S3 connection with the write capability enabled"
|
||||
)
|
||||
|
||||
|
||||
def soft_delete_connector_space(
|
||||
session: Session,
|
||||
space: FileConnectorSpace,
|
||||
|
||||
@@ -136,11 +136,6 @@ def connector_profile_usable_for_import(profile: ConnectorProfile) -> bool:
|
||||
and descriptor.import_supported
|
||||
):
|
||||
return False
|
||||
# These SDK paths intentionally fail closed until every connection peer and
|
||||
# SDK-managed redirect/referral can be pinned and revalidated.
|
||||
if profile.provider in {"s3", "smb"}:
|
||||
return False
|
||||
|
||||
# Prove that the initial root browse performed by the current Files UI is
|
||||
# policy-allowed. A later selected remote path/item is checked again.
|
||||
return connector_policy_decision(
|
||||
|
||||
@@ -0,0 +1,438 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
import hashlib
|
||||
from typing import Any, Mapping
|
||||
|
||||
from govoplan_core.core.recovery import (
|
||||
RecoveryGuaranteeError,
|
||||
RecoveryMode,
|
||||
RecoveryPlan,
|
||||
RecoveryStatus,
|
||||
)
|
||||
from govoplan_core.core.recovery_runtime import (
|
||||
RecoveryOperationBusy,
|
||||
RecoveryOperationStateConflict,
|
||||
begin_durable_recovery_operation,
|
||||
)
|
||||
from govoplan_core.core.runtime_coordination import process_runtime_identity
|
||||
from govoplan_core.db.session import get_database
|
||||
from govoplan_files.backend.storage.common import FileStorageError
|
||||
from govoplan_files.backend.storage.connector_browse import (
|
||||
ConnectorBrowseError,
|
||||
ConnectorBrowseUnsupported,
|
||||
_clean,
|
||||
_s3_bucket,
|
||||
_s3_client,
|
||||
_s3_object_key,
|
||||
normalize_connector_browse_path,
|
||||
)
|
||||
from govoplan_files.backend.storage.connector_profiles import ConnectorProfile
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ConnectorWriteResult:
|
||||
recovery_operation_id: str
|
||||
status: str
|
||||
replayed: bool
|
||||
provider: str
|
||||
remote_path: str
|
||||
revision: str | None
|
||||
checksum_sha256: str
|
||||
size_bytes: int
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _HeadProbe:
|
||||
observed: Mapping[str, Any] | None
|
||||
verified: bool
|
||||
exception_type: str | None = None
|
||||
|
||||
|
||||
def write_connector_file(
|
||||
profile: ConnectorProfile,
|
||||
*,
|
||||
tenant_id: str,
|
||||
library_id: str | None,
|
||||
remote_path: str,
|
||||
data: bytes,
|
||||
content_type: str | None,
|
||||
expected_revision: str | None,
|
||||
idempotency_key: str,
|
||||
) -> ConnectorWriteResult:
|
||||
if profile.provider != "s3" or "write" not in profile.capabilities:
|
||||
raise FileStorageError(
|
||||
"Remote writes require an S3 connection with the write capability enabled"
|
||||
)
|
||||
try:
|
||||
normalized_path = normalize_connector_browse_path(remote_path)
|
||||
except ConnectorBrowseError as exc:
|
||||
raise FileStorageError(str(exc)) from exc
|
||||
if not normalized_path:
|
||||
raise FileStorageError("Remote write requires an object path")
|
||||
try:
|
||||
bucket = _s3_bucket(profile, library_id)
|
||||
key = _s3_object_key(profile, normalized_path)
|
||||
except ConnectorBrowseError as exc:
|
||||
raise FileStorageError(str(exc)) from exc
|
||||
if not bucket or not key:
|
||||
raise FileStorageError("S3 remote write requires a bucket and object path")
|
||||
checksum = hashlib.sha256(data).hexdigest()
|
||||
target_digest = hashlib.sha256(f"{bucket}:{key}".encode("utf-8")).hexdigest()
|
||||
request = {
|
||||
"tenant_id": tenant_id,
|
||||
"connector_profile_id": profile.id,
|
||||
"provider": profile.provider,
|
||||
"remote_target_sha256": target_digest,
|
||||
"content_sha256": checksum,
|
||||
"size_bytes": len(data),
|
||||
"expected_revision": expected_revision,
|
||||
}
|
||||
try:
|
||||
started = begin_durable_recovery_operation(
|
||||
get_database().SessionLocal,
|
||||
identity=process_runtime_identity(),
|
||||
module_id="files",
|
||||
operation_type="connector-s3-write",
|
||||
idempotency_key=f"files-connector-write:{idempotency_key}",
|
||||
request=request,
|
||||
recovery_plan=RecoveryPlan(
|
||||
mode=RecoveryMode.FORWARD_RECOVERY,
|
||||
preconditions=(
|
||||
"the connector space is explicitly configured for two-way writes",
|
||||
"the connection and inherited policy allow the exact remote path",
|
||||
"a conditional create or expected remote revision prevents blind overwrite",
|
||||
),
|
||||
forward_recovery_steps=(
|
||||
"inspect provider metadata for the request and content digests",
|
||||
"resolve success only for an exact digest match or retry after verified absence",
|
||||
),
|
||||
verification_steps=(
|
||||
"read S3 object metadata after the write",
|
||||
"match the recorded request and content digests without downloading content",
|
||||
),
|
||||
),
|
||||
precondition_evidence={
|
||||
"remote_target_sha256": target_digest,
|
||||
"content_sha256": checksum,
|
||||
"size_bytes": len(data),
|
||||
"conditional_write": True,
|
||||
},
|
||||
lease_resource_key=f"files:connector:{tenant_id}:{profile.id}:{target_digest[:40]}",
|
||||
lease_ttl_seconds=15 * 60,
|
||||
resource_type="file_connector_object",
|
||||
resource_id=target_digest,
|
||||
metadata={
|
||||
"resources": ["postgresql", "s3-connector"],
|
||||
"provider": "s3",
|
||||
"connector_profile_id": profile.id,
|
||||
},
|
||||
block_unresolved_resource=True,
|
||||
)
|
||||
except (RecoveryOperationBusy, RecoveryOperationStateConflict) as exc:
|
||||
raise FileStorageError(
|
||||
"This remote object is owned by another write or unresolved recovery operation"
|
||||
) from exc
|
||||
except (RecoveryGuaranteeError, RuntimeError, ValueError) as exc:
|
||||
raise FileStorageError(
|
||||
"The Files recovery ledger is unavailable; the remote object was not changed"
|
||||
) from exc
|
||||
if started.replayed or started.operation is None:
|
||||
return ConnectorWriteResult(
|
||||
recovery_operation_id=started.operation_id,
|
||||
status=started.status,
|
||||
replayed=True,
|
||||
provider="s3",
|
||||
remote_path=normalized_path,
|
||||
revision=None,
|
||||
checksum_sha256=checksum,
|
||||
size_bytes=len(data),
|
||||
)
|
||||
operation = started.operation
|
||||
try:
|
||||
client = _s3_client(profile)
|
||||
except (ConnectorBrowseError, ConnectorBrowseUnsupported) as exc:
|
||||
operation.reject(
|
||||
summary="The S3 client was unavailable before the remote effect",
|
||||
evidence=_verified(
|
||||
{"external_effect_started": False}, exception_type=type(exc).__name__
|
||||
),
|
||||
)
|
||||
raise FileStorageError(str(exc)) from exc
|
||||
try:
|
||||
try:
|
||||
before = _head_object(client, bucket=bucket, key=key)
|
||||
except Exception as exc:
|
||||
operation.reject(
|
||||
summary="Remote preconditions could not be inspected before the effect",
|
||||
evidence=_verified(
|
||||
{"external_effect_started": False},
|
||||
exception_type=type(exc).__name__,
|
||||
),
|
||||
)
|
||||
raise FileStorageError("S3 remote revision lookup failed") from exc
|
||||
expected = _normalize_revision(expected_revision)
|
||||
if before is not None and expected is None:
|
||||
operation.reject(
|
||||
summary="A blind remote overwrite was rejected",
|
||||
evidence=_verified(
|
||||
{"remote_object_exists": True, "expected_revision_supplied": False}
|
||||
),
|
||||
)
|
||||
raise FileStorageError(
|
||||
"The remote object already exists; reload it and supply its expected revision"
|
||||
)
|
||||
if expected is not None and (
|
||||
before is None or expected not in _observed_revisions(before)
|
||||
):
|
||||
operation.reject(
|
||||
summary="The expected remote revision did not match",
|
||||
evidence=_verified(
|
||||
{
|
||||
"remote_object_exists": before is not None,
|
||||
"expected_revision_matches": False,
|
||||
}
|
||||
),
|
||||
)
|
||||
raise FileStorageError("The remote object changed; reload before writing")
|
||||
params: dict[str, object] = {
|
||||
"Bucket": bucket,
|
||||
"Key": key,
|
||||
"Body": data,
|
||||
"Metadata": {
|
||||
"govoplan-sha256": checksum,
|
||||
"govoplan-request-id": started.operation_id,
|
||||
},
|
||||
}
|
||||
if content_type:
|
||||
params["ContentType"] = content_type
|
||||
if before is None:
|
||||
params["IfNoneMatch"] = "*"
|
||||
else:
|
||||
params["IfMatch"] = str(before.get("ETag") or expected_revision or "")
|
||||
try:
|
||||
client.put_object(**params)
|
||||
except Exception as exc:
|
||||
probe = _probe_head_object(client, bucket=bucket, key=key)
|
||||
if probe.verified and _matches_effect(
|
||||
probe.observed,
|
||||
checksum=checksum,
|
||||
operation_id=started.operation_id,
|
||||
):
|
||||
operation.succeed(
|
||||
evidence=_success_evidence(
|
||||
probe.observed,
|
||||
checksum=checksum,
|
||||
operation_id=started.operation_id,
|
||||
)
|
||||
)
|
||||
return _result(
|
||||
started.operation_id,
|
||||
normalized_path,
|
||||
checksum,
|
||||
len(data),
|
||||
probe.observed,
|
||||
)
|
||||
operation.unresolved(
|
||||
status=RecoveryStatus.OUTCOME_UNKNOWN,
|
||||
summary="The S3 write outcome requires reconciliation",
|
||||
evidence={
|
||||
"remote_target_sha256": target_digest,
|
||||
"observed": _public_observation(probe),
|
||||
"exception_type": type(exc).__name__,
|
||||
},
|
||||
failure_summary="Inspect the remote object metadata before retrying this path",
|
||||
)
|
||||
return ConnectorWriteResult(
|
||||
recovery_operation_id=started.operation_id,
|
||||
status=RecoveryStatus.OUTCOME_UNKNOWN.value,
|
||||
replayed=False,
|
||||
provider="s3",
|
||||
remote_path=normalized_path,
|
||||
revision=_revision(probe.observed),
|
||||
checksum_sha256=checksum,
|
||||
size_bytes=len(data),
|
||||
)
|
||||
probe = _probe_head_object(client, bucket=bucket, key=key)
|
||||
if not probe.verified:
|
||||
operation.unresolved(
|
||||
status=RecoveryStatus.OUTCOME_UNKNOWN,
|
||||
summary="The S3 write returned but provider evidence could not be queried",
|
||||
evidence={
|
||||
"remote_target_sha256": target_digest,
|
||||
"observed": _public_observation(probe),
|
||||
},
|
||||
failure_summary="Inspect the remote object metadata before retrying this path",
|
||||
)
|
||||
return ConnectorWriteResult(
|
||||
recovery_operation_id=started.operation_id,
|
||||
status=RecoveryStatus.OUTCOME_UNKNOWN.value,
|
||||
replayed=False,
|
||||
provider="s3",
|
||||
remote_path=normalized_path,
|
||||
revision=None,
|
||||
checksum_sha256=checksum,
|
||||
size_bytes=len(data),
|
||||
)
|
||||
observed = probe.observed
|
||||
if not _matches_effect(
|
||||
observed, checksum=checksum, operation_id=started.operation_id
|
||||
):
|
||||
operation.unresolved(
|
||||
status=RecoveryStatus.RECOVERY_REQUIRED,
|
||||
summary="The S3 write returned but exact provider evidence did not match",
|
||||
evidence={
|
||||
"remote_target_sha256": target_digest,
|
||||
"observed": _public_observation(probe),
|
||||
},
|
||||
failure_summary="Reconcile the remote object before another write",
|
||||
)
|
||||
return ConnectorWriteResult(
|
||||
recovery_operation_id=started.operation_id,
|
||||
status=RecoveryStatus.RECOVERY_REQUIRED.value,
|
||||
replayed=False,
|
||||
provider="s3",
|
||||
remote_path=normalized_path,
|
||||
revision=_revision(observed),
|
||||
checksum_sha256=checksum,
|
||||
size_bytes=len(data),
|
||||
)
|
||||
operation.succeed(
|
||||
evidence=_success_evidence(
|
||||
observed, checksum=checksum, operation_id=started.operation_id
|
||||
)
|
||||
)
|
||||
return _result(
|
||||
started.operation_id,
|
||||
normalized_path,
|
||||
checksum,
|
||||
len(data),
|
||||
observed,
|
||||
)
|
||||
finally:
|
||||
close = getattr(client, "close", None)
|
||||
if callable(close):
|
||||
close()
|
||||
|
||||
|
||||
def _head_object(client: Any, *, bucket: str, key: str) -> Mapping[str, Any] | None:
|
||||
try:
|
||||
response = client.head_object(Bucket=bucket, Key=key)
|
||||
except Exception as exc:
|
||||
if _is_not_found(exc):
|
||||
return None
|
||||
raise
|
||||
if not isinstance(response, Mapping):
|
||||
raise FileStorageError("S3 connector returned invalid object metadata")
|
||||
return response
|
||||
|
||||
|
||||
def _probe_head_object(client: Any, *, bucket: str, key: str) -> _HeadProbe:
|
||||
try:
|
||||
return _HeadProbe(
|
||||
observed=_head_object(client, bucket=bucket, key=key), verified=True
|
||||
)
|
||||
except Exception as exc:
|
||||
return _HeadProbe(
|
||||
observed=None,
|
||||
verified=False,
|
||||
exception_type=type(exc).__name__,
|
||||
)
|
||||
|
||||
|
||||
def _is_not_found(exc: Exception) -> bool:
|
||||
response = getattr(exc, "response", None)
|
||||
if not isinstance(response, Mapping):
|
||||
return False
|
||||
error = response.get("Error")
|
||||
code = error.get("Code") if isinstance(error, Mapping) else None
|
||||
status = response.get("ResponseMetadata")
|
||||
http_status = status.get("HTTPStatusCode") if isinstance(status, Mapping) else None
|
||||
return str(code).casefold() in {"404", "nosuchkey", "notfound"} or http_status == 404
|
||||
|
||||
|
||||
def _metadata(observed: Mapping[str, Any] | None) -> Mapping[str, Any]:
|
||||
value = observed.get("Metadata") if observed else None
|
||||
return value if isinstance(value, Mapping) else {}
|
||||
|
||||
|
||||
def _matches_effect(
|
||||
observed: Mapping[str, Any] | None, *, checksum: str, operation_id: str
|
||||
) -> bool:
|
||||
metadata = _metadata(observed)
|
||||
return bool(
|
||||
observed is not None
|
||||
and _clean(metadata.get("govoplan-sha256")) == checksum
|
||||
and _clean(metadata.get("govoplan-request-id")) == operation_id
|
||||
)
|
||||
|
||||
|
||||
def _revision(observed: Mapping[str, Any] | None) -> str | None:
|
||||
if observed is None:
|
||||
return None
|
||||
return _clean(observed.get("VersionId") or observed.get("ETag"))
|
||||
|
||||
|
||||
def _normalize_revision(value: str | None) -> str | None:
|
||||
cleaned = _clean(value)
|
||||
return cleaned.strip('"') if cleaned else None
|
||||
|
||||
|
||||
def _observed_revisions(observed: Mapping[str, Any]) -> set[str]:
|
||||
return {
|
||||
normalized
|
||||
for value in (observed.get("VersionId"), observed.get("ETag"))
|
||||
if (normalized := _normalize_revision(_clean(value))) is not None
|
||||
}
|
||||
|
||||
|
||||
def _success_evidence(
|
||||
observed: Mapping[str, Any] | None, *, checksum: str, operation_id: str
|
||||
) -> dict[str, object]:
|
||||
return _verified(
|
||||
{
|
||||
"remote_object_present": observed is not None,
|
||||
"content_digest_matches": _clean(_metadata(observed).get("govoplan-sha256")) == checksum,
|
||||
"request_marker_matches": _clean(_metadata(observed).get("govoplan-request-id")) == operation_id,
|
||||
},
|
||||
revision=_revision(observed),
|
||||
content_sha256=checksum,
|
||||
)
|
||||
|
||||
|
||||
def _public_observation(probe: _HeadProbe) -> dict[str, object]:
|
||||
observed = probe.observed
|
||||
return {
|
||||
"probe_verified": probe.verified,
|
||||
"probe_exception_type": probe.exception_type,
|
||||
"present": observed is not None,
|
||||
"revision": _revision(observed),
|
||||
"has_content_digest": bool(_clean(_metadata(observed).get("govoplan-sha256"))),
|
||||
"has_request_marker": bool(_clean(_metadata(observed).get("govoplan-request-id"))),
|
||||
}
|
||||
|
||||
|
||||
def _verified(checks: dict[str, object], **details: object) -> dict[str, object]:
|
||||
return {"verified": True, "checks": checks, **details}
|
||||
|
||||
|
||||
def _result(
|
||||
operation_id: str,
|
||||
remote_path: str,
|
||||
checksum: str,
|
||||
size_bytes: int,
|
||||
observed: Mapping[str, Any] | None,
|
||||
) -> ConnectorWriteResult:
|
||||
return ConnectorWriteResult(
|
||||
recovery_operation_id=operation_id,
|
||||
status=RecoveryStatus.SUCCEEDED.value,
|
||||
replayed=False,
|
||||
provider="s3",
|
||||
remote_path=remote_path,
|
||||
revision=_revision(observed),
|
||||
checksum_sha256=checksum,
|
||||
size_bytes=size_bytes,
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["ConnectorWriteResult", "write_connector_file"]
|
||||
@@ -0,0 +1,93 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from govoplan_core.core.encryption import (
|
||||
ContentProtectionRequest,
|
||||
ContentUnprotectionRequest,
|
||||
ProtectedContent,
|
||||
encryption_content_cipher,
|
||||
)
|
||||
from govoplan_files.backend.runtime import get_registry
|
||||
from govoplan_files.backend.storage.common import FileStorageError
|
||||
|
||||
|
||||
FILES_PROTECTION_PROFILE = "files-server-envelope-v1"
|
||||
|
||||
|
||||
def protect_blob_content(
|
||||
session: object,
|
||||
*,
|
||||
tenant_id: str,
|
||||
blob_id: str,
|
||||
vault_id: str,
|
||||
ciphertext_ref: str,
|
||||
plaintext: bytes,
|
||||
actor_id: str,
|
||||
content_type: str | None,
|
||||
) -> ProtectedContent:
|
||||
capability = encryption_content_cipher(get_registry())
|
||||
if capability is None:
|
||||
raise FileStorageError(
|
||||
"File encryption was requested, but the Encryption module is unavailable."
|
||||
)
|
||||
try:
|
||||
return capability.protect_content(
|
||||
session,
|
||||
request=ContentProtectionRequest(
|
||||
tenant_id=tenant_id,
|
||||
owner_module="files",
|
||||
resource_type="file_blob",
|
||||
resource_id=blob_id,
|
||||
profile_id=FILES_PROTECTION_PROFILE,
|
||||
vault_id=vault_id,
|
||||
ciphertext_ref=ciphertext_ref,
|
||||
plaintext=plaintext,
|
||||
policy_decision_ref="files:explicit-vault-selection:v1",
|
||||
idempotency_key=f"file-blob:{blob_id}:content:v1",
|
||||
actor_id=actor_id,
|
||||
metadata={
|
||||
"content_type": content_type or "application/octet-stream",
|
||||
},
|
||||
),
|
||||
)
|
||||
except Exception as exc:
|
||||
raise FileStorageError(
|
||||
"Managed file content could not be protected by the configured vault."
|
||||
) from exc
|
||||
|
||||
|
||||
def unprotect_blob_content(
|
||||
session: object,
|
||||
*,
|
||||
tenant_id: str,
|
||||
blob_id: str,
|
||||
envelope_id: str,
|
||||
ciphertext: bytes,
|
||||
) -> bytes:
|
||||
capability = encryption_content_cipher(get_registry())
|
||||
if capability is None:
|
||||
raise FileStorageError(
|
||||
"This file is encrypted and cannot be read while Encryption is unavailable."
|
||||
)
|
||||
try:
|
||||
return capability.unprotect_content(
|
||||
session,
|
||||
request=ContentUnprotectionRequest(
|
||||
tenant_id=tenant_id,
|
||||
owner_module="files",
|
||||
resource_type="file_blob",
|
||||
resource_id=blob_id,
|
||||
envelope_id=envelope_id,
|
||||
ciphertext=ciphertext,
|
||||
),
|
||||
)
|
||||
except Exception as exc:
|
||||
raise FileStorageError(
|
||||
"Managed file content could not be opened with its protection envelope."
|
||||
) from exc
|
||||
|
||||
|
||||
__all__ = [
|
||||
"FILES_PROTECTION_PROFILE",
|
||||
"protect_blob_content",
|
||||
"unprotect_blob_content",
|
||||
]
|
||||
@@ -2,21 +2,32 @@ from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import mimetypes
|
||||
from datetime import datetime
|
||||
from pathlib import PurePosixPath
|
||||
from typing import Any, Iterable
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import and_, or_
|
||||
from sqlalchemy import and_, exists, func, or_
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from govoplan_core.core.campaigns import CAPABILITY_CAMPAIGNS_ACCESS, CampaignAccessProvider
|
||||
from govoplan_files.backend.db.models import CampaignAttachmentUse, FileAsset, FileBlob, FileShare, FileVersion
|
||||
from govoplan_files.backend.runtime import get_registry, settings
|
||||
from govoplan_files.backend.storage.access import ensure_owner_access, ensure_share_target_exists, user_group_ids
|
||||
from govoplan_files.backend.storage.backends import StorageBackendError, get_storage_backend
|
||||
from govoplan_files.backend.storage.backends import (
|
||||
StorageBackendError,
|
||||
StorageObjectMissing,
|
||||
get_storage_backend,
|
||||
)
|
||||
from govoplan_files.backend.storage.common import FileConflictResolution, FileStorageError, UploadedStoredFile, utcnow
|
||||
from govoplan_files.backend.storage.paths import filename_from_path, join_folder_filename, normalize_folder, normalize_logical_path, safe_storage_component
|
||||
from govoplan_files.backend.storage.paths import filename_from_path, join_folder_filename, normalize_folder, normalize_logical_path
|
||||
from govoplan_files.backend.storage.provenance import source_provenance_from_metadata
|
||||
from govoplan_files.backend.storage.recovery import begin_blob_write_recovery
|
||||
from govoplan_files.backend.storage.integrity import (
|
||||
QUARANTINED_BLOB_STATUSES,
|
||||
read_verified_blob_bytes,
|
||||
)
|
||||
from govoplan_files.backend.storage.share_state import effective_file_share_clause
|
||||
|
||||
|
||||
def _campaign_access_provider() -> CampaignAccessProvider:
|
||||
@@ -51,8 +62,9 @@ def _storage_backend_name() -> str:
|
||||
return settings.file_storage_backend.lower().strip()
|
||||
|
||||
|
||||
def _storage_key(*, tenant_id: str, checksum: str, filename: str) -> str:
|
||||
return f"tenants/{tenant_id}/files/{checksum[:2]}/{uuid4().hex}-{safe_storage_component(filename)}"
|
||||
def _storage_key(*, tenant_id: str, checksum: str) -> str:
|
||||
# Object locators remain opaque so recovery evidence never persists names.
|
||||
return f"tenants/{tenant_id}/files/{checksum[:2]}/{uuid4().hex}.blob"
|
||||
|
||||
|
||||
def _get_or_create_blob(
|
||||
@@ -62,40 +74,144 @@ def _get_or_create_blob(
|
||||
data: bytes,
|
||||
filename: str,
|
||||
content_type: str | None,
|
||||
actor_id: str,
|
||||
encryption_vault_id: str | None = None,
|
||||
) -> FileBlob:
|
||||
checksum = hashlib.sha256(data).hexdigest()
|
||||
size = len(data)
|
||||
vault_id = str(encryption_vault_id or "").strip() or None
|
||||
protection_discriminator = f"vault:{vault_id}" if vault_id else "plaintext"
|
||||
blob = (
|
||||
session.query(FileBlob)
|
||||
.filter(FileBlob.tenant_id == tenant_id, FileBlob.checksum_sha256 == checksum, FileBlob.size_bytes == size)
|
||||
.filter(FileBlob.tenant_id == tenant_id, FileBlob.checksum_sha256 == checksum, FileBlob.size_bytes == size, FileBlob.protection_discriminator == protection_discriminator)
|
||||
.with_for_update()
|
||||
.one_or_none()
|
||||
)
|
||||
if blob:
|
||||
backend = get_storage_backend()
|
||||
if not backend.exists(blob.storage_key):
|
||||
repair_required = (
|
||||
blob.integrity_status in QUARANTINED_BLOB_STATUSES
|
||||
or blob.quarantined_at is not None
|
||||
)
|
||||
try:
|
||||
backend.stat(blob.storage_key)
|
||||
except StorageObjectMissing:
|
||||
repair_required = True
|
||||
except StorageBackendError as exc:
|
||||
raise FileStorageError(str(exc)) from exc
|
||||
if repair_required:
|
||||
repair_token = hashlib.sha256(
|
||||
repr(
|
||||
(
|
||||
blob.integrity_status,
|
||||
blob.integrity_checked_at,
|
||||
blob.quarantined_at,
|
||||
blob.storage_checksum_sha256,
|
||||
)
|
||||
).encode("utf-8")
|
||||
).hexdigest()
|
||||
recovery = begin_blob_write_recovery(
|
||||
session,
|
||||
backend=backend,
|
||||
tenant_id=tenant_id,
|
||||
blob_id=blob.id,
|
||||
storage_key=blob.storage_key,
|
||||
semantic_checksum_sha256=checksum,
|
||||
semantic_size_bytes=size,
|
||||
protection_discriminator=protection_discriminator,
|
||||
created_new=False,
|
||||
repair_token=repair_token,
|
||||
)
|
||||
stored_data = data
|
||||
expected_envelope_id = blob.encryption_envelope_id
|
||||
if vault_id:
|
||||
from govoplan_files.backend.storage.content_protection import protect_blob_content
|
||||
|
||||
protected = protect_blob_content(
|
||||
session,
|
||||
tenant_id=tenant_id,
|
||||
blob_id=blob.id,
|
||||
vault_id=vault_id,
|
||||
ciphertext_ref=blob.storage_key,
|
||||
plaintext=data,
|
||||
actor_id=actor_id,
|
||||
content_type=content_type,
|
||||
)
|
||||
if blob.encryption_envelope_id not in {None, protected.envelope.envelope_id}:
|
||||
raise FileStorageError("The existing encrypted blob has another protection envelope.")
|
||||
stored_data = protected.ciphertext
|
||||
blob.encryption_envelope_id = protected.envelope.envelope_id
|
||||
expected_envelope_id = protected.envelope.envelope_id
|
||||
blob.storage_checksum_sha256 = hashlib.sha256(stored_data).hexdigest()
|
||||
blob.storage_size_bytes = len(stored_data)
|
||||
recovery.prepare_stored_bytes(
|
||||
stored_data,
|
||||
envelope_id=expected_envelope_id,
|
||||
)
|
||||
try:
|
||||
backend.put_bytes(blob.storage_key, data, content_type=content_type)
|
||||
backend.put_bytes(blob.storage_key, stored_data, content_type="application/octet-stream" if vault_id else content_type)
|
||||
except StorageBackendError as exc:
|
||||
raise FileStorageError(str(exc)) from exc
|
||||
blob.integrity_status = "verified"
|
||||
blob.integrity_checked_at = utcnow()
|
||||
blob.integrity_failure = None
|
||||
blob.quarantined_at = None
|
||||
blob.ref_count += 1
|
||||
session.add(blob)
|
||||
return blob
|
||||
|
||||
storage_key = _storage_key(tenant_id=tenant_id, checksum=checksum, filename=filename)
|
||||
blob_id = str(uuid4())
|
||||
storage_key = _storage_key(tenant_id=tenant_id, checksum=checksum)
|
||||
backend = get_storage_backend()
|
||||
recovery = begin_blob_write_recovery(
|
||||
session,
|
||||
backend=backend,
|
||||
tenant_id=tenant_id,
|
||||
blob_id=blob_id,
|
||||
storage_key=storage_key,
|
||||
semantic_checksum_sha256=checksum,
|
||||
semantic_size_bytes=size,
|
||||
protection_discriminator=protection_discriminator,
|
||||
created_new=True,
|
||||
)
|
||||
stored_data = data
|
||||
envelope_id = None
|
||||
if vault_id:
|
||||
from govoplan_files.backend.storage.content_protection import protect_blob_content
|
||||
|
||||
protected = protect_blob_content(
|
||||
session,
|
||||
tenant_id=tenant_id,
|
||||
blob_id=blob_id,
|
||||
vault_id=vault_id,
|
||||
ciphertext_ref=storage_key,
|
||||
plaintext=data,
|
||||
actor_id=actor_id,
|
||||
content_type=content_type,
|
||||
)
|
||||
stored_data = protected.ciphertext
|
||||
envelope_id = protected.envelope.envelope_id
|
||||
recovery.prepare_stored_bytes(stored_data, envelope_id=envelope_id)
|
||||
try:
|
||||
backend.put_bytes(storage_key, data, content_type=content_type)
|
||||
backend.put_bytes(storage_key, stored_data, content_type="application/octet-stream" if vault_id else content_type)
|
||||
except StorageBackendError as exc:
|
||||
raise FileStorageError(str(exc)) from exc
|
||||
blob = FileBlob(
|
||||
id=blob_id,
|
||||
tenant_id=tenant_id,
|
||||
storage_backend=_storage_backend_name(),
|
||||
storage_bucket=_storage_bucket_name(),
|
||||
storage_key=storage_key,
|
||||
checksum_sha256=checksum,
|
||||
size_bytes=size,
|
||||
protection_discriminator=protection_discriminator,
|
||||
encryption_envelope_id=envelope_id,
|
||||
storage_checksum_sha256=hashlib.sha256(stored_data).hexdigest() if vault_id else None,
|
||||
storage_size_bytes=len(stored_data) if vault_id else None,
|
||||
content_type=content_type,
|
||||
ref_count=1,
|
||||
integrity_status="verified",
|
||||
integrity_checked_at=utcnow(),
|
||||
)
|
||||
session.add(blob)
|
||||
session.flush()
|
||||
@@ -120,6 +236,7 @@ def create_file_asset(
|
||||
conflict_strategy: str = "reject",
|
||||
conflict_resolutions: Iterable[FileConflictResolution] | None = None,
|
||||
is_admin: bool = False,
|
||||
encryption_vault_id: str | None = None,
|
||||
) -> UploadedStoredFile:
|
||||
owner_type = owner_type.lower().strip()
|
||||
ensure_owner_access(session, tenant_id=tenant_id, owner_type=owner_type, owner_id=owner_id, user_id=user_id, is_admin=is_admin)
|
||||
@@ -147,7 +264,7 @@ def create_file_asset(
|
||||
elif action == "rename":
|
||||
logical_path = _next_available_logical_path(session, tenant_id=tenant_id, owner_type=owner_type, owner_id=owner_id, desired_path=logical_path)
|
||||
|
||||
blob = _get_or_create_blob(session, tenant_id=tenant_id, data=data, filename=safe_filename, content_type=content_type)
|
||||
blob = _get_or_create_blob(session, tenant_id=tenant_id, data=data, filename=safe_filename, content_type=content_type, actor_id=user_id, encryption_vault_id=encryption_vault_id)
|
||||
asset = FileAsset(
|
||||
tenant_id=tenant_id,
|
||||
owner_type=owner_type,
|
||||
@@ -279,6 +396,7 @@ def update_file_asset_content(
|
||||
data: bytes,
|
||||
content_type: str | None,
|
||||
metadata: dict[str, Any],
|
||||
encryption_vault_id: str | None = None,
|
||||
) -> tuple[UploadedStoredFile, str]:
|
||||
if asset.tenant_id != tenant_id or asset.deleted_at is not None:
|
||||
raise FileStorageError("File not found")
|
||||
@@ -289,10 +407,23 @@ def update_file_asset_content(
|
||||
checksum = hashlib.sha256(data).hexdigest()
|
||||
asset.metadata_ = metadata
|
||||
session.add(asset)
|
||||
if current_blob.checksum_sha256 == checksum and current_blob.size_bytes == len(data):
|
||||
inherited_vault_id = encryption_vault_id
|
||||
if inherited_vault_id is None and current_blob.encryption_envelope_id:
|
||||
prefix = "vault:"
|
||||
if current_blob.protection_discriminator.startswith(prefix):
|
||||
inherited_vault_id = current_blob.protection_discriminator[len(prefix) :]
|
||||
inherited_vault_id = str(inherited_vault_id or "").strip() or None
|
||||
target_protection = (
|
||||
f"vault:{inherited_vault_id}" if inherited_vault_id else "plaintext"
|
||||
)
|
||||
if (
|
||||
current_blob.checksum_sha256 == checksum
|
||||
and current_blob.size_bytes == len(data)
|
||||
and current_blob.protection_discriminator == target_protection
|
||||
):
|
||||
return UploadedStoredFile(asset=asset, version=current_version, blob=current_blob), "unchanged"
|
||||
|
||||
blob = _get_or_create_blob(session, tenant_id=tenant_id, data=data, filename=safe_filename, content_type=content_type)
|
||||
blob = _get_or_create_blob(session, tenant_id=tenant_id, data=data, filename=safe_filename, content_type=content_type, actor_id=user_id, encryption_vault_id=inherited_vault_id)
|
||||
version = FileVersion(
|
||||
tenant_id=tenant_id,
|
||||
file_asset_id=asset.id,
|
||||
@@ -328,7 +459,7 @@ def get_asset_for_user(session: Session, *, tenant_id: str, user_id: str, asset_
|
||||
.filter(
|
||||
FileShare.tenant_id == tenant_id,
|
||||
FileShare.file_asset_id == asset.id,
|
||||
FileShare.revoked_at.is_(None),
|
||||
effective_file_share_clause(),
|
||||
FileShare.permission.in_(permission_values),
|
||||
or_(
|
||||
(FileShare.target_type == "user") & (FileShare.target_id == user_id),
|
||||
@@ -343,6 +474,32 @@ def get_asset_for_user(session: Session, *, tenant_id: str, user_id: str, asset_
|
||||
return asset
|
||||
|
||||
|
||||
def get_asset_for_share_management(
|
||||
session: Session,
|
||||
*,
|
||||
tenant_id: str,
|
||||
user_id: str,
|
||||
asset_id: str,
|
||||
is_admin: bool = False,
|
||||
) -> FileAsset:
|
||||
asset = session.get(FileAsset, asset_id)
|
||||
if not asset or asset.tenant_id != tenant_id or asset.deleted_at is not None:
|
||||
raise FileStorageError("File not found")
|
||||
if is_admin:
|
||||
return asset
|
||||
owns_asset = (
|
||||
asset.owner_type == "user"
|
||||
and asset.owner_user_id == user_id
|
||||
) or (
|
||||
asset.owner_type == "group"
|
||||
and asset.owner_group_id
|
||||
in user_group_ids(session, tenant_id=tenant_id, user_id=user_id)
|
||||
)
|
||||
if not owns_asset:
|
||||
raise FileStorageError("Only file owners and administrators can manage shares")
|
||||
return asset
|
||||
|
||||
|
||||
def list_assets_for_user(
|
||||
session: Session,
|
||||
*,
|
||||
@@ -352,6 +509,8 @@ def list_assets_for_user(
|
||||
owner_id: str | None = None,
|
||||
campaign_id: str | None = None,
|
||||
path_prefix: str | None = None,
|
||||
campaign_usage: str | None = None,
|
||||
audit_relevant: bool | None = None,
|
||||
include_deleted: bool = False,
|
||||
is_admin: bool = False,
|
||||
) -> list[FileAsset]:
|
||||
@@ -363,12 +522,49 @@ def list_assets_for_user(
|
||||
owner_id=owner_id,
|
||||
campaign_id=campaign_id,
|
||||
path_prefix=path_prefix,
|
||||
campaign_usage=campaign_usage,
|
||||
audit_relevant=audit_relevant,
|
||||
include_deleted=include_deleted,
|
||||
is_admin=is_admin,
|
||||
)
|
||||
return query.order_by(FileAsset.display_path.asc(), FileAsset.updated_at.desc(), FileAsset.id.asc()).all()
|
||||
|
||||
|
||||
def list_recent_assets_for_user(
|
||||
session: Session,
|
||||
*,
|
||||
tenant_id: str,
|
||||
user_id: str,
|
||||
limit: int,
|
||||
owner_type: str | None = None,
|
||||
owner_id: str | None = None,
|
||||
campaign_id: str | None = None,
|
||||
path_prefix: str | None = None,
|
||||
campaign_usage: str | None = None,
|
||||
audit_relevant: bool | None = None,
|
||||
is_admin: bool = False,
|
||||
) -> list[FileAsset]:
|
||||
"""Return a bounded recent projection through the normal access query."""
|
||||
|
||||
query = _asset_visibility_query_for_user(
|
||||
session,
|
||||
tenant_id=tenant_id,
|
||||
user_id=user_id,
|
||||
owner_type=owner_type,
|
||||
owner_id=owner_id,
|
||||
campaign_id=campaign_id,
|
||||
path_prefix=path_prefix,
|
||||
campaign_usage=campaign_usage,
|
||||
audit_relevant=audit_relevant,
|
||||
is_admin=is_admin,
|
||||
)
|
||||
return (
|
||||
query.order_by(FileAsset.updated_at.desc(), FileAsset.id.asc())
|
||||
.limit(max(1, limit))
|
||||
.all()
|
||||
)
|
||||
|
||||
|
||||
def list_assets_for_user_window(
|
||||
session: Session,
|
||||
*,
|
||||
@@ -378,6 +574,8 @@ def list_assets_for_user_window(
|
||||
owner_id: str | None = None,
|
||||
campaign_id: str | None = None,
|
||||
path_prefix: str | None = None,
|
||||
campaign_usage: str | None = None,
|
||||
audit_relevant: bool | None = None,
|
||||
include_deleted: bool = False,
|
||||
is_admin: bool = False,
|
||||
page_size: int,
|
||||
@@ -393,6 +591,8 @@ def list_assets_for_user_window(
|
||||
owner_id=owner_id,
|
||||
campaign_id=campaign_id,
|
||||
path_prefix=path_prefix,
|
||||
campaign_usage=campaign_usage,
|
||||
audit_relevant=audit_relevant,
|
||||
include_deleted=include_deleted,
|
||||
is_admin=is_admin,
|
||||
)
|
||||
@@ -417,6 +617,8 @@ def _asset_visibility_query_for_user(
|
||||
owner_id: str | None = None,
|
||||
campaign_id: str | None = None,
|
||||
path_prefix: str | None = None,
|
||||
campaign_usage: str | None = None,
|
||||
audit_relevant: bool | None = None,
|
||||
include_deleted: bool = False,
|
||||
is_admin: bool = False,
|
||||
):
|
||||
@@ -430,30 +632,97 @@ def _asset_visibility_query_for_user(
|
||||
if owner_type == "group" and owner_id:
|
||||
query = query.filter(FileAsset.owner_group_id == owner_id)
|
||||
if campaign_id:
|
||||
query = query.join(FileShare, FileShare.file_asset_id == FileAsset.id).filter(
|
||||
campaign_share = exists().where(
|
||||
FileShare.tenant_id == tenant_id,
|
||||
FileShare.file_asset_id == FileAsset.id,
|
||||
FileShare.target_type == "campaign",
|
||||
FileShare.target_id == campaign_id,
|
||||
FileShare.revoked_at.is_(None),
|
||||
effective_file_share_clause(),
|
||||
)
|
||||
query = query.filter(campaign_share)
|
||||
elif not is_admin and not owner_type:
|
||||
group_ids = user_group_ids(session, tenant_id=tenant_id, user_id=user_id)
|
||||
query = query.outerjoin(FileShare, FileShare.file_asset_id == FileAsset.id).filter(
|
||||
active_share = exists().where(
|
||||
FileShare.tenant_id == tenant_id,
|
||||
FileShare.file_asset_id == FileAsset.id,
|
||||
effective_file_share_clause(),
|
||||
or_(
|
||||
(FileShare.target_type == "user") & (FileShare.target_id == user_id),
|
||||
(FileShare.target_type == "group") & (FileShare.target_id.in_(group_ids)),
|
||||
(FileShare.target_type == "tenant") & (FileShare.target_id == tenant_id),
|
||||
)
|
||||
)
|
||||
query = query.filter(
|
||||
or_(
|
||||
(FileAsset.owner_type == "user") & (FileAsset.owner_user_id == user_id),
|
||||
(FileAsset.owner_type == "group") & (FileAsset.owner_group_id.in_(group_ids)),
|
||||
(FileShare.revoked_at.is_(None)) & (FileShare.target_type == "user") & (FileShare.target_id == user_id),
|
||||
(FileShare.revoked_at.is_(None)) & (FileShare.target_type == "group") & (FileShare.target_id.in_(group_ids)),
|
||||
(FileShare.revoked_at.is_(None)) & (FileShare.target_type == "tenant") & (FileShare.target_id == tenant_id),
|
||||
active_share,
|
||||
)
|
||||
)
|
||||
if path_prefix:
|
||||
prefix = normalize_folder(path_prefix)
|
||||
if prefix:
|
||||
query = query.filter(FileAsset.display_path.like(f"{prefix}/%"))
|
||||
campaign_share_exists = exists().where(
|
||||
FileShare.tenant_id == tenant_id,
|
||||
FileShare.file_asset_id == FileAsset.id,
|
||||
FileShare.target_type == "campaign",
|
||||
effective_file_share_clause(),
|
||||
)
|
||||
campaign_use_exists = exists().where(
|
||||
CampaignAttachmentUse.tenant_id == tenant_id,
|
||||
CampaignAttachmentUse.file_asset_id == FileAsset.id,
|
||||
)
|
||||
if campaign_usage == "linked":
|
||||
query = query.filter(or_(campaign_share_exists, campaign_use_exists))
|
||||
elif campaign_usage == "unlinked":
|
||||
query = query.filter(~or_(campaign_share_exists, campaign_use_exists))
|
||||
|
||||
if audit_relevant is not None:
|
||||
sent_use_exists = exists().where(
|
||||
CampaignAttachmentUse.tenant_id == tenant_id,
|
||||
CampaignAttachmentUse.file_asset_id == FileAsset.id,
|
||||
CampaignAttachmentUse.use_stage == "sent",
|
||||
)
|
||||
query = query.filter(sent_use_exists if audit_relevant else ~sent_use_exists)
|
||||
return query
|
||||
|
||||
|
||||
def count_assets_for_user(
|
||||
session: Session,
|
||||
*,
|
||||
tenant_id: str,
|
||||
user_id: str,
|
||||
owner_type: str | None = None,
|
||||
owner_id: str | None = None,
|
||||
campaign_id: str | None = None,
|
||||
path_prefix: str | None = None,
|
||||
campaign_usage: str | None = None,
|
||||
audit_relevant: bool | None = None,
|
||||
include_deleted: bool = False,
|
||||
is_admin: bool = False,
|
||||
) -> int:
|
||||
query = _asset_visibility_query_for_user(
|
||||
session,
|
||||
tenant_id=tenant_id,
|
||||
user_id=user_id,
|
||||
owner_type=owner_type,
|
||||
owner_id=owner_id,
|
||||
campaign_id=campaign_id,
|
||||
path_prefix=path_prefix,
|
||||
campaign_usage=campaign_usage,
|
||||
audit_relevant=audit_relevant,
|
||||
include_deleted=include_deleted,
|
||||
is_admin=is_admin,
|
||||
)
|
||||
return int(
|
||||
query.order_by(None)
|
||||
.with_entities(func.count(func.distinct(FileAsset.id)))
|
||||
.scalar()
|
||||
or 0
|
||||
)
|
||||
|
||||
|
||||
def current_version_and_blob(session: Session, asset: FileAsset) -> tuple[FileVersion, FileBlob]:
|
||||
if not asset.current_version_id:
|
||||
raise FileStorageError("File has no current version")
|
||||
@@ -495,10 +764,22 @@ def current_versions_and_blobs(session: Session, assets: Iterable[FileAsset]) ->
|
||||
def read_asset_bytes(session: Session, asset: FileAsset) -> tuple[bytes, FileVersion, FileBlob]:
|
||||
version, blob = current_version_and_blob(session, asset)
|
||||
backend = get_storage_backend()
|
||||
try:
|
||||
return backend.get_bytes(blob.storage_key), version, blob
|
||||
except StorageBackendError as exc:
|
||||
raise FileStorageError(str(exc)) from exc
|
||||
return read_verified_blob_bytes(blob, backend=backend), version, blob
|
||||
|
||||
|
||||
def read_asset_version_bytes(
|
||||
session: Session,
|
||||
asset: FileAsset,
|
||||
version_id: str,
|
||||
) -> tuple[bytes, FileVersion, FileBlob]:
|
||||
version = session.get(FileVersion, version_id)
|
||||
if version is None or version.file_asset_id != asset.id:
|
||||
raise FileStorageError("File version not found")
|
||||
blob = session.get(FileBlob, version.blob_id)
|
||||
if blob is None or blob.tenant_id != asset.tenant_id:
|
||||
raise FileStorageError("File blob not found")
|
||||
backend = get_storage_backend()
|
||||
return read_verified_blob_bytes(blob, backend=backend), version, blob
|
||||
|
||||
|
||||
def share_file(
|
||||
@@ -510,6 +791,7 @@ def share_file(
|
||||
target_id: str,
|
||||
permission: str,
|
||||
user_id: str,
|
||||
expires_at: datetime | None = None,
|
||||
) -> FileShare:
|
||||
target_type = target_type.lower().strip()
|
||||
permission = permission.lower().strip()
|
||||
@@ -519,6 +801,7 @@ def share_file(
|
||||
raise FileStorageError("Unsupported share target")
|
||||
if permission not in {"read", "write", "manage"}:
|
||||
raise FileStorageError("Unsupported file permission")
|
||||
_validate_share_expiry(expires_at)
|
||||
if target_type in {"user", "group", "tenant"}:
|
||||
ensure_share_target_exists(tenant_id=tenant_id, target_type=target_type, target_id=target_id)
|
||||
if target_type == "campaign":
|
||||
@@ -536,6 +819,7 @@ def share_file(
|
||||
)
|
||||
if existing:
|
||||
existing.permission = permission
|
||||
existing.expires_at = expires_at
|
||||
session.add(existing)
|
||||
return existing
|
||||
share = FileShare(
|
||||
@@ -545,6 +829,7 @@ def share_file(
|
||||
target_id=target_id,
|
||||
permission=permission,
|
||||
created_by_user_id=user_id,
|
||||
expires_at=expires_at,
|
||||
)
|
||||
session.add(share)
|
||||
return share
|
||||
@@ -559,6 +844,7 @@ def share_files(
|
||||
target_id: str,
|
||||
permission: str,
|
||||
user_id: str,
|
||||
expires_at: datetime | None = None,
|
||||
) -> list[FileShare]:
|
||||
target_type = target_type.lower().strip()
|
||||
permission = permission.lower().strip()
|
||||
@@ -566,6 +852,7 @@ def share_files(
|
||||
raise FileStorageError("Unsupported share target")
|
||||
if permission not in {"read", "write", "manage"}:
|
||||
raise FileStorageError("Unsupported file permission")
|
||||
_validate_share_expiry(expires_at)
|
||||
if target_type in {"user", "group", "tenant"}:
|
||||
ensure_share_target_exists(tenant_id=tenant_id, target_type=target_type, target_id=target_id)
|
||||
if target_type == "campaign":
|
||||
@@ -593,6 +880,7 @@ def share_files(
|
||||
existing = existing_by_asset.get(asset.id)
|
||||
if existing:
|
||||
existing.permission = permission
|
||||
existing.expires_at = expires_at
|
||||
session.add(existing)
|
||||
shares.append(existing)
|
||||
continue
|
||||
@@ -603,12 +891,88 @@ def share_files(
|
||||
target_id=target_id,
|
||||
permission=permission,
|
||||
created_by_user_id=user_id,
|
||||
expires_at=expires_at,
|
||||
)
|
||||
session.add(share)
|
||||
shares.append(share)
|
||||
return shares
|
||||
|
||||
|
||||
def list_file_shares(
|
||||
session: Session,
|
||||
*,
|
||||
tenant_id: str,
|
||||
asset_id: str,
|
||||
include_inactive: bool = False,
|
||||
) -> list[FileShare]:
|
||||
query = session.query(FileShare).filter(
|
||||
FileShare.tenant_id == tenant_id,
|
||||
FileShare.file_asset_id == asset_id,
|
||||
)
|
||||
if not include_inactive:
|
||||
query = query.filter(effective_file_share_clause())
|
||||
return query.order_by(FileShare.created_at.desc(), FileShare.id.desc()).all()
|
||||
|
||||
|
||||
def revoke_file_share(
|
||||
session: Session,
|
||||
*,
|
||||
tenant_id: str,
|
||||
asset_id: str,
|
||||
share_id: str,
|
||||
user_id: str,
|
||||
) -> tuple[FileShare, bool]:
|
||||
share = (
|
||||
session.query(FileShare)
|
||||
.filter(
|
||||
FileShare.id == share_id,
|
||||
FileShare.tenant_id == tenant_id,
|
||||
FileShare.file_asset_id == asset_id,
|
||||
)
|
||||
.one_or_none()
|
||||
)
|
||||
if share is None:
|
||||
raise FileStorageError("File share not found")
|
||||
if share.revoked_at is not None:
|
||||
return share, False
|
||||
share.revoked_at = utcnow()
|
||||
share.revoked_by_user_id = user_id
|
||||
session.add(share)
|
||||
return share, True
|
||||
|
||||
|
||||
def current_file_share_for_target(
|
||||
session: Session,
|
||||
*,
|
||||
tenant_id: str,
|
||||
asset_id: str,
|
||||
target_type: str,
|
||||
target_id: str,
|
||||
) -> FileShare | None:
|
||||
return (
|
||||
session.query(FileShare)
|
||||
.filter(
|
||||
FileShare.tenant_id == tenant_id,
|
||||
FileShare.file_asset_id == asset_id,
|
||||
FileShare.target_type == target_type.lower().strip(),
|
||||
FileShare.target_id == target_id,
|
||||
FileShare.revoked_at.is_(None),
|
||||
)
|
||||
.order_by(FileShare.created_at.desc(), FileShare.id.desc())
|
||||
.first()
|
||||
)
|
||||
|
||||
|
||||
def _validate_share_expiry(expires_at: datetime | None) -> None:
|
||||
if expires_at is None:
|
||||
return
|
||||
from govoplan_files.backend.storage.share_state import file_share_is_active
|
||||
|
||||
candidate = FileShare(expires_at=expires_at)
|
||||
if not file_share_is_active(candidate):
|
||||
raise FileStorageError("File share expiry must be in the future")
|
||||
|
||||
|
||||
|
||||
def soft_delete_assets(session: Session, assets: Iterable[FileAsset]) -> int:
|
||||
count = 0
|
||||
|
||||
@@ -185,15 +185,23 @@ class _OutboundPolicyHTTPTransport(httpx.BaseTransport):
|
||||
self._connection_pool.close()
|
||||
|
||||
|
||||
_CONNECTOR_HTTP_CLIENT = httpx.Client(
|
||||
transport=_OutboundPolicyHTTPTransport(),
|
||||
follow_redirects=False,
|
||||
timeout=15.0,
|
||||
)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _stream_connector_request(method: str, url: str, **kwargs: Any) -> Iterator[httpx.Response]:
|
||||
with httpx.Client(
|
||||
transport=_OutboundPolicyHTTPTransport(),
|
||||
follow_redirects=False,
|
||||
timeout=kwargs.pop("timeout", 15.0),
|
||||
) as client:
|
||||
with client.stream(method, url, **kwargs) as response:
|
||||
yield response
|
||||
timeout = kwargs.pop("timeout", 15.0)
|
||||
with _CONNECTOR_HTTP_CLIENT.stream(
|
||||
method,
|
||||
url,
|
||||
timeout=timeout,
|
||||
**kwargs,
|
||||
) as response:
|
||||
yield response
|
||||
|
||||
|
||||
def request_connector_bytes(
|
||||
|
||||
@@ -0,0 +1,550 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from dataclasses import dataclass
|
||||
|
||||
from sqlalchemy.orm import Session, object_session
|
||||
|
||||
from govoplan_files.backend.db.models import (
|
||||
FileBlob,
|
||||
FileIntegrityFinding,
|
||||
FileIntegrityScan,
|
||||
)
|
||||
from govoplan_files.backend.storage.backends import (
|
||||
StorageBackend,
|
||||
StorageBackendError,
|
||||
StorageObjectMissing,
|
||||
get_storage_backend,
|
||||
)
|
||||
from govoplan_files.backend.storage.recovery import (
|
||||
begin_orphan_cleanup_recovery,
|
||||
)
|
||||
from govoplan_files.backend.storage.common import FileStorageError, utcnow
|
||||
|
||||
|
||||
TERMINAL_SCAN_STATUSES = {"completed", "cancelled"}
|
||||
QUARANTINED_BLOB_STATUSES = {
|
||||
"missing",
|
||||
"size_mismatch",
|
||||
"checksum_mismatch",
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class BlobInspection:
|
||||
valid: bool
|
||||
kind: str
|
||||
observed_size_bytes: int | None = None
|
||||
observed_checksum_sha256: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class IntegrityActionResult:
|
||||
action: str
|
||||
changed: bool
|
||||
dry_run: bool
|
||||
finding: FileIntegrityFinding
|
||||
inspection: BlobInspection | None = None
|
||||
|
||||
|
||||
def storage_prefix_for_tenant(tenant_id: str) -> str:
|
||||
return f"tenants/{tenant_id}/files/"
|
||||
|
||||
|
||||
def create_integrity_scan(
|
||||
session: Session,
|
||||
*,
|
||||
tenant_id: str,
|
||||
user_id: str,
|
||||
verify_checksums: bool = True,
|
||||
batch_size: int = 100,
|
||||
backend: StorageBackend | None = None,
|
||||
) -> FileIntegrityScan:
|
||||
active_backend = backend or get_storage_backend()
|
||||
scan = FileIntegrityScan(
|
||||
tenant_id=tenant_id,
|
||||
storage_backend=active_backend.name,
|
||||
storage_prefix=storage_prefix_for_tenant(tenant_id),
|
||||
verify_checksums=verify_checksums,
|
||||
batch_size=max(1, min(int(batch_size), 1000)),
|
||||
created_by_user_id=user_id,
|
||||
)
|
||||
session.add(scan)
|
||||
session.flush()
|
||||
return scan
|
||||
|
||||
|
||||
def run_integrity_scan_batch(
|
||||
session: Session,
|
||||
scan: FileIntegrityScan,
|
||||
*,
|
||||
backend: StorageBackend | None = None,
|
||||
) -> FileIntegrityScan:
|
||||
if scan.status in TERMINAL_SCAN_STATUSES:
|
||||
return scan
|
||||
active_backend = backend or get_storage_backend()
|
||||
if active_backend.name != scan.storage_backend:
|
||||
raise FileStorageError(
|
||||
"The configured storage backend changed after this integrity scan started"
|
||||
)
|
||||
if scan.started_at is None:
|
||||
scan.started_at = utcnow()
|
||||
scan.status = "running"
|
||||
scan.last_error = None
|
||||
if scan.phase == "blobs":
|
||||
_scan_blob_batch(session, scan, active_backend)
|
||||
elif scan.phase == "objects":
|
||||
_scan_object_batch(session, scan, active_backend)
|
||||
else:
|
||||
scan.phase = "completed"
|
||||
scan.status = "completed"
|
||||
scan.completed_at = utcnow()
|
||||
scan.revision += 1
|
||||
session.add(scan)
|
||||
return scan
|
||||
|
||||
|
||||
def mark_integrity_scan_failed(
|
||||
scan: FileIntegrityScan,
|
||||
*,
|
||||
error: Exception,
|
||||
) -> None:
|
||||
scan.status = "failed"
|
||||
scan.last_error = type(error).__name__[:255]
|
||||
scan.revision += 1
|
||||
|
||||
|
||||
def inspect_blob(
|
||||
blob: FileBlob,
|
||||
*,
|
||||
backend: StorageBackend,
|
||||
verify_checksum: bool = True,
|
||||
) -> BlobInspection:
|
||||
expected_size = blob.storage_size_bytes if blob.storage_size_bytes is not None else blob.size_bytes
|
||||
expected_checksum = blob.storage_checksum_sha256 if blob.storage_checksum_sha256 is not None else blob.checksum_sha256
|
||||
try:
|
||||
info = backend.stat(blob.storage_key)
|
||||
except StorageObjectMissing:
|
||||
return BlobInspection(valid=False, kind="missing")
|
||||
if info.size_bytes != expected_size:
|
||||
return BlobInspection(
|
||||
valid=False,
|
||||
kind="size_mismatch",
|
||||
observed_size_bytes=info.size_bytes,
|
||||
)
|
||||
if not verify_checksum:
|
||||
return BlobInspection(
|
||||
valid=True,
|
||||
kind="verified",
|
||||
observed_size_bytes=info.size_bytes,
|
||||
)
|
||||
digest = hashlib.sha256()
|
||||
observed_size = 0
|
||||
for chunk in backend.iter_bytes(blob.storage_key):
|
||||
observed_size += len(chunk)
|
||||
digest.update(chunk)
|
||||
observed_checksum = digest.hexdigest()
|
||||
if observed_size != expected_size:
|
||||
return BlobInspection(
|
||||
valid=False,
|
||||
kind="size_mismatch",
|
||||
observed_size_bytes=observed_size,
|
||||
observed_checksum_sha256=observed_checksum,
|
||||
)
|
||||
if observed_checksum != expected_checksum:
|
||||
return BlobInspection(
|
||||
valid=False,
|
||||
kind="checksum_mismatch",
|
||||
observed_size_bytes=observed_size,
|
||||
observed_checksum_sha256=observed_checksum,
|
||||
)
|
||||
return BlobInspection(
|
||||
valid=True,
|
||||
kind="verified",
|
||||
observed_size_bytes=observed_size,
|
||||
observed_checksum_sha256=observed_checksum,
|
||||
)
|
||||
|
||||
|
||||
def apply_blob_inspection(
|
||||
session: Session,
|
||||
blob: FileBlob,
|
||||
inspection: BlobInspection,
|
||||
*,
|
||||
checked_at=None,
|
||||
) -> None:
|
||||
timestamp = checked_at or utcnow()
|
||||
blob.integrity_checked_at = timestamp
|
||||
if inspection.valid:
|
||||
blob.integrity_status = "verified"
|
||||
blob.integrity_failure = None
|
||||
blob.quarantined_at = None
|
||||
else:
|
||||
blob.integrity_status = inspection.kind
|
||||
blob.integrity_failure = inspection.kind
|
||||
blob.quarantined_at = blob.quarantined_at or timestamp
|
||||
session.add(blob)
|
||||
|
||||
|
||||
def ensure_blob_is_readable(blob: FileBlob) -> None:
|
||||
if blob.integrity_status in QUARANTINED_BLOB_STATUSES or blob.quarantined_at:
|
||||
raise FileStorageError(
|
||||
"Managed file content is quarantined because integrity verification failed"
|
||||
)
|
||||
|
||||
|
||||
def read_verified_blob_bytes(
|
||||
blob: FileBlob,
|
||||
*,
|
||||
backend: StorageBackend,
|
||||
) -> bytes:
|
||||
ensure_blob_is_readable(blob)
|
||||
try:
|
||||
data = backend.get_bytes(blob.storage_key)
|
||||
except StorageBackendError as exc:
|
||||
raise FileStorageError("Managed file content is not available") from exc
|
||||
expected_storage_size = blob.storage_size_bytes if blob.storage_size_bytes is not None else blob.size_bytes
|
||||
expected_storage_checksum = blob.storage_checksum_sha256 if blob.storage_checksum_sha256 is not None else blob.checksum_sha256
|
||||
if len(data) != expected_storage_size:
|
||||
raise FileStorageError(
|
||||
"Managed file content failed its recorded size verification"
|
||||
)
|
||||
if hashlib.sha256(data).hexdigest() != expected_storage_checksum:
|
||||
raise FileStorageError(
|
||||
"Managed file content failed its recorded checksum verification"
|
||||
)
|
||||
if blob.encryption_envelope_id:
|
||||
session = object_session(blob)
|
||||
if session is None:
|
||||
raise FileStorageError("Encrypted managed content requires an attached database session")
|
||||
from govoplan_files.backend.storage.content_protection import unprotect_blob_content
|
||||
|
||||
data = unprotect_blob_content(
|
||||
session,
|
||||
tenant_id=blob.tenant_id,
|
||||
blob_id=blob.id,
|
||||
envelope_id=blob.encryption_envelope_id,
|
||||
ciphertext=data,
|
||||
)
|
||||
if len(data) != blob.size_bytes:
|
||||
raise FileStorageError("Decrypted managed file content failed its recorded size verification")
|
||||
if hashlib.sha256(data).hexdigest() != blob.checksum_sha256:
|
||||
raise FileStorageError("Decrypted managed file content failed its recorded checksum verification")
|
||||
return data
|
||||
|
||||
|
||||
def recheck_integrity_finding(
|
||||
session: Session,
|
||||
finding: FileIntegrityFinding,
|
||||
*,
|
||||
user_id: str,
|
||||
dry_run: bool,
|
||||
backend: StorageBackend | None = None,
|
||||
) -> IntegrityActionResult:
|
||||
if finding.kind == "orphan_object" or not finding.blob_id:
|
||||
raise FileStorageError("Only blob integrity findings can be rechecked")
|
||||
blob = session.get(FileBlob, finding.blob_id)
|
||||
if blob is None or blob.tenant_id != finding.tenant_id:
|
||||
raise FileStorageError("Integrity finding blob no longer exists")
|
||||
active_backend = backend or get_storage_backend()
|
||||
scan = session.get(FileIntegrityScan, finding.scan_id)
|
||||
if scan is None or scan.storage_backend != active_backend.name:
|
||||
raise FileStorageError(
|
||||
"The configured storage backend does not match the integrity finding"
|
||||
)
|
||||
inspection = inspect_blob(
|
||||
blob,
|
||||
backend=active_backend,
|
||||
verify_checksum=True,
|
||||
)
|
||||
changed = False
|
||||
if not dry_run:
|
||||
previous = (
|
||||
blob.integrity_status,
|
||||
blob.quarantined_at,
|
||||
finding.state,
|
||||
)
|
||||
apply_blob_inspection(session, blob, inspection)
|
||||
_update_finding_from_inspection(finding, inspection)
|
||||
if inspection.valid:
|
||||
finding.state = "resolved"
|
||||
finding.resolved_at = utcnow()
|
||||
finding.resolved_by_user_id = user_id
|
||||
session.add(finding)
|
||||
finding.revision += 1
|
||||
changed = previous != (
|
||||
blob.integrity_status,
|
||||
blob.quarantined_at,
|
||||
finding.state,
|
||||
)
|
||||
return IntegrityActionResult(
|
||||
action="recheck",
|
||||
changed=changed,
|
||||
dry_run=dry_run,
|
||||
finding=finding,
|
||||
inspection=inspection,
|
||||
)
|
||||
|
||||
|
||||
def cleanup_orphan_finding(
|
||||
session: Session,
|
||||
finding: FileIntegrityFinding,
|
||||
*,
|
||||
user_id: str,
|
||||
dry_run: bool,
|
||||
backend: StorageBackend | None = None,
|
||||
) -> IntegrityActionResult:
|
||||
if finding.kind != "orphan_object":
|
||||
raise FileStorageError("Only orphan-object findings can be cleaned up")
|
||||
scan = session.get(FileIntegrityScan, finding.scan_id)
|
||||
if scan is None or scan.tenant_id != finding.tenant_id:
|
||||
raise FileStorageError("Integrity scan not found")
|
||||
if not finding.storage_key.startswith(scan.storage_prefix):
|
||||
raise FileStorageError("Orphan cleanup is outside the scan storage scope")
|
||||
referenced = (
|
||||
session.query(FileBlob.id)
|
||||
.filter(
|
||||
FileBlob.tenant_id == finding.tenant_id,
|
||||
FileBlob.storage_backend == scan.storage_backend,
|
||||
FileBlob.storage_key == finding.storage_key,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if referenced:
|
||||
if not dry_run and finding.state != "resolved":
|
||||
finding.state = "resolved"
|
||||
finding.resolved_at = utcnow()
|
||||
finding.resolved_by_user_id = user_id
|
||||
finding.revision += 1
|
||||
session.add(finding)
|
||||
return IntegrityActionResult(
|
||||
action="retained_referenced",
|
||||
changed=True,
|
||||
dry_run=False,
|
||||
finding=finding,
|
||||
)
|
||||
return IntegrityActionResult(
|
||||
action="retained_referenced",
|
||||
changed=False,
|
||||
dry_run=dry_run,
|
||||
finding=finding,
|
||||
)
|
||||
if finding.state == "deleted":
|
||||
return IntegrityActionResult(
|
||||
action="already_deleted",
|
||||
changed=False,
|
||||
dry_run=dry_run,
|
||||
finding=finding,
|
||||
)
|
||||
if dry_run:
|
||||
return IntegrityActionResult(
|
||||
action="would_delete",
|
||||
changed=False,
|
||||
dry_run=True,
|
||||
finding=finding,
|
||||
)
|
||||
active_backend = backend or get_storage_backend()
|
||||
if active_backend.name != scan.storage_backend:
|
||||
raise FileStorageError(
|
||||
"The configured storage backend does not match the integrity finding"
|
||||
)
|
||||
begin_orphan_cleanup_recovery(
|
||||
session,
|
||||
finding,
|
||||
backend=active_backend,
|
||||
user_id=user_id,
|
||||
)
|
||||
try:
|
||||
active_backend.stat(finding.storage_key)
|
||||
except StorageObjectMissing:
|
||||
action = "already_absent"
|
||||
else:
|
||||
active_backend.delete(finding.storage_key)
|
||||
action = "deleted"
|
||||
finding.state = "deleted"
|
||||
finding.resolved_at = utcnow()
|
||||
finding.resolved_by_user_id = user_id
|
||||
finding.revision += 1
|
||||
session.add(finding)
|
||||
return IntegrityActionResult(
|
||||
action=action,
|
||||
changed=True,
|
||||
dry_run=False,
|
||||
finding=finding,
|
||||
)
|
||||
|
||||
|
||||
def _scan_blob_batch(
|
||||
session: Session,
|
||||
scan: FileIntegrityScan,
|
||||
backend: StorageBackend,
|
||||
) -> None:
|
||||
query = session.query(FileBlob).filter(
|
||||
FileBlob.tenant_id == scan.tenant_id,
|
||||
FileBlob.storage_backend == scan.storage_backend,
|
||||
)
|
||||
if scan.blob_cursor:
|
||||
query = query.filter(FileBlob.id > scan.blob_cursor)
|
||||
blobs = query.order_by(FileBlob.id.asc()).limit(scan.batch_size).all()
|
||||
for blob in blobs:
|
||||
inspection = inspect_blob(
|
||||
blob,
|
||||
backend=backend,
|
||||
verify_checksum=scan.verify_checksums,
|
||||
)
|
||||
apply_blob_inspection(session, blob, inspection)
|
||||
scan.scanned_blob_count += 1
|
||||
if inspection.valid:
|
||||
scan.verified_blob_count += 1
|
||||
_resolve_scan_blob_findings(session, scan, blob)
|
||||
else:
|
||||
scan.quarantined_blob_count += 1
|
||||
_record_blob_finding(session, scan, blob, inspection)
|
||||
scan.blob_cursor = blob.id
|
||||
if len(blobs) < scan.batch_size:
|
||||
scan.phase = "objects"
|
||||
scan.object_cursor = None
|
||||
|
||||
|
||||
def _scan_object_batch(
|
||||
session: Session,
|
||||
scan: FileIntegrityScan,
|
||||
backend: StorageBackend,
|
||||
) -> None:
|
||||
page = backend.list_objects(
|
||||
prefix=scan.storage_prefix,
|
||||
after=scan.object_cursor,
|
||||
limit=scan.batch_size,
|
||||
)
|
||||
keys = [item.key for item in page.objects]
|
||||
referenced_keys = {
|
||||
row[0]
|
||||
for row in session.query(FileBlob.storage_key)
|
||||
.filter(
|
||||
FileBlob.tenant_id == scan.tenant_id,
|
||||
FileBlob.storage_backend == scan.storage_backend,
|
||||
FileBlob.storage_key.in_(keys),
|
||||
)
|
||||
.all()
|
||||
} if keys else set()
|
||||
for item in page.objects:
|
||||
scan.scanned_object_count += 1
|
||||
if item.key in referenced_keys:
|
||||
continue
|
||||
scan.orphan_object_count += 1
|
||||
_record_orphan_finding(
|
||||
session,
|
||||
scan,
|
||||
storage_key=item.key,
|
||||
size_bytes=item.size_bytes,
|
||||
)
|
||||
scan.object_cursor = page.next_cursor
|
||||
if page.next_cursor is None:
|
||||
scan.phase = "completed"
|
||||
scan.status = "completed"
|
||||
scan.completed_at = utcnow()
|
||||
|
||||
|
||||
def _record_blob_finding(
|
||||
session: Session,
|
||||
scan: FileIntegrityScan,
|
||||
blob: FileBlob,
|
||||
inspection: BlobInspection,
|
||||
) -> FileIntegrityFinding:
|
||||
finding = (
|
||||
session.query(FileIntegrityFinding)
|
||||
.filter(
|
||||
FileIntegrityFinding.scan_id == scan.id,
|
||||
FileIntegrityFinding.blob_id == blob.id,
|
||||
FileIntegrityFinding.kind == inspection.kind,
|
||||
)
|
||||
.one_or_none()
|
||||
)
|
||||
if finding is None:
|
||||
finding = FileIntegrityFinding(
|
||||
scan_id=scan.id,
|
||||
tenant_id=scan.tenant_id,
|
||||
kind=inspection.kind,
|
||||
blob_id=blob.id,
|
||||
storage_key=blob.storage_key,
|
||||
expected_size_bytes=blob.storage_size_bytes if blob.storage_size_bytes is not None else blob.size_bytes,
|
||||
expected_checksum_sha256=blob.storage_checksum_sha256 if blob.storage_checksum_sha256 is not None else blob.checksum_sha256,
|
||||
)
|
||||
else:
|
||||
finding.revision += 1
|
||||
_update_finding_from_inspection(finding, inspection)
|
||||
session.add(finding)
|
||||
return finding
|
||||
|
||||
|
||||
def _record_orphan_finding(
|
||||
session: Session,
|
||||
scan: FileIntegrityScan,
|
||||
*,
|
||||
storage_key: str,
|
||||
size_bytes: int,
|
||||
) -> FileIntegrityFinding:
|
||||
finding = (
|
||||
session.query(FileIntegrityFinding)
|
||||
.filter(
|
||||
FileIntegrityFinding.scan_id == scan.id,
|
||||
FileIntegrityFinding.kind == "orphan_object",
|
||||
FileIntegrityFinding.storage_key == storage_key,
|
||||
)
|
||||
.one_or_none()
|
||||
)
|
||||
if finding is None:
|
||||
finding = FileIntegrityFinding(
|
||||
scan_id=scan.id,
|
||||
tenant_id=scan.tenant_id,
|
||||
kind="orphan_object",
|
||||
storage_key=storage_key,
|
||||
observed_size_bytes=size_bytes,
|
||||
)
|
||||
session.add(finding)
|
||||
return finding
|
||||
|
||||
|
||||
def _resolve_scan_blob_findings(
|
||||
session: Session,
|
||||
scan: FileIntegrityScan,
|
||||
blob: FileBlob,
|
||||
) -> None:
|
||||
for finding in (
|
||||
session.query(FileIntegrityFinding)
|
||||
.filter(
|
||||
FileIntegrityFinding.scan_id == scan.id,
|
||||
FileIntegrityFinding.blob_id == blob.id,
|
||||
FileIntegrityFinding.state == "open",
|
||||
)
|
||||
.all()
|
||||
):
|
||||
finding.state = "resolved"
|
||||
finding.resolved_at = utcnow()
|
||||
finding.revision += 1
|
||||
session.add(finding)
|
||||
|
||||
|
||||
def _update_finding_from_inspection(
|
||||
finding: FileIntegrityFinding,
|
||||
inspection: BlobInspection,
|
||||
) -> None:
|
||||
finding.observed_size_bytes = inspection.observed_size_bytes
|
||||
finding.observed_checksum_sha256 = inspection.observed_checksum_sha256
|
||||
|
||||
|
||||
__all__ = [
|
||||
"BlobInspection",
|
||||
"IntegrityActionResult",
|
||||
"QUARANTINED_BLOB_STATUSES",
|
||||
"apply_blob_inspection",
|
||||
"cleanup_orphan_finding",
|
||||
"create_integrity_scan",
|
||||
"ensure_blob_is_readable",
|
||||
"inspect_blob",
|
||||
"mark_integrity_scan_failed",
|
||||
"read_verified_blob_bytes",
|
||||
"recheck_integrity_finding",
|
||||
"run_integrity_scan_batch",
|
||||
"storage_prefix_for_tenant",
|
||||
]
|
||||
@@ -0,0 +1,796 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
import hashlib
|
||||
import json
|
||||
from typing import Callable, Iterable
|
||||
|
||||
from sqlalchemy import func, inspect, or_
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from govoplan_core.core.recovery import (
|
||||
RecoveryGuaranteeError,
|
||||
RecoveryMode,
|
||||
RecoveryPlan,
|
||||
RecoveryStatus,
|
||||
)
|
||||
from govoplan_core.core.recovery_runtime import (
|
||||
RecoveryOperationBusy,
|
||||
RecoveryOperationStateConflict,
|
||||
begin_durable_recovery_operation,
|
||||
)
|
||||
from govoplan_core.core.runtime_coordination import process_runtime_identity
|
||||
from govoplan_core.db.session import get_database
|
||||
from govoplan_files.backend.db.models import (
|
||||
CampaignAttachmentUse,
|
||||
FileAsset,
|
||||
FileBlob,
|
||||
FileConnectorSpace,
|
||||
FileFolder,
|
||||
FileFormEvidenceGrant,
|
||||
FileShare,
|
||||
FileVersion,
|
||||
)
|
||||
from govoplan_files.backend.storage.access import ensure_owner_access
|
||||
from govoplan_files.backend.storage.backends import (
|
||||
StorageBackendError,
|
||||
get_storage_backend,
|
||||
)
|
||||
from govoplan_files.backend.storage.common import FileStorageError, utcnow
|
||||
from govoplan_files.backend.storage.connector_spaces import connector_space_owner_id
|
||||
from govoplan_files.backend.storage.paths import normalize_folder
|
||||
from govoplan_files.backend.storage.share_state import effective_file_share_clause
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PurgePreviewItem:
|
||||
file_id: str
|
||||
filename: str
|
||||
lifecycle_revision: int
|
||||
deleted_at: datetime | None
|
||||
retained_until: datetime | None
|
||||
legal_hold: bool
|
||||
blockers: tuple[str, ...]
|
||||
blob_ids: tuple[str, ...]
|
||||
|
||||
def digest_payload(self) -> dict[str, object]:
|
||||
return {
|
||||
"file_id": self.file_id,
|
||||
"filename": self.filename,
|
||||
"lifecycle_revision": self.lifecycle_revision,
|
||||
"deleted_at": _iso(self.deleted_at),
|
||||
"retained_until": _iso(self.retained_until),
|
||||
"legal_hold": self.legal_hold,
|
||||
"blockers": list(self.blockers),
|
||||
"blob_ids": list(self.blob_ids),
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PurgePreview:
|
||||
preview_sha256: str
|
||||
items: tuple[PurgePreviewItem, ...]
|
||||
|
||||
@property
|
||||
def eligible(self) -> bool:
|
||||
return all(not item.blockers for item in self.items)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PurgeResult:
|
||||
recovery_operation_id: str
|
||||
status: str
|
||||
replayed: bool
|
||||
purged_files: int
|
||||
released_blobs: int
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class BlobGcResult:
|
||||
inspected_blobs: int
|
||||
deleted_blobs: int
|
||||
unresolved_operation_ids: tuple[str, ...]
|
||||
|
||||
|
||||
def set_asset_lifecycle(
|
||||
session: Session,
|
||||
asset: FileAsset,
|
||||
*,
|
||||
retained_until: datetime | None,
|
||||
legal_hold: bool,
|
||||
reason: str,
|
||||
expected_revision: int,
|
||||
) -> FileAsset:
|
||||
if asset.lifecycle_revision != expected_revision:
|
||||
raise FileStorageError("File lifecycle settings changed; reload before saving")
|
||||
asset.retained_until = retained_until
|
||||
asset.legal_hold = bool(legal_hold)
|
||||
asset.lifecycle_reason = reason.strip()
|
||||
asset.lifecycle_revision += 1
|
||||
session.add(asset)
|
||||
session.flush()
|
||||
return asset
|
||||
|
||||
|
||||
def get_asset_for_lifecycle(
|
||||
session: Session,
|
||||
*,
|
||||
tenant_id: str,
|
||||
user_id: str,
|
||||
asset_id: str,
|
||||
is_admin: bool,
|
||||
) -> FileAsset:
|
||||
asset = session.get(FileAsset, asset_id)
|
||||
if asset is None or asset.tenant_id != tenant_id:
|
||||
raise FileStorageError("File not found")
|
||||
ensure_owner_access(
|
||||
session,
|
||||
tenant_id=tenant_id,
|
||||
owner_type=asset.owner_type,
|
||||
owner_id=_asset_owner_id(asset),
|
||||
user_id=user_id,
|
||||
is_admin=is_admin,
|
||||
)
|
||||
return asset
|
||||
|
||||
|
||||
def restore_asset(session: Session, asset: FileAsset) -> bool:
|
||||
if asset.deleted_at is None:
|
||||
return False
|
||||
collision = _asset_owner_query(session, asset).filter(
|
||||
FileAsset.id != asset.id,
|
||||
FileAsset.display_path == asset.display_path,
|
||||
FileAsset.deleted_at.is_(None),
|
||||
).first()
|
||||
if collision is not None:
|
||||
raise FileStorageError(
|
||||
"The file cannot be restored because its path is already in use"
|
||||
)
|
||||
asset.deleted_at = None
|
||||
asset.lifecycle_revision += 1
|
||||
session.add(asset)
|
||||
session.flush()
|
||||
return True
|
||||
|
||||
|
||||
def restore_folder(
|
||||
session: Session,
|
||||
*,
|
||||
tenant_id: str,
|
||||
owner_type: str,
|
||||
owner_id: str,
|
||||
user_id: str,
|
||||
path: str,
|
||||
recursive: bool,
|
||||
is_admin: bool,
|
||||
) -> tuple[int, int]:
|
||||
ensure_owner_access(
|
||||
session,
|
||||
tenant_id=tenant_id,
|
||||
owner_type=owner_type,
|
||||
owner_id=owner_id,
|
||||
user_id=user_id,
|
||||
is_admin=is_admin,
|
||||
)
|
||||
normalized = normalize_folder(path)
|
||||
if not normalized:
|
||||
raise FileStorageError("Folder path is required")
|
||||
prefix = f"{normalized}/"
|
||||
folders = _folder_owner_query(
|
||||
session, tenant_id=tenant_id, owner_type=owner_type, owner_id=owner_id
|
||||
).filter(FileFolder.deleted_at.is_not(None))
|
||||
assets = _asset_owner_query_by_values(
|
||||
session, tenant_id=tenant_id, owner_type=owner_type, owner_id=owner_id
|
||||
).filter(FileAsset.deleted_at.is_not(None))
|
||||
if recursive:
|
||||
folders = folders.filter(
|
||||
(FileFolder.path == normalized) | FileFolder.path.like(f"{prefix}%")
|
||||
)
|
||||
assets = assets.filter(FileAsset.display_path.like(f"{prefix}%"))
|
||||
else:
|
||||
folders = folders.filter(FileFolder.path == normalized)
|
||||
assets = assets.filter(False)
|
||||
folder_rows = folders.order_by(FileFolder.path.asc()).all()
|
||||
asset_rows = assets.order_by(FileAsset.display_path.asc()).all()
|
||||
if not folder_rows:
|
||||
raise FileStorageError("Deleted folder not found")
|
||||
for folder in folder_rows:
|
||||
collision = _folder_owner_query(
|
||||
session,
|
||||
tenant_id=tenant_id,
|
||||
owner_type=owner_type,
|
||||
owner_id=owner_id,
|
||||
).filter(
|
||||
FileFolder.id != folder.id,
|
||||
FileFolder.path == folder.path,
|
||||
FileFolder.deleted_at.is_(None),
|
||||
).first()
|
||||
if collision is not None:
|
||||
raise FileStorageError(
|
||||
f"The folder cannot be restored because its path is in use: {folder.path}"
|
||||
)
|
||||
for asset in asset_rows:
|
||||
collision = _asset_owner_query(session, asset).filter(
|
||||
FileAsset.id != asset.id,
|
||||
FileAsset.display_path == asset.display_path,
|
||||
FileAsset.deleted_at.is_(None),
|
||||
).first()
|
||||
if collision is not None:
|
||||
raise FileStorageError(
|
||||
f"A file cannot be restored because its path is in use: {asset.display_path}"
|
||||
)
|
||||
for folder in folder_rows:
|
||||
folder.deleted_at = None
|
||||
session.add(folder)
|
||||
for asset in asset_rows:
|
||||
asset.deleted_at = None
|
||||
asset.lifecycle_revision += 1
|
||||
session.add(asset)
|
||||
session.flush()
|
||||
return len(folder_rows), len(asset_rows)
|
||||
|
||||
|
||||
def restore_connector_space(
|
||||
session: Session,
|
||||
*,
|
||||
tenant_id: str,
|
||||
user_id: str,
|
||||
space_id: str,
|
||||
is_admin: bool,
|
||||
) -> FileConnectorSpace:
|
||||
space = session.get(FileConnectorSpace, space_id)
|
||||
if space is None or space.tenant_id != tenant_id or space.deleted_at is None:
|
||||
raise FileStorageError("Deleted connector space not found")
|
||||
ensure_owner_access(
|
||||
session,
|
||||
tenant_id=tenant_id,
|
||||
owner_type=space.owner_type,
|
||||
owner_id=connector_space_owner_id(space),
|
||||
user_id=user_id,
|
||||
is_admin=is_admin,
|
||||
)
|
||||
collision = _connector_space_owner_query(session, space).filter(
|
||||
FileConnectorSpace.id != space.id,
|
||||
FileConnectorSpace.label == space.label,
|
||||
FileConnectorSpace.deleted_at.is_(None),
|
||||
).first()
|
||||
if collision is not None:
|
||||
raise FileStorageError(
|
||||
"The connector space cannot be restored because its label is in use"
|
||||
)
|
||||
space.deleted_at = None
|
||||
space.is_active = True
|
||||
session.add(space)
|
||||
session.flush()
|
||||
return space
|
||||
|
||||
|
||||
def preview_asset_purge(
|
||||
session: Session, *, tenant_id: str, file_ids: Iterable[str]
|
||||
) -> PurgePreview:
|
||||
normalized_ids = sorted(set(str(value).strip() for value in file_ids if str(value).strip()))
|
||||
if not normalized_ids or len(normalized_ids) > 100:
|
||||
raise FileStorageError("Purge preview must contain between 1 and 100 files")
|
||||
assets = (
|
||||
session.query(FileAsset)
|
||||
.filter(FileAsset.tenant_id == tenant_id, FileAsset.id.in_(normalized_ids))
|
||||
.order_by(FileAsset.id.asc())
|
||||
.all()
|
||||
)
|
||||
by_id = {asset.id: asset for asset in assets}
|
||||
items: list[PurgePreviewItem] = []
|
||||
for file_id in normalized_ids:
|
||||
asset = by_id.get(file_id)
|
||||
if asset is None:
|
||||
items.append(
|
||||
PurgePreviewItem(
|
||||
file_id=file_id,
|
||||
filename="",
|
||||
lifecycle_revision=0,
|
||||
deleted_at=None,
|
||||
retained_until=None,
|
||||
legal_hold=False,
|
||||
blockers=("not_found",),
|
||||
blob_ids=(),
|
||||
)
|
||||
)
|
||||
continue
|
||||
blockers = _asset_purge_blockers(session, asset)
|
||||
blob_ids = tuple(
|
||||
sorted(
|
||||
row[0]
|
||||
for row in session.query(FileVersion.blob_id)
|
||||
.filter(FileVersion.file_asset_id == asset.id)
|
||||
.distinct()
|
||||
.all()
|
||||
)
|
||||
)
|
||||
items.append(
|
||||
PurgePreviewItem(
|
||||
file_id=asset.id,
|
||||
filename=asset.filename,
|
||||
lifecycle_revision=asset.lifecycle_revision,
|
||||
deleted_at=asset.deleted_at,
|
||||
retained_until=asset.retained_until,
|
||||
legal_hold=asset.legal_hold,
|
||||
blockers=tuple(blockers),
|
||||
blob_ids=blob_ids,
|
||||
)
|
||||
)
|
||||
payload = [item.digest_payload() for item in items]
|
||||
digest = hashlib.sha256(
|
||||
json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||
).hexdigest()
|
||||
return PurgePreview(preview_sha256=digest, items=tuple(items))
|
||||
|
||||
|
||||
def execute_asset_purge(
|
||||
session: Session,
|
||||
*,
|
||||
tenant_id: str,
|
||||
file_ids: Iterable[str],
|
||||
preview_sha256: str,
|
||||
idempotency_key: str,
|
||||
approval_reference: str,
|
||||
before_commit: Callable[[list[FileAsset]], None] | None = None,
|
||||
) -> PurgeResult:
|
||||
normalized_ids = sorted(set(str(value).strip() for value in file_ids if str(value).strip()))
|
||||
if not normalized_ids or len(normalized_ids) > 100:
|
||||
raise FileStorageError("Purge must contain between 1 and 100 files")
|
||||
request = {
|
||||
"tenant_id": tenant_id,
|
||||
"file_ids": normalized_ids,
|
||||
"preview_sha256": preview_sha256,
|
||||
"approval_reference": approval_reference,
|
||||
}
|
||||
try:
|
||||
started = begin_durable_recovery_operation(
|
||||
get_database().SessionLocal,
|
||||
identity=process_runtime_identity(),
|
||||
module_id="files",
|
||||
operation_type="asset-hard-purge",
|
||||
idempotency_key=f"files-asset-purge:{idempotency_key}",
|
||||
request=request,
|
||||
recovery_plan=RecoveryPlan(
|
||||
mode=RecoveryMode.IRREVERSIBLE,
|
||||
preconditions=(
|
||||
"the actor holds the dedicated Files purge permission",
|
||||
"the signed-off preview still matches current lifecycle state",
|
||||
"every target is soft-deleted and free of retention or legal-hold blockers",
|
||||
),
|
||||
verification_steps=(
|
||||
"verify every target FileAsset and FileVersion row is absent",
|
||||
"recalculate retained FileBlob reference counts before later garbage collection",
|
||||
),
|
||||
approval_reference=approval_reference,
|
||||
),
|
||||
precondition_evidence={
|
||||
"preview_sha256": preview_sha256,
|
||||
"target_count": len(normalized_ids),
|
||||
},
|
||||
lease_resource_key=f"files:purge:{tenant_id}",
|
||||
lease_ttl_seconds=15 * 60,
|
||||
resource_type="file_asset_batch",
|
||||
resource_id=preview_sha256,
|
||||
metadata={"resources": ["postgresql"], "bounded_target_count": len(normalized_ids)},
|
||||
block_unresolved_resource=True,
|
||||
)
|
||||
except (RecoveryOperationBusy, RecoveryOperationStateConflict) as exc:
|
||||
raise FileStorageError(
|
||||
"Another purge or unresolved retry owns the tenant purge fence"
|
||||
) from exc
|
||||
except (RecoveryGuaranteeError, RuntimeError, ValueError) as exc:
|
||||
raise FileStorageError("The Files recovery ledger is unavailable; nothing was purged") from exc
|
||||
if started.replayed or started.operation is None:
|
||||
return PurgeResult(
|
||||
recovery_operation_id=started.operation_id,
|
||||
status=started.status,
|
||||
replayed=True,
|
||||
purged_files=0,
|
||||
released_blobs=0,
|
||||
)
|
||||
operation = started.operation
|
||||
try:
|
||||
preview = preview_asset_purge(session, tenant_id=tenant_id, file_ids=normalized_ids)
|
||||
if preview.preview_sha256 != preview_sha256:
|
||||
operation.reject(
|
||||
summary="The purge preview became stale before execution",
|
||||
evidence=_verified_evidence(
|
||||
{"preview_matches": False},
|
||||
expected_preview_sha256=preview_sha256,
|
||||
observed_preview_sha256=preview.preview_sha256,
|
||||
),
|
||||
)
|
||||
raise FileStorageError("The purge preview is stale; preview again")
|
||||
if not preview.eligible:
|
||||
operation.reject(
|
||||
summary="The purge was blocked by current lifecycle policy",
|
||||
evidence=_verified_evidence(
|
||||
{"lifecycle_policy_allows": False},
|
||||
blocked_files=[
|
||||
{"file_id": item.file_id, "blockers": list(item.blockers)}
|
||||
for item in preview.items
|
||||
if item.blockers
|
||||
]
|
||||
),
|
||||
)
|
||||
raise FileStorageError("One or more files are not eligible for purge")
|
||||
assets = (
|
||||
session.query(FileAsset)
|
||||
.filter(FileAsset.tenant_id == tenant_id, FileAsset.id.in_(normalized_ids))
|
||||
.with_for_update()
|
||||
.all()
|
||||
)
|
||||
blob_ids = sorted({blob_id for item in preview.items for blob_id in item.blob_ids})
|
||||
if before_commit is not None:
|
||||
before_commit(assets)
|
||||
session.query(FileShare).filter(
|
||||
FileShare.file_asset_id.in_(normalized_ids)
|
||||
).delete(synchronize_session=False)
|
||||
session.query(FileVersion).filter(
|
||||
FileVersion.file_asset_id.in_(normalized_ids)
|
||||
).delete(synchronize_session=False)
|
||||
for asset in assets:
|
||||
session.delete(asset)
|
||||
session.flush()
|
||||
released_blobs = 0
|
||||
for blob_id in blob_ids:
|
||||
references = int(
|
||||
session.query(func.count(FileVersion.id))
|
||||
.filter(FileVersion.blob_id == blob_id)
|
||||
.scalar()
|
||||
or 0
|
||||
)
|
||||
blob = session.get(FileBlob, blob_id)
|
||||
if blob is not None:
|
||||
blob.ref_count = references
|
||||
session.add(blob)
|
||||
if references == 0:
|
||||
released_blobs += 1
|
||||
operation.commit_verified_success(
|
||||
session,
|
||||
evidence=_verified_evidence(
|
||||
{
|
||||
"assets_absent": True,
|
||||
"versions_absent": True,
|
||||
"blob_reference_counts_recalculated": True,
|
||||
},
|
||||
purged_file_ids=normalized_ids,
|
||||
purged_files=len(assets),
|
||||
released_blobs=released_blobs,
|
||||
preview_sha256=preview_sha256,
|
||||
),
|
||||
)
|
||||
return PurgeResult(
|
||||
recovery_operation_id=started.operation_id,
|
||||
status=RecoveryStatus.SUCCEEDED.value,
|
||||
replayed=False,
|
||||
purged_files=len(assets),
|
||||
released_blobs=released_blobs,
|
||||
)
|
||||
except FileStorageError:
|
||||
session.rollback()
|
||||
if not operation.closed:
|
||||
operation.release_unresolved()
|
||||
raise
|
||||
except Exception as exc:
|
||||
session.rollback()
|
||||
if not operation.closed:
|
||||
operation.reject(
|
||||
summary="The purge failed before its database transaction committed",
|
||||
evidence=_verified_evidence(
|
||||
{"database_transaction_committed": False},
|
||||
exception_type=type(exc).__name__,
|
||||
),
|
||||
)
|
||||
raise
|
||||
|
||||
|
||||
def garbage_collect_unreferenced_blobs(
|
||||
session: Session,
|
||||
*,
|
||||
tenant_id: str,
|
||||
limit: int,
|
||||
approval_reference: str,
|
||||
) -> BlobGcResult:
|
||||
candidate_rows = (
|
||||
session.query(FileBlob)
|
||||
.filter(FileBlob.tenant_id == tenant_id)
|
||||
.filter(
|
||||
or_(FileBlob.retained_until.is_(None), FileBlob.retained_until <= utcnow())
|
||||
)
|
||||
.filter(
|
||||
~session.query(FileVersion.id)
|
||||
.filter(FileVersion.blob_id == FileBlob.id)
|
||||
.exists()
|
||||
)
|
||||
.order_by(FileBlob.created_at.asc(), FileBlob.id.asc())
|
||||
.limit(limit)
|
||||
.all()
|
||||
)
|
||||
candidates = tuple(
|
||||
(row.id, row.storage_key) for row in candidate_rows
|
||||
)
|
||||
# Close the read snapshot before the independent recovery transaction takes
|
||||
# its durable SQLite/PostgreSQL write lock.
|
||||
session.commit()
|
||||
deleted = 0
|
||||
unresolved: list[str] = []
|
||||
backend = get_storage_backend()
|
||||
for candidate_id, storage_key in candidates:
|
||||
key_digest = hashlib.sha256(storage_key.encode("utf-8")).hexdigest()
|
||||
request = {
|
||||
"tenant_id": tenant_id,
|
||||
"blob_id": candidate_id,
|
||||
"storage_key_sha256": key_digest,
|
||||
"approval_reference": approval_reference,
|
||||
}
|
||||
try:
|
||||
started = begin_durable_recovery_operation(
|
||||
get_database().SessionLocal,
|
||||
identity=process_runtime_identity(),
|
||||
module_id="files",
|
||||
operation_type="blob-garbage-collection",
|
||||
idempotency_key=(
|
||||
f"files-blob-gc:{candidate_id}:{key_digest[:12]}:"
|
||||
f"{hashlib.sha256(approval_reference.encode('utf-8')).hexdigest()[:12]}"
|
||||
),
|
||||
request=request,
|
||||
recovery_plan=RecoveryPlan(
|
||||
mode=RecoveryMode.FORWARD_RECOVERY,
|
||||
preconditions=(
|
||||
"the caller holds dedicated Files purge authority",
|
||||
"a fresh database check shows no FileVersion reference",
|
||||
),
|
||||
forward_recovery_steps=(
|
||||
"verify the object is absent",
|
||||
"delete the unreferenced FileBlob row after absence is proven",
|
||||
),
|
||||
verification_steps=(
|
||||
"recheck FileVersion references while holding the blob fence",
|
||||
"probe the exact managed storage key after deletion",
|
||||
),
|
||||
approval_reference=approval_reference,
|
||||
),
|
||||
precondition_evidence={
|
||||
"blob_id": candidate_id,
|
||||
"storage_key_sha256": key_digest,
|
||||
"observed_reference_count": 0,
|
||||
},
|
||||
lease_resource_key=f"files:blob:{tenant_id}:{candidate_id}",
|
||||
lease_ttl_seconds=15 * 60,
|
||||
resource_type="file_blob",
|
||||
resource_id=candidate_id,
|
||||
metadata={"resources": ["postgresql", "object-storage"], "storage_backend": backend.name},
|
||||
block_unresolved_resource=True,
|
||||
)
|
||||
except (RecoveryOperationBusy, RecoveryOperationStateConflict):
|
||||
continue
|
||||
except (RecoveryGuaranteeError, RuntimeError, ValueError) as exc:
|
||||
raise FileStorageError(
|
||||
"The Files recovery ledger is unavailable; no blob bytes were deleted"
|
||||
) from exc
|
||||
if started.replayed or started.operation is None:
|
||||
continue
|
||||
operation = started.operation
|
||||
try:
|
||||
blob = (
|
||||
session.query(FileBlob)
|
||||
.filter(FileBlob.id == candidate_id, FileBlob.tenant_id == tenant_id)
|
||||
.with_for_update()
|
||||
.one_or_none()
|
||||
)
|
||||
references = int(
|
||||
session.query(func.count(FileVersion.id))
|
||||
.filter(FileVersion.blob_id == candidate_id)
|
||||
.scalar()
|
||||
or 0
|
||||
)
|
||||
blob_retained = bool(
|
||||
blob is not None
|
||||
and blob.retained_until is not None
|
||||
and _as_utc(blob.retained_until) > utcnow()
|
||||
)
|
||||
except Exception as exc:
|
||||
session.rollback()
|
||||
if not operation.closed:
|
||||
operation.reject(
|
||||
summary="Blob eligibility could not be rechecked before deletion",
|
||||
evidence=_verified_evidence(
|
||||
{"external_effect_started": False},
|
||||
exception_type=type(exc).__name__,
|
||||
),
|
||||
)
|
||||
raise FileStorageError(
|
||||
"Blob eligibility could not be rechecked; no bytes were deleted"
|
||||
) from exc
|
||||
if blob is None or references or blob_retained:
|
||||
operation.reject(
|
||||
summary="Blob garbage collection was no longer eligible",
|
||||
evidence=_verified_evidence(
|
||||
{
|
||||
"blob_present": blob is not None,
|
||||
"reference_count_zero": references == 0,
|
||||
"blob_retention_expired": not blob_retained,
|
||||
},
|
||||
reference_count=references,
|
||||
),
|
||||
)
|
||||
session.rollback()
|
||||
continue
|
||||
try:
|
||||
backend.delete(blob.storage_key)
|
||||
object_present = backend.exists(blob.storage_key)
|
||||
except (StorageBackendError, OSError) as exc:
|
||||
try:
|
||||
object_present = backend.exists(blob.storage_key)
|
||||
except (StorageBackendError, OSError):
|
||||
object_present = None
|
||||
session.rollback()
|
||||
if object_present is True:
|
||||
operation.fail(
|
||||
summary="The unreferenced blob object was retained",
|
||||
evidence={"object_present": True, "exception_type": type(exc).__name__},
|
||||
)
|
||||
else:
|
||||
operation.unresolved(
|
||||
status=RecoveryStatus.OUTCOME_UNKNOWN,
|
||||
summary="The blob deletion outcome could not be verified",
|
||||
evidence={"object_present": object_present, "exception_type": type(exc).__name__},
|
||||
failure_summary="Reconcile the exact blob key before retrying garbage collection",
|
||||
)
|
||||
unresolved.append(started.operation_id)
|
||||
continue
|
||||
if object_present:
|
||||
session.rollback()
|
||||
operation.fail(
|
||||
summary="The object store did not delete the unreferenced blob",
|
||||
evidence={"object_present": True},
|
||||
)
|
||||
continue
|
||||
try:
|
||||
session.delete(blob)
|
||||
session.flush()
|
||||
operation.commit_verified_success(
|
||||
session,
|
||||
evidence=_verified_evidence(
|
||||
{
|
||||
"object_absent": True,
|
||||
"database_blob_absent": True,
|
||||
"reference_count_zero": True,
|
||||
},
|
||||
blob_id=candidate_id,
|
||||
storage_key_sha256=key_digest,
|
||||
object_present=False,
|
||||
database_blob_present=False,
|
||||
reference_count=0,
|
||||
),
|
||||
)
|
||||
deleted += 1
|
||||
except Exception as exc:
|
||||
session.rollback()
|
||||
if not operation.closed:
|
||||
operation.unresolved(
|
||||
status=RecoveryStatus.RECOVERY_REQUIRED,
|
||||
summary="The object is absent but blob metadata still requires forward recovery",
|
||||
evidence={"object_present": False, "exception_type": type(exc).__name__},
|
||||
failure_summary="Delete the FileBlob row only after rechecking all FileVersion references",
|
||||
)
|
||||
unresolved.append(started.operation_id)
|
||||
return BlobGcResult(
|
||||
inspected_blobs=len(candidates),
|
||||
deleted_blobs=deleted,
|
||||
unresolved_operation_ids=tuple(unresolved),
|
||||
)
|
||||
|
||||
|
||||
def _asset_purge_blockers(session: Session, asset: FileAsset) -> list[str]:
|
||||
blockers: list[str] = []
|
||||
if asset.deleted_at is None:
|
||||
blockers.append("not_soft_deleted")
|
||||
if asset.legal_hold:
|
||||
blockers.append("legal_hold")
|
||||
if asset.retained_until is not None and _as_utc(asset.retained_until) > utcnow():
|
||||
blockers.append("retention_active")
|
||||
if _table_exists(session, CampaignAttachmentUse.__tablename__) and session.query(
|
||||
CampaignAttachmentUse.id
|
||||
).filter(CampaignAttachmentUse.file_asset_id == asset.id).first() is not None:
|
||||
blockers.append("campaign_evidence")
|
||||
if _table_exists(session, FileFormEvidenceGrant.__tablename__) and session.query(
|
||||
FileFormEvidenceGrant.id
|
||||
).filter(FileFormEvidenceGrant.file_asset_id == asset.id).first() is not None:
|
||||
blockers.append("form_evidence")
|
||||
if session.query(FileShare.id).filter(
|
||||
FileShare.file_asset_id == asset.id,
|
||||
effective_file_share_clause(),
|
||||
).first() is not None:
|
||||
blockers.append("active_share")
|
||||
return blockers
|
||||
|
||||
|
||||
def _asset_owner_id(asset: FileAsset) -> str:
|
||||
owner_id = asset.owner_user_id if asset.owner_type == "user" else asset.owner_group_id
|
||||
if not owner_id:
|
||||
raise FileStorageError("File has no valid owner")
|
||||
return owner_id
|
||||
|
||||
|
||||
def _asset_owner_query(session: Session, asset: FileAsset):
|
||||
return _asset_owner_query_by_values(
|
||||
session,
|
||||
tenant_id=asset.tenant_id,
|
||||
owner_type=asset.owner_type,
|
||||
owner_id=_asset_owner_id(asset),
|
||||
)
|
||||
|
||||
|
||||
def _asset_owner_query_by_values(
|
||||
session: Session, *, tenant_id: str, owner_type: str, owner_id: str
|
||||
):
|
||||
query = session.query(FileAsset).filter(
|
||||
FileAsset.tenant_id == tenant_id, FileAsset.owner_type == owner_type
|
||||
)
|
||||
if owner_type == "user":
|
||||
return query.filter(FileAsset.owner_user_id == owner_id)
|
||||
if owner_type == "group":
|
||||
return query.filter(FileAsset.owner_group_id == owner_id)
|
||||
raise FileStorageError("Files must be owned by a user or group")
|
||||
|
||||
|
||||
def _folder_owner_query(
|
||||
session: Session, *, tenant_id: str, owner_type: str, owner_id: str
|
||||
):
|
||||
query = session.query(FileFolder).filter(
|
||||
FileFolder.tenant_id == tenant_id, FileFolder.owner_type == owner_type
|
||||
)
|
||||
if owner_type == "user":
|
||||
return query.filter(FileFolder.owner_user_id == owner_id)
|
||||
if owner_type == "group":
|
||||
return query.filter(FileFolder.owner_group_id == owner_id)
|
||||
raise FileStorageError("Folders must be owned by a user or group")
|
||||
|
||||
|
||||
def _connector_space_owner_query(session: Session, space: FileConnectorSpace):
|
||||
query = session.query(FileConnectorSpace).filter(
|
||||
FileConnectorSpace.tenant_id == space.tenant_id,
|
||||
FileConnectorSpace.owner_type == space.owner_type,
|
||||
)
|
||||
owner_id = connector_space_owner_id(space)
|
||||
if space.owner_type == "user":
|
||||
return query.filter(FileConnectorSpace.owner_user_id == owner_id)
|
||||
return query.filter(FileConnectorSpace.owner_group_id == owner_id)
|
||||
|
||||
|
||||
def _as_utc(value: datetime) -> datetime:
|
||||
return value.replace(tzinfo=UTC) if value.tzinfo is None else value.astimezone(UTC)
|
||||
|
||||
|
||||
def _iso(value: datetime | None) -> str | None:
|
||||
return _as_utc(value).isoformat() if value is not None else None
|
||||
|
||||
|
||||
def _table_exists(session: Session, table_name: str) -> bool:
|
||||
return bool(session.bind is not None and inspect(session.bind).has_table(table_name))
|
||||
|
||||
|
||||
def _verified_evidence(
|
||||
checks: dict[str, object], **details: object
|
||||
) -> dict[str, object]:
|
||||
return {"verified": True, "checks": checks, **details}
|
||||
|
||||
|
||||
__all__ = [
|
||||
"BlobGcResult",
|
||||
"PurgePreview",
|
||||
"PurgePreviewItem",
|
||||
"PurgeResult",
|
||||
"execute_asset_purge",
|
||||
"garbage_collect_unreferenced_blobs",
|
||||
"get_asset_for_lifecycle",
|
||||
"preview_asset_purge",
|
||||
"restore_asset",
|
||||
"restore_connector_space",
|
||||
"restore_folder",
|
||||
"set_asset_lifecycle",
|
||||
]
|
||||
@@ -0,0 +1,931 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
import hashlib
|
||||
from threading import Lock
|
||||
from typing import Protocol
|
||||
|
||||
from sqlalchemy import event, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from govoplan_core.core.recovery import (
|
||||
RecoveryGuaranteeError,
|
||||
RecoveryMode,
|
||||
RecoveryOperation,
|
||||
RecoveryPlan,
|
||||
RecoveryStatus,
|
||||
plan_recovery_operation,
|
||||
prepare_recovery_operation,
|
||||
start_recovery_operation,
|
||||
verify_recovery_evidence_chain,
|
||||
)
|
||||
from govoplan_core.core.recovery_runtime import (
|
||||
DurableRecoveryOperation,
|
||||
DurableRecoveryStart,
|
||||
RecoveryOperationBusy,
|
||||
RecoveryOperationStateConflict,
|
||||
begin_durable_recovery_operation,
|
||||
)
|
||||
from govoplan_core.core.runtime_coordination import (
|
||||
RuntimeIdentity,
|
||||
acquire_lease,
|
||||
process_runtime_identity,
|
||||
release_lease,
|
||||
)
|
||||
from govoplan_core.db.session import get_database
|
||||
from govoplan_files.backend.db.models import (
|
||||
FileBlob,
|
||||
FileIntegrityFinding,
|
||||
FileIntegrityScan,
|
||||
)
|
||||
from govoplan_files.backend.storage.backends import (
|
||||
StorageBackend,
|
||||
StorageBackendError,
|
||||
StorageObjectMissing,
|
||||
)
|
||||
from govoplan_files.backend.storage.common import FileStorageError
|
||||
|
||||
|
||||
_PENDING_EFFECTS_KEY = "govoplan_files_pending_recovery_effects"
|
||||
_HOOKS_INSTALLED_KEY = "govoplan_files_recovery_hooks_installed"
|
||||
_ROLLBACK_ERRORS_KEY = "govoplan_files_recovery_rollback_errors"
|
||||
_SQLITE_FENCES: set[str] = set()
|
||||
_SQLITE_FENCES_LOCK = Lock()
|
||||
|
||||
|
||||
class _PendingEffect(Protocol):
|
||||
def settle(self, *, committed: bool) -> None: ...
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class PendingBlobWrite:
|
||||
operation: DurableRecoveryOperation
|
||||
backend: StorageBackend
|
||||
tenant_id: str
|
||||
blob_id: str
|
||||
storage_key: str
|
||||
semantic_checksum_sha256: str
|
||||
semantic_size_bytes: int
|
||||
protection_discriminator: str
|
||||
created_new: bool
|
||||
expected_storage_checksum_sha256: str | None = None
|
||||
expected_storage_size_bytes: int | None = None
|
||||
expected_envelope_id: str | None = None
|
||||
sqlite_fence_key: str | None = None
|
||||
restart_after_rollback: Callable[[], DurableRecoveryOperation] | None = None
|
||||
|
||||
def prepare_stored_bytes(
|
||||
self,
|
||||
data: bytes,
|
||||
*,
|
||||
envelope_id: str | None,
|
||||
) -> None:
|
||||
"""Retain process-local verification evidence without recording content."""
|
||||
|
||||
self.expected_storage_checksum_sha256 = hashlib.sha256(data).hexdigest()
|
||||
self.expected_storage_size_bytes = len(data)
|
||||
self.expected_envelope_id = envelope_id
|
||||
|
||||
def settle(self, *, committed: bool) -> None:
|
||||
if not committed and self.restart_after_rollback is not None:
|
||||
try:
|
||||
self.operation = self.restart_after_rollback()
|
||||
except Exception:
|
||||
# SQLite cannot make the pre-effect ledger row independent of
|
||||
# the caller transaction. If reconstructing the evidence row
|
||||
# also fails, still avoid leaving a known new orphan behind.
|
||||
if self.created_new:
|
||||
try:
|
||||
self.backend.delete(self.storage_key)
|
||||
except (StorageBackendError, OSError):
|
||||
pass
|
||||
raise
|
||||
evidence = _blob_write_evidence(self)
|
||||
if _blob_write_complete(evidence):
|
||||
self.operation.succeed(evidence=evidence)
|
||||
return
|
||||
|
||||
if self.created_new and evidence.get("database_blob_present") is False:
|
||||
self._settle_unreferenced_new_object(evidence)
|
||||
return
|
||||
|
||||
if not self.created_new and _object_matches(evidence):
|
||||
if _forward_complete_blob_repair(self):
|
||||
completed = _blob_write_evidence(self)
|
||||
if _blob_write_complete(completed):
|
||||
self.operation.succeed(evidence=completed)
|
||||
return
|
||||
|
||||
if evidence.get("database_blob_present") is True and not _object_matches(
|
||||
evidence
|
||||
):
|
||||
_quarantine_blob_after_failed_verification(self, evidence)
|
||||
evidence = _blob_write_evidence(self)
|
||||
|
||||
status = (
|
||||
RecoveryStatus.OUTCOME_UNKNOWN
|
||||
if not evidence.get("verified")
|
||||
else RecoveryStatus.RECOVERY_REQUIRED
|
||||
)
|
||||
transaction = "committed" if committed else "rolled back"
|
||||
self.operation.unresolved(
|
||||
status=status,
|
||||
summary=f"Managed Files blob write remained unresolved after the database transaction {transaction}",
|
||||
evidence=evidence,
|
||||
failure_summary=(
|
||||
"The managed object and Files blob metadata require reconciliation"
|
||||
),
|
||||
)
|
||||
|
||||
def _settle_unreferenced_new_object(self, evidence: dict[str, object]) -> None:
|
||||
object_present = evidence.get("object_present")
|
||||
if object_present is False:
|
||||
self.operation.reject(
|
||||
summary="The Files blob write left no durable object or database row",
|
||||
evidence=evidence,
|
||||
)
|
||||
return
|
||||
if object_present is not True:
|
||||
self.operation.unresolved(
|
||||
status=RecoveryStatus.OUTCOME_UNKNOWN,
|
||||
summary="The unreferenced Files object could not be probed",
|
||||
evidence=evidence,
|
||||
failure_summary="Object storage availability prevented upload compensation",
|
||||
)
|
||||
return
|
||||
try:
|
||||
self.backend.delete(self.storage_key)
|
||||
except (StorageBackendError, OSError):
|
||||
self.operation.unresolved(
|
||||
status=RecoveryStatus.RECOVERY_REQUIRED,
|
||||
summary="An unreferenced Files object could not be compensated",
|
||||
evidence=evidence,
|
||||
failure_summary="Delete the unreferenced managed object after verifying that no FileBlob references it",
|
||||
)
|
||||
return
|
||||
recovered = _blob_write_evidence(self)
|
||||
if (
|
||||
recovered.get("verified") is True
|
||||
and recovered.get("database_blob_present") is False
|
||||
and recovered.get("object_present") is False
|
||||
):
|
||||
self.operation.compensate(
|
||||
failure_summary="The Files database transaction did not retain the new blob",
|
||||
failure_evidence=evidence,
|
||||
recovery_evidence=recovered,
|
||||
)
|
||||
return
|
||||
self.operation.unresolved(
|
||||
status=RecoveryStatus.RECOVERY_REQUIRED,
|
||||
summary="Files upload compensation could not be verified",
|
||||
evidence=recovered,
|
||||
failure_summary="The unreferenced managed object requires operator reconciliation",
|
||||
)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class PendingOrphanCleanup:
|
||||
operation: DurableRecoveryOperation
|
||||
backend: StorageBackend
|
||||
finding_id: str
|
||||
tenant_id: str
|
||||
storage_key: str
|
||||
resolved_by_user_id: str
|
||||
|
||||
def settle(self, *, committed: bool) -> None:
|
||||
del committed
|
||||
evidence = _orphan_cleanup_evidence(self)
|
||||
if _orphan_cleanup_complete(evidence):
|
||||
self.operation.succeed(evidence=evidence)
|
||||
return
|
||||
if (
|
||||
evidence.get("verified") is True
|
||||
and evidence.get("object_present") is True
|
||||
and evidence.get("finding_deleted") is False
|
||||
):
|
||||
self.operation.reject(
|
||||
summary="The orphan object was retained and the cleanup finding stayed open",
|
||||
evidence=evidence,
|
||||
)
|
||||
return
|
||||
if evidence.get("object_present") is False:
|
||||
if _forward_complete_orphan_finding(self):
|
||||
completed = _orphan_cleanup_evidence(self)
|
||||
if _orphan_cleanup_complete(completed):
|
||||
self.operation.succeed(evidence=completed)
|
||||
return
|
||||
status = (
|
||||
RecoveryStatus.OUTCOME_UNKNOWN
|
||||
if not evidence.get("verified")
|
||||
else RecoveryStatus.RECOVERY_REQUIRED
|
||||
)
|
||||
self.operation.unresolved(
|
||||
status=status,
|
||||
summary="Files orphan cleanup requires reconciliation",
|
||||
evidence=evidence,
|
||||
failure_summary="Recheck the object and integrity-finding state before another cleanup attempt",
|
||||
)
|
||||
|
||||
|
||||
def begin_blob_write_recovery(
|
||||
session: Session,
|
||||
*,
|
||||
backend: StorageBackend,
|
||||
tenant_id: str,
|
||||
blob_id: str,
|
||||
storage_key: str,
|
||||
semantic_checksum_sha256: str,
|
||||
semantic_size_bytes: int,
|
||||
protection_discriminator: str,
|
||||
created_new: bool,
|
||||
repair_token: str | None = None,
|
||||
) -> PendingBlobWrite:
|
||||
disposition = "create" if created_new else "repair"
|
||||
state_token = repair_token or blob_id
|
||||
key_digest = hashlib.sha256(storage_key.encode("utf-8")).hexdigest()
|
||||
idempotency_key = (
|
||||
f"files-blob-{disposition}:{blob_id}:{state_token[:48]}"
|
||||
)
|
||||
try:
|
||||
identity = process_runtime_identity()
|
||||
except RuntimeError as exc:
|
||||
raise FileStorageError(
|
||||
"The Files recovery ledger is unavailable; no object was written"
|
||||
) from exc
|
||||
lease_resource_key = f"files:blob:{tenant_id}:{blob_id}"
|
||||
request = {
|
||||
"tenant_id": tenant_id,
|
||||
"blob_id": blob_id,
|
||||
"storage_key": storage_key if created_new else None,
|
||||
"storage_key_sha256": key_digest,
|
||||
"semantic_checksum_sha256": semantic_checksum_sha256,
|
||||
"semantic_size_bytes": semantic_size_bytes,
|
||||
"protection_discriminator": protection_discriminator,
|
||||
"disposition": disposition,
|
||||
}
|
||||
recovery_plan = RecoveryPlan(
|
||||
mode=(
|
||||
RecoveryMode.COMPENSATION
|
||||
if created_new
|
||||
else RecoveryMode.FORWARD_RECOVERY
|
||||
),
|
||||
preconditions=(
|
||||
"the caller has Files write authority for the target owner",
|
||||
"the storage key belongs to the tenant Files namespace",
|
||||
"the request records digests rather than file contents",
|
||||
),
|
||||
compensation_steps=(
|
||||
"verify that no FileBlob references the newly reserved key",
|
||||
"delete only that unreferenced key and verify absence",
|
||||
)
|
||||
if created_new
|
||||
else (),
|
||||
forward_recovery_steps=(
|
||||
"verify the expected bytes at the existing blob key",
|
||||
"forward-complete matching integrity metadata or quarantine the blob",
|
||||
)
|
||||
if not created_new
|
||||
else (),
|
||||
verification_steps=(
|
||||
"reload FileBlob metadata through an independent session",
|
||||
"stream and hash the managed object independently",
|
||||
),
|
||||
)
|
||||
precondition_evidence = {
|
||||
"blob_id": blob_id,
|
||||
"storage_key_sha256": key_digest,
|
||||
"semantic_checksum_sha256": semantic_checksum_sha256,
|
||||
"semantic_size_bytes": semantic_size_bytes,
|
||||
"created_new": created_new,
|
||||
}
|
||||
metadata = {
|
||||
"resources": ["postgresql", "object-storage"],
|
||||
"storage_backend": backend.name,
|
||||
"durability_mode": "independent_transaction",
|
||||
}
|
||||
session_factory = get_database().SessionLocal
|
||||
sqlite_mode = session.get_bind().dialect.name == "sqlite"
|
||||
sqlite_fence_key: str | None = None
|
||||
|
||||
def start_independent(*, reconstructed_after_rollback: bool = False) -> DurableRecoveryStart:
|
||||
return begin_durable_recovery_operation(
|
||||
session_factory,
|
||||
identity=identity,
|
||||
module_id="files",
|
||||
operation_type=f"blob-{disposition}",
|
||||
idempotency_key=idempotency_key,
|
||||
request=request,
|
||||
recovery_plan=recovery_plan,
|
||||
precondition_evidence={
|
||||
**precondition_evidence,
|
||||
**(
|
||||
{
|
||||
"sqlite_caller_transaction_rolled_back": True,
|
||||
"effect_may_have_preceded_durable_intent": True,
|
||||
}
|
||||
if reconstructed_after_rollback
|
||||
else {}
|
||||
),
|
||||
},
|
||||
lease_resource_key=lease_resource_key,
|
||||
lease_ttl_seconds=15 * 60,
|
||||
resource_type="file_blob",
|
||||
resource_id=blob_id,
|
||||
metadata={
|
||||
**metadata,
|
||||
"durability_mode": (
|
||||
"sqlite_post_rollback_reconstruction"
|
||||
if reconstructed_after_rollback
|
||||
else "independent_transaction"
|
||||
),
|
||||
},
|
||||
)
|
||||
|
||||
try:
|
||||
if sqlite_mode:
|
||||
_reserve_sqlite_fence(lease_resource_key)
|
||||
sqlite_fence_key = lease_resource_key
|
||||
started = _begin_caller_transaction_recovery_operation(
|
||||
session,
|
||||
session_factory=session_factory,
|
||||
identity=identity,
|
||||
module_id="files",
|
||||
operation_type=f"blob-{disposition}",
|
||||
idempotency_key=idempotency_key,
|
||||
request=request,
|
||||
recovery_plan=recovery_plan,
|
||||
precondition_evidence={
|
||||
**precondition_evidence,
|
||||
"sqlite_caller_transaction": True,
|
||||
"reduced_crash_durability": True,
|
||||
},
|
||||
lease_resource_key=lease_resource_key,
|
||||
lease_ttl_seconds=15 * 60,
|
||||
resource_type="file_blob",
|
||||
resource_id=blob_id,
|
||||
metadata={
|
||||
**metadata,
|
||||
"resources": ["sqlite", "object-storage"],
|
||||
"durability_mode": "sqlite_caller_transaction",
|
||||
},
|
||||
)
|
||||
else:
|
||||
started = start_independent()
|
||||
except (RecoveryOperationBusy, RecoveryOperationStateConflict) as exc:
|
||||
if sqlite_fence_key is not None:
|
||||
_release_sqlite_fence(sqlite_fence_key)
|
||||
raise FileStorageError(
|
||||
"This managed blob is already owned by another recovery operation"
|
||||
) from exc
|
||||
except (RecoveryGuaranteeError, RuntimeError) as exc:
|
||||
if sqlite_fence_key is not None:
|
||||
_release_sqlite_fence(sqlite_fence_key)
|
||||
raise FileStorageError(
|
||||
"The Files recovery ledger is unavailable; no object was written"
|
||||
) from exc
|
||||
if started.replayed or started.operation is None:
|
||||
if sqlite_fence_key is not None:
|
||||
_release_sqlite_fence(sqlite_fence_key)
|
||||
raise FileStorageError(
|
||||
"The matching Files blob operation was already completed; reload before retrying"
|
||||
)
|
||||
pending = PendingBlobWrite(
|
||||
operation=started.operation,
|
||||
backend=backend,
|
||||
tenant_id=tenant_id,
|
||||
blob_id=blob_id,
|
||||
storage_key=storage_key,
|
||||
semantic_checksum_sha256=semantic_checksum_sha256,
|
||||
semantic_size_bytes=semantic_size_bytes,
|
||||
protection_discriminator=protection_discriminator,
|
||||
created_new=created_new,
|
||||
sqlite_fence_key=sqlite_fence_key,
|
||||
restart_after_rollback=(
|
||||
lambda: _require_started_operation(
|
||||
start_independent(reconstructed_after_rollback=True)
|
||||
)
|
||||
if sqlite_mode
|
||||
else None
|
||||
),
|
||||
)
|
||||
_register_pending_effect(session, pending)
|
||||
return pending
|
||||
|
||||
|
||||
def begin_orphan_cleanup_recovery(
|
||||
session: Session,
|
||||
finding: FileIntegrityFinding,
|
||||
*,
|
||||
backend: StorageBackend,
|
||||
user_id: str,
|
||||
) -> PendingOrphanCleanup:
|
||||
key_digest = hashlib.sha256(finding.storage_key.encode("utf-8")).hexdigest()
|
||||
try:
|
||||
started = begin_durable_recovery_operation(
|
||||
get_database().SessionLocal,
|
||||
identity=process_runtime_identity(),
|
||||
module_id="files",
|
||||
operation_type="integrity-orphan-cleanup",
|
||||
idempotency_key=f"files-orphan-cleanup:{finding.id}",
|
||||
request={
|
||||
"tenant_id": finding.tenant_id,
|
||||
"finding_id": finding.id,
|
||||
"storage_key_sha256": key_digest,
|
||||
},
|
||||
recovery_plan=RecoveryPlan(
|
||||
mode=RecoveryMode.FORWARD_RECOVERY,
|
||||
preconditions=(
|
||||
"the integrity finding identifies an unreferenced object",
|
||||
"the object key remains inside the completed scan scope",
|
||||
"a fresh database reference check found no FileBlob",
|
||||
),
|
||||
forward_recovery_steps=(
|
||||
"verify object absence independently",
|
||||
"mark the durable finding deleted only after absence is proven",
|
||||
),
|
||||
verification_steps=(
|
||||
"reload the finding and its scan through an independent session",
|
||||
"probe the original storage key through the configured backend",
|
||||
),
|
||||
),
|
||||
precondition_evidence={
|
||||
"finding_id": finding.id,
|
||||
"scan_id": finding.scan_id,
|
||||
"storage_key_sha256": key_digest,
|
||||
"finding_state": finding.state,
|
||||
},
|
||||
lease_resource_key=(
|
||||
f"files:orphan-cleanup:{finding.tenant_id}:{key_digest[:40]}"
|
||||
),
|
||||
lease_ttl_seconds=15 * 60,
|
||||
resource_type="file_integrity_finding",
|
||||
resource_id=finding.id,
|
||||
metadata={
|
||||
"resources": ["postgresql", "object-storage"],
|
||||
"storage_backend": backend.name,
|
||||
},
|
||||
)
|
||||
except (RecoveryOperationBusy, RecoveryOperationStateConflict) as exc:
|
||||
raise FileStorageError(
|
||||
"This orphan object is already owned by another recovery operation"
|
||||
) from exc
|
||||
except (RecoveryGuaranteeError, RuntimeError) as exc:
|
||||
raise FileStorageError(
|
||||
"The Files recovery ledger is unavailable; no object was deleted"
|
||||
) from exc
|
||||
if started.replayed or started.operation is None:
|
||||
raise FileStorageError(
|
||||
"This orphan cleanup was already completed; reload the finding"
|
||||
)
|
||||
pending = PendingOrphanCleanup(
|
||||
operation=started.operation,
|
||||
backend=backend,
|
||||
finding_id=finding.id,
|
||||
tenant_id=finding.tenant_id,
|
||||
storage_key=finding.storage_key,
|
||||
resolved_by_user_id=user_id,
|
||||
)
|
||||
_register_pending_effect(session, pending)
|
||||
return pending
|
||||
|
||||
|
||||
def _register_pending_effect(session: Session, effect: _PendingEffect) -> None:
|
||||
if not session.in_transaction():
|
||||
session.begin()
|
||||
pending = session.info.setdefault(_PENDING_EFFECTS_KEY, [])
|
||||
pending.append(effect)
|
||||
if session.info.get(_HOOKS_INSTALLED_KEY):
|
||||
return
|
||||
event.listen(session, "after_commit", _after_session_commit)
|
||||
event.listen(session, "after_rollback", _after_session_rollback)
|
||||
session.info[_HOOKS_INSTALLED_KEY] = True
|
||||
|
||||
|
||||
def _after_session_commit(session: Session) -> None:
|
||||
# SQLAlchemy also emits after_commit for a released SAVEPOINT. A Files
|
||||
# batch may open one while creating a later recovery row; settle only when
|
||||
# the outer business transaction has actually become visible.
|
||||
if session.in_nested_transaction():
|
||||
return
|
||||
_settle_pending_effects(session, committed=True)
|
||||
|
||||
|
||||
def _after_session_rollback(session: Session) -> None:
|
||||
if session.in_nested_transaction():
|
||||
return
|
||||
try:
|
||||
_settle_pending_effects(session, committed=False)
|
||||
except RecoveryGuaranteeError as exc:
|
||||
session.info.setdefault(_ROLLBACK_ERRORS_KEY, []).append(str(exc))
|
||||
|
||||
|
||||
def _settle_pending_effects(session: Session, *, committed: bool) -> None:
|
||||
pending = list(session.info.pop(_PENDING_EFFECTS_KEY, []))
|
||||
failures: list[Exception] = []
|
||||
for effect in pending:
|
||||
try:
|
||||
effect.settle(committed=committed)
|
||||
except Exception as exc: # preserve every effect's chance to settle
|
||||
failures.append(exc)
|
||||
operation = getattr(effect, "operation", None)
|
||||
if operation is not None and not operation.closed:
|
||||
try:
|
||||
operation.release_unresolved()
|
||||
except Exception:
|
||||
pass
|
||||
finally:
|
||||
sqlite_fence_key = getattr(effect, "sqlite_fence_key", None)
|
||||
if sqlite_fence_key:
|
||||
_release_sqlite_fence(sqlite_fence_key)
|
||||
if failures:
|
||||
raise RecoveryGuaranteeError(
|
||||
f"{len(failures)} Files recovery operation(s) could not be finalized"
|
||||
) from failures[0]
|
||||
|
||||
|
||||
def _reserve_sqlite_fence(resource_key: str) -> None:
|
||||
with _SQLITE_FENCES_LOCK:
|
||||
if resource_key in _SQLITE_FENCES:
|
||||
raise RecoveryOperationBusy(
|
||||
f"Another local SQLite transaction owns {resource_key}"
|
||||
)
|
||||
_SQLITE_FENCES.add(resource_key)
|
||||
|
||||
|
||||
def _release_sqlite_fence(resource_key: str) -> None:
|
||||
with _SQLITE_FENCES_LOCK:
|
||||
_SQLITE_FENCES.discard(resource_key)
|
||||
|
||||
|
||||
def _require_started_operation(started: DurableRecoveryStart) -> DurableRecoveryOperation:
|
||||
if started.replayed or started.operation is None:
|
||||
raise RecoveryOperationStateConflict(started.operation_id, started.status)
|
||||
return started.operation
|
||||
|
||||
|
||||
def _begin_caller_transaction_recovery_operation(
|
||||
session: Session,
|
||||
*,
|
||||
session_factory: Callable[[], Session],
|
||||
identity: RuntimeIdentity,
|
||||
module_id: str,
|
||||
operation_type: str,
|
||||
idempotency_key: str,
|
||||
request: dict[str, object],
|
||||
recovery_plan: RecoveryPlan,
|
||||
precondition_evidence: dict[str, object],
|
||||
lease_resource_key: str,
|
||||
lease_ttl_seconds: int,
|
||||
resource_type: str,
|
||||
resource_id: str,
|
||||
metadata: dict[str, object],
|
||||
) -> DurableRecoveryStart:
|
||||
"""Start SQLite recovery evidence inside the caller transaction.
|
||||
|
||||
SQLite permits only one writer, so opening the normal independent ledger
|
||||
transaction after an earlier member of a batch has written will deadlock
|
||||
until the busy timeout. The caller-transaction mode preserves fencing,
|
||||
request hashing, checkpoints, commit verification, and rollback
|
||||
compensation, but cannot make the intent survive a hard process loss
|
||||
before the caller commits. PostgreSQL never uses this reduced mode.
|
||||
"""
|
||||
|
||||
claim = acquire_lease(
|
||||
session,
|
||||
installation_id=identity.installation_id,
|
||||
resource_key=lease_resource_key,
|
||||
holder_node_id=identity.node_id,
|
||||
holder_incarnation=identity.incarnation,
|
||||
ttl_seconds=lease_ttl_seconds,
|
||||
metadata={
|
||||
"module_id": module_id,
|
||||
"operation_type": operation_type,
|
||||
"durability_mode": "sqlite_caller_transaction",
|
||||
},
|
||||
)
|
||||
if claim is None:
|
||||
raise RecoveryOperationBusy(
|
||||
f"Another runtime owns the recovery fence for {lease_resource_key}"
|
||||
)
|
||||
existing = session.execute(
|
||||
select(RecoveryOperation).where(
|
||||
RecoveryOperation.installation_id == identity.installation_id,
|
||||
RecoveryOperation.module_id == module_id,
|
||||
RecoveryOperation.idempotency_key == idempotency_key,
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
operation = plan_recovery_operation(
|
||||
session,
|
||||
installation_id=identity.installation_id,
|
||||
module_id=module_id,
|
||||
operation_type=operation_type,
|
||||
idempotency_key=idempotency_key,
|
||||
request=request,
|
||||
recovery_plan=recovery_plan,
|
||||
resource_type=resource_type,
|
||||
resource_id=resource_id,
|
||||
lease_claim=claim,
|
||||
metadata=metadata,
|
||||
)
|
||||
if existing is not None:
|
||||
if operation.status == RecoveryStatus.SUCCEEDED.value:
|
||||
release_lease(session, claim)
|
||||
return DurableRecoveryStart(
|
||||
operation_id=operation.id,
|
||||
status=operation.status,
|
||||
replayed=True,
|
||||
operation=None,
|
||||
)
|
||||
raise RecoveryOperationStateConflict(operation.id, operation.status)
|
||||
prepare_recovery_operation(
|
||||
session,
|
||||
operation,
|
||||
evidence=precondition_evidence,
|
||||
lease_claim=claim,
|
||||
)
|
||||
start_recovery_operation(
|
||||
session,
|
||||
operation,
|
||||
evidence={"lease_resource_key": lease_resource_key},
|
||||
lease_claim=claim,
|
||||
)
|
||||
if not verify_recovery_evidence_chain(session, operation.id):
|
||||
raise RecoveryGuaranteeError(
|
||||
"Recovery checkpoint chain verification failed before side effects"
|
||||
)
|
||||
return DurableRecoveryStart(
|
||||
operation_id=operation.id,
|
||||
status=operation.status,
|
||||
replayed=False,
|
||||
operation=DurableRecoveryOperation(
|
||||
session_factory=session_factory,
|
||||
operation_id=operation.id,
|
||||
lease_claim=claim,
|
||||
lease_ttl_seconds=lease_ttl_seconds,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _blob_write_evidence(effect: PendingBlobWrite) -> dict[str, object]:
|
||||
database_blob_present: bool | None
|
||||
database_matches: bool | None
|
||||
database_integrity_verified: bool | None
|
||||
try:
|
||||
with effect.operation.session_factory() as session:
|
||||
blob = session.get(FileBlob, effect.blob_id)
|
||||
database_blob_present = blob is not None
|
||||
database_matches = bool(
|
||||
blob is not None
|
||||
and blob.tenant_id == effect.tenant_id
|
||||
and blob.storage_key == effect.storage_key
|
||||
and blob.checksum_sha256 == effect.semantic_checksum_sha256
|
||||
and blob.size_bytes == effect.semantic_size_bytes
|
||||
and blob.protection_discriminator
|
||||
== effect.protection_discriminator
|
||||
and blob.encryption_envelope_id == effect.expected_envelope_id
|
||||
and (
|
||||
effect.expected_storage_checksum_sha256
|
||||
== (
|
||||
blob.storage_checksum_sha256
|
||||
or blob.checksum_sha256
|
||||
)
|
||||
)
|
||||
) if blob is not None else False
|
||||
database_integrity_verified = bool(
|
||||
blob is not None
|
||||
and blob.integrity_status == "verified"
|
||||
and blob.quarantined_at is None
|
||||
) if blob is not None else False
|
||||
except Exception:
|
||||
database_blob_present = None
|
||||
database_matches = None
|
||||
database_integrity_verified = None
|
||||
|
||||
object_present: bool | None
|
||||
observed_size: int | None = None
|
||||
observed_checksum: str | None = None
|
||||
try:
|
||||
digest = hashlib.sha256()
|
||||
observed_size = 0
|
||||
for chunk in effect.backend.iter_bytes(effect.storage_key):
|
||||
observed_size += len(chunk)
|
||||
digest.update(chunk)
|
||||
observed_checksum = digest.hexdigest()
|
||||
object_present = True
|
||||
except StorageObjectMissing:
|
||||
object_present = False
|
||||
except (StorageBackendError, OSError):
|
||||
object_present = None
|
||||
|
||||
object_size_matches = (
|
||||
observed_size == effect.expected_storage_size_bytes
|
||||
if object_present is True and effect.expected_storage_size_bytes is not None
|
||||
else False if object_present is False else None
|
||||
)
|
||||
object_checksum_matches = (
|
||||
observed_checksum == effect.expected_storage_checksum_sha256
|
||||
if object_present is True
|
||||
and effect.expected_storage_checksum_sha256 is not None
|
||||
else False if object_present is False else None
|
||||
)
|
||||
verified = database_blob_present is not None and object_present is not None
|
||||
return {
|
||||
"verified": verified,
|
||||
"checks": {
|
||||
"database_reloaded": database_blob_present is not None,
|
||||
"object_probed": object_present is not None,
|
||||
"database_matches_request": database_matches,
|
||||
"database_integrity_verified": database_integrity_verified,
|
||||
"object_size_matches": object_size_matches,
|
||||
"object_checksum_matches": object_checksum_matches,
|
||||
},
|
||||
"blob_id": effect.blob_id,
|
||||
"database_blob_present": database_blob_present,
|
||||
"database_matches_request": database_matches,
|
||||
"database_integrity_verified": database_integrity_verified,
|
||||
"object_present": object_present,
|
||||
"observed_size_bytes": observed_size,
|
||||
"observed_checksum_sha256": observed_checksum,
|
||||
"expected_storage_size_bytes": effect.expected_storage_size_bytes,
|
||||
"expected_storage_checksum_sha256": effect.expected_storage_checksum_sha256,
|
||||
}
|
||||
|
||||
|
||||
def _blob_write_complete(evidence: dict[str, object]) -> bool:
|
||||
return bool(
|
||||
evidence.get("verified") is True
|
||||
and evidence.get("database_matches_request") is True
|
||||
and evidence.get("database_integrity_verified") is True
|
||||
and _object_matches(evidence)
|
||||
)
|
||||
|
||||
|
||||
def _object_matches(evidence: dict[str, object]) -> bool:
|
||||
checks = evidence.get("checks")
|
||||
return bool(
|
||||
isinstance(checks, dict)
|
||||
and evidence.get("object_present") is True
|
||||
and checks.get("object_size_matches") is True
|
||||
and checks.get("object_checksum_matches") is True
|
||||
)
|
||||
|
||||
|
||||
def _forward_complete_blob_repair(effect: PendingBlobWrite) -> bool:
|
||||
try:
|
||||
with effect.operation.session_factory() as session:
|
||||
blob = session.get(FileBlob, effect.blob_id)
|
||||
if (
|
||||
blob is None
|
||||
or blob.tenant_id != effect.tenant_id
|
||||
or blob.storage_key != effect.storage_key
|
||||
or blob.checksum_sha256 != effect.semantic_checksum_sha256
|
||||
or blob.size_bytes != effect.semantic_size_bytes
|
||||
or blob.protection_discriminator
|
||||
!= effect.protection_discriminator
|
||||
or blob.encryption_envelope_id != effect.expected_envelope_id
|
||||
):
|
||||
return False
|
||||
blob.storage_checksum_sha256 = (
|
||||
effect.expected_storage_checksum_sha256
|
||||
if effect.expected_envelope_id
|
||||
else None
|
||||
)
|
||||
blob.storage_size_bytes = (
|
||||
effect.expected_storage_size_bytes
|
||||
if effect.expected_envelope_id
|
||||
else None
|
||||
)
|
||||
blob.integrity_status = "verified"
|
||||
blob.integrity_checked_at = datetime.now(UTC)
|
||||
blob.integrity_failure = None
|
||||
blob.quarantined_at = None
|
||||
session.add(blob)
|
||||
session.commit()
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _quarantine_blob_after_failed_verification(
|
||||
effect: PendingBlobWrite,
|
||||
evidence: dict[str, object],
|
||||
) -> None:
|
||||
try:
|
||||
with effect.operation.session_factory() as session:
|
||||
blob = session.get(FileBlob, effect.blob_id)
|
||||
if blob is None or blob.storage_key != effect.storage_key:
|
||||
return
|
||||
blob.integrity_status = (
|
||||
"missing"
|
||||
if evidence.get("object_present") is False
|
||||
else "checksum_mismatch"
|
||||
)
|
||||
blob.integrity_checked_at = datetime.now(UTC)
|
||||
blob.integrity_failure = "recovery_verification_failed"
|
||||
blob.quarantined_at = datetime.now(UTC)
|
||||
session.add(blob)
|
||||
session.commit()
|
||||
except Exception:
|
||||
return
|
||||
|
||||
|
||||
def _orphan_cleanup_evidence(effect: PendingOrphanCleanup) -> dict[str, object]:
|
||||
try:
|
||||
object_present: bool | None = effect.backend.exists(effect.storage_key)
|
||||
except (StorageBackendError, OSError):
|
||||
object_present = None
|
||||
finding_present: bool | None
|
||||
finding_deleted: bool | None
|
||||
still_unreferenced: bool | None
|
||||
try:
|
||||
with effect.operation.session_factory() as session:
|
||||
finding = session.get(FileIntegrityFinding, effect.finding_id)
|
||||
finding_present = finding is not None
|
||||
finding_deleted = bool(
|
||||
finding is not None and finding.state == "deleted"
|
||||
) if finding is not None else False
|
||||
referenced = (
|
||||
session.query(FileBlob.id)
|
||||
.filter(
|
||||
FileBlob.tenant_id == effect.tenant_id,
|
||||
FileBlob.storage_key == effect.storage_key,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
still_unreferenced = referenced is None
|
||||
except Exception:
|
||||
finding_present = None
|
||||
finding_deleted = None
|
||||
still_unreferenced = None
|
||||
verified = object_present is not None and finding_present is not None
|
||||
return {
|
||||
"verified": verified,
|
||||
"checks": {
|
||||
"database_reloaded": finding_present is not None,
|
||||
"object_probed": object_present is not None,
|
||||
"finding_marked_deleted": finding_deleted,
|
||||
"object_absent": (
|
||||
not object_present if object_present is not None else None
|
||||
),
|
||||
"still_unreferenced": still_unreferenced,
|
||||
},
|
||||
"finding_id": effect.finding_id,
|
||||
"finding_present": finding_present,
|
||||
"finding_deleted": finding_deleted,
|
||||
"object_present": object_present,
|
||||
"still_unreferenced": still_unreferenced,
|
||||
}
|
||||
|
||||
|
||||
def _orphan_cleanup_complete(evidence: dict[str, object]) -> bool:
|
||||
return bool(
|
||||
evidence.get("verified") is True
|
||||
and evidence.get("finding_deleted") is True
|
||||
and evidence.get("object_present") is False
|
||||
and evidence.get("still_unreferenced") is True
|
||||
)
|
||||
|
||||
|
||||
def _forward_complete_orphan_finding(effect: PendingOrphanCleanup) -> bool:
|
||||
try:
|
||||
with effect.operation.session_factory() as session:
|
||||
finding = session.get(FileIntegrityFinding, effect.finding_id)
|
||||
if (
|
||||
finding is None
|
||||
or finding.tenant_id != effect.tenant_id
|
||||
or finding.storage_key != effect.storage_key
|
||||
):
|
||||
return False
|
||||
scan = session.get(FileIntegrityScan, finding.scan_id)
|
||||
if scan is None or not finding.storage_key.startswith(
|
||||
scan.storage_prefix
|
||||
):
|
||||
return False
|
||||
referenced = (
|
||||
session.query(FileBlob.id)
|
||||
.filter(
|
||||
FileBlob.tenant_id == effect.tenant_id,
|
||||
FileBlob.storage_key == effect.storage_key,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if referenced:
|
||||
return False
|
||||
finding.state = "deleted"
|
||||
finding.resolved_at = datetime.now(UTC)
|
||||
finding.resolved_by_user_id = effect.resolved_by_user_id
|
||||
session.add(finding)
|
||||
session.commit()
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
__all__ = [
|
||||
"PendingBlobWrite",
|
||||
"PendingOrphanCleanup",
|
||||
"begin_blob_write_recovery",
|
||||
"begin_orphan_cleanup_recovery",
|
||||
]
|
||||
@@ -0,0 +1,331 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import inspect
|
||||
import socket
|
||||
import threading
|
||||
from functools import lru_cache
|
||||
from importlib import import_module
|
||||
from typing import Any
|
||||
|
||||
from govoplan_core.security.outbound_http import (
|
||||
OutboundHttpError,
|
||||
create_outbound_connection,
|
||||
)
|
||||
|
||||
|
||||
class SdkPeerPinningError(RuntimeError):
|
||||
"""Raised when an optional SDK cannot be bound to the pinned transport."""
|
||||
|
||||
|
||||
_BOTOCORE_CLIENT_CREATION_LOCK = threading.Lock()
|
||||
|
||||
|
||||
def create_pinned_s3_client(**kwargs: Any) -> Any:
|
||||
"""Construct an S3 client whose first and every later socket is pinned.
|
||||
|
||||
Botocore does not expose the HTTP-session class through boto3's public
|
||||
client API. Its endpoint creator does expose that seam, so this function
|
||||
replaces the creator only for the bounded client-construction operation.
|
||||
The lock avoids an unsafe interleaving with another GovOPlaN client
|
||||
construction. Any unrelated botocore client created during the short
|
||||
replacement window also receives the stricter transport.
|
||||
"""
|
||||
|
||||
try:
|
||||
boto3 = import_module("boto3")
|
||||
botocore_args = import_module("botocore.args")
|
||||
except ImportError as exc:
|
||||
raise SdkPeerPinningError(
|
||||
"S3 connector browsing requires the optional boto3 dependency"
|
||||
) from exc
|
||||
|
||||
pinned_session_cls = _botocore_transport_types()[0]
|
||||
original_creator = getattr(botocore_args, "EndpointCreator", None)
|
||||
if original_creator is None or not hasattr(original_creator, "create_endpoint"):
|
||||
raise SdkPeerPinningError(
|
||||
"The installed botocore release does not expose the endpoint transport seam required for peer pinning"
|
||||
)
|
||||
|
||||
class PinnedEndpointCreator(original_creator): # type: ignore[misc, valid-type]
|
||||
def create_endpoint(self, *args: Any, **endpoint_kwargs: Any) -> Any:
|
||||
endpoint_kwargs["http_session_cls"] = pinned_session_cls
|
||||
return super().create_endpoint(*args, **endpoint_kwargs)
|
||||
|
||||
with _BOTOCORE_CLIENT_CREATION_LOCK:
|
||||
current_creator = getattr(botocore_args, "EndpointCreator", None)
|
||||
if current_creator is not original_creator:
|
||||
raise SdkPeerPinningError(
|
||||
"Botocore endpoint construction changed concurrently; refusing to create an unproven S3 client"
|
||||
)
|
||||
botocore_args.EndpointCreator = PinnedEndpointCreator
|
||||
try:
|
||||
session_type = getattr(getattr(boto3, "session", None), "Session", None)
|
||||
if session_type is None:
|
||||
raise SdkPeerPinningError(
|
||||
"The installed boto3 release does not expose its isolated session constructor"
|
||||
)
|
||||
client = session_type().client("s3", **kwargs)
|
||||
finally:
|
||||
if getattr(botocore_args, "EndpointCreator", None) is PinnedEndpointCreator:
|
||||
botocore_args.EndpointCreator = original_creator
|
||||
|
||||
http_session = getattr(getattr(client, "_endpoint", None), "http_session", None)
|
||||
if http_session is None or not isinstance(http_session, pinned_session_cls):
|
||||
close = getattr(client, "close", None)
|
||||
if callable(close):
|
||||
close()
|
||||
raise SdkPeerPinningError(
|
||||
"Botocore did not install the required pinned HTTP transport; the S3 client was discarded"
|
||||
)
|
||||
return client
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _botocore_transport_types() -> tuple[type[Any], type[Any], type[Any]]:
|
||||
try:
|
||||
awsrequest = import_module("botocore.awsrequest")
|
||||
httpsession = import_module("botocore.httpsession")
|
||||
urllib3_exceptions = import_module("urllib3.exceptions")
|
||||
except ImportError as exc:
|
||||
raise SdkPeerPinningError(
|
||||
"S3 connector browsing requires a compatible botocore HTTP transport"
|
||||
) from exc
|
||||
|
||||
required = (
|
||||
"AWSHTTPConnection",
|
||||
"AWSHTTPSConnection",
|
||||
"AWSHTTPConnectionPool",
|
||||
"AWSHTTPSConnectionPool",
|
||||
)
|
||||
if any(not hasattr(awsrequest, name) for name in required) or not hasattr(
|
||||
httpsession, "URLLib3Session"
|
||||
):
|
||||
raise SdkPeerPinningError(
|
||||
"The installed botocore release is missing the connection classes required for peer pinning"
|
||||
)
|
||||
|
||||
def pinned_new_connection(connection: Any) -> socket.socket:
|
||||
hostname = str(getattr(connection, "_dns_host", "") or "").strip()
|
||||
port = int(getattr(connection, "port", 0) or 0)
|
||||
if not hostname or not port:
|
||||
raise urllib3_exceptions.NewConnectionError(
|
||||
connection, "Pinned S3 connection is missing its target authority"
|
||||
)
|
||||
try:
|
||||
return create_outbound_connection(
|
||||
hostname,
|
||||
port,
|
||||
timeout=getattr(connection, "timeout", None),
|
||||
source_address=getattr(connection, "source_address", None),
|
||||
socket_options=getattr(connection, "socket_options", None),
|
||||
label="S3 connector peer",
|
||||
)
|
||||
except socket.timeout as exc:
|
||||
raise urllib3_exceptions.ConnectTimeoutError(
|
||||
connection,
|
||||
f"Connection to {hostname} timed out while selecting an approved peer",
|
||||
) from exc
|
||||
except (OSError, OutboundHttpError, ValueError) as exc:
|
||||
raise urllib3_exceptions.NewConnectionError(
|
||||
connection,
|
||||
f"S3 connector peer was rejected: {exc}",
|
||||
) from exc
|
||||
|
||||
class PinnedAWSHTTPConnection(awsrequest.AWSHTTPConnection):
|
||||
_new_conn = pinned_new_connection
|
||||
|
||||
class PinnedAWSHTTPSConnection(awsrequest.AWSHTTPSConnection):
|
||||
_new_conn = pinned_new_connection
|
||||
|
||||
class PinnedAWSHTTPConnectionPool(awsrequest.AWSHTTPConnectionPool):
|
||||
ConnectionCls = PinnedAWSHTTPConnection
|
||||
|
||||
class PinnedAWSHTTPSConnectionPool(awsrequest.AWSHTTPSConnectionPool):
|
||||
ConnectionCls = PinnedAWSHTTPSConnection
|
||||
|
||||
class PinnedURLLib3Session(httpsession.URLLib3Session):
|
||||
def __init__(self, *args: Any, **kwargs: Any) -> None:
|
||||
# A proxy would select the final target outside this process. Until
|
||||
# proxy peer delegation is modeled, connector traffic is direct.
|
||||
kwargs["proxies"] = {}
|
||||
super().__init__(*args, **kwargs)
|
||||
self._pool_classes_by_scheme = {
|
||||
"http": PinnedAWSHTTPConnectionPool,
|
||||
"https": PinnedAWSHTTPSConnectionPool,
|
||||
}
|
||||
manager = getattr(self, "_manager", None)
|
||||
if manager is None or not hasattr(manager, "pool_classes_by_scheme"):
|
||||
raise SdkPeerPinningError(
|
||||
"The installed botocore pool manager cannot enforce pinned connection classes"
|
||||
)
|
||||
manager.pool_classes_by_scheme = self._pool_classes_by_scheme
|
||||
|
||||
return PinnedURLLib3Session, PinnedAWSHTTPConnection, PinnedAWSHTTPSConnection
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def pinned_smb_connection_cache() -> dict[str, Any]:
|
||||
"""Return the Files-owned cache; unproven process-global sessions are never reused."""
|
||||
|
||||
return {}
|
||||
|
||||
|
||||
def install_pinned_smb_transport(smbclient: Any) -> Any:
|
||||
"""Install a process-wide fail-closed smbclient session factory.
|
||||
|
||||
smbclient routes initial connections, reconnects, and DFS targets through
|
||||
``smbclient._pool.register_session``. Replacing that single factory with a
|
||||
behavior-compatible implementation ensures every target uses a socket
|
||||
selected by Core's deployment-wide outbound policy.
|
||||
"""
|
||||
|
||||
try:
|
||||
pool = import_module("smbclient._pool")
|
||||
connection_module = import_module("smbprotocol.connection")
|
||||
session_module = import_module("smbprotocol.session")
|
||||
transport_module = import_module("smbprotocol.transport")
|
||||
except ImportError as exc:
|
||||
raise SdkPeerPinningError(
|
||||
"SMB connector browsing requires the optional smbprotocol dependency"
|
||||
) from exc
|
||||
|
||||
current = getattr(pool, "register_session", None)
|
||||
if getattr(current, "__govoplan_peer_pinned__", False):
|
||||
expected_tcp = getattr(current, "__govoplan_pinned_tcp__", None)
|
||||
if expected_tcp is None or getattr(connection_module, "Tcp", None) is not expected_tcp:
|
||||
raise SdkPeerPinningError(
|
||||
"The installed SMB transport changed after peer pinning; refusing to reuse the session factory"
|
||||
)
|
||||
return smbclient
|
||||
required_parameters = {
|
||||
"server",
|
||||
"username",
|
||||
"password",
|
||||
"port",
|
||||
"encrypt",
|
||||
"connection_timeout",
|
||||
"connection_cache",
|
||||
"auth_protocol",
|
||||
"require_signing",
|
||||
}
|
||||
if current is None or not required_parameters.issubset(
|
||||
inspect.signature(current).parameters
|
||||
):
|
||||
raise SdkPeerPinningError(
|
||||
"The installed smbprotocol release does not expose the session seam required for peer pinning"
|
||||
)
|
||||
|
||||
class PinnedTcp(transport_module.Tcp):
|
||||
def connect(self) -> None:
|
||||
with self._sock_lock:
|
||||
if self.connected:
|
||||
return
|
||||
try:
|
||||
self._sock = create_outbound_connection(
|
||||
self.server,
|
||||
int(self.port),
|
||||
timeout=self.timeout,
|
||||
label="SMB connector peer",
|
||||
)
|
||||
except (OSError, OutboundHttpError, ValueError) as exc:
|
||||
raise ValueError(
|
||||
f"SMB connector peer '{self.server}:{self.port}' was rejected: {exc}"
|
||||
) from exc
|
||||
self._sock.settimeout(None)
|
||||
self.connected = True
|
||||
|
||||
# Connection.connect() instantiates the module-level Tcp symbol on every
|
||||
# reconnect. Replacing it process-wide is deliberate: an SMB connection
|
||||
# created by another code path must become stricter, never bypass Files'
|
||||
# peer boundary.
|
||||
if not hasattr(connection_module, "Tcp"):
|
||||
raise SdkPeerPinningError(
|
||||
"The installed smbprotocol release cannot install the pinned TCP transport"
|
||||
)
|
||||
connection_module.Tcp = PinnedTcp
|
||||
|
||||
def pinned_register_session(
|
||||
server: str,
|
||||
username: str | None = None,
|
||||
password: str | None = None,
|
||||
port: int = 445,
|
||||
encrypt: bool | None = None,
|
||||
connection_timeout: float = 60,
|
||||
connection_cache: dict[str, Any] | None = None,
|
||||
auth_protocol: str = "negotiate",
|
||||
require_signing: bool = True,
|
||||
) -> Any:
|
||||
cache = pinned_smb_connection_cache() if connection_cache is None else connection_cache
|
||||
connection_key = f"{server.lower()}:{port}"
|
||||
connection = cache.get(connection_key)
|
||||
transport = getattr(connection, "transport", None)
|
||||
if connection is not None and not isinstance(transport, PinnedTcp):
|
||||
disconnect = getattr(connection, "disconnect", None)
|
||||
if callable(disconnect):
|
||||
try:
|
||||
disconnect(close=True)
|
||||
except Exception:
|
||||
pass
|
||||
cache.pop(connection_key, None)
|
||||
connection = None
|
||||
if connection is None or not getattr(connection.transport, "connected", False):
|
||||
connection = connection_module.Connection(
|
||||
pool.ClientConfig().client_guid,
|
||||
server,
|
||||
port,
|
||||
require_signing=require_signing,
|
||||
)
|
||||
connection.transport = PinnedTcp(server, port)
|
||||
connection.connect(timeout=connection_timeout)
|
||||
if not isinstance(connection.transport, PinnedTcp):
|
||||
disconnect = getattr(connection, "disconnect", None)
|
||||
if callable(disconnect):
|
||||
try:
|
||||
disconnect(close=True)
|
||||
except Exception:
|
||||
pass
|
||||
raise SdkPeerPinningError(
|
||||
"smbprotocol replaced the required pinned TCP transport during connection setup"
|
||||
)
|
||||
cache[connection_key] = connection
|
||||
|
||||
session = next(
|
||||
(
|
||||
item
|
||||
for item in connection.session_table.values()
|
||||
if username is None or item.username == username
|
||||
),
|
||||
None,
|
||||
)
|
||||
if session is None:
|
||||
session = session_module.Session(
|
||||
connection,
|
||||
username=username,
|
||||
password=password,
|
||||
require_encryption=(encrypt is True),
|
||||
auth_protocol=auth_protocol,
|
||||
)
|
||||
session.connect()
|
||||
elif encrypt is not None:
|
||||
if session.encrypt_data and not encrypt:
|
||||
raise ValueError(
|
||||
"Cannot disable encryption on an already negotiated session."
|
||||
)
|
||||
if not session.encrypt_data and encrypt:
|
||||
session.encrypt = True
|
||||
return session
|
||||
|
||||
pinned_register_session.__govoplan_peer_pinned__ = True # type: ignore[attr-defined]
|
||||
pinned_register_session.__govoplan_pinned_tcp__ = PinnedTcp # type: ignore[attr-defined]
|
||||
pool.register_session = pinned_register_session
|
||||
if hasattr(smbclient, "register_session"):
|
||||
smbclient.register_session = pinned_register_session
|
||||
return smbclient
|
||||
|
||||
|
||||
__all__ = [
|
||||
"SdkPeerPinningError",
|
||||
"create_pinned_s3_client",
|
||||
"install_pinned_smb_transport",
|
||||
"pinned_smb_connection_cache",
|
||||
]
|
||||
@@ -0,0 +1,34 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import and_, or_
|
||||
|
||||
from govoplan_files.backend.db.models import FileShare
|
||||
from govoplan_files.backend.storage.common import utcnow
|
||||
|
||||
|
||||
def effective_file_share_clause(*, at: datetime | None = None):
|
||||
checked_at = at or utcnow()
|
||||
return and_(
|
||||
FileShare.revoked_at.is_(None),
|
||||
or_(FileShare.expires_at.is_(None), FileShare.expires_at > checked_at),
|
||||
)
|
||||
|
||||
|
||||
def file_share_is_active(share: FileShare, *, at: datetime | None = None) -> bool:
|
||||
if share.revoked_at is not None:
|
||||
return False
|
||||
if share.expires_at is None:
|
||||
return True
|
||||
checked_at = _aware_utc(at or utcnow())
|
||||
return _aware_utc(share.expires_at) > checked_at
|
||||
|
||||
|
||||
def _aware_utc(value: datetime) -> datetime:
|
||||
if value.tzinfo is None:
|
||||
return value.replace(tzinfo=timezone.utc)
|
||||
return value.astimezone(timezone.utc)
|
||||
|
||||
|
||||
__all__ = ["effective_file_share_clause", "file_share_is_active"]
|
||||
@@ -1,15 +1,23 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from unittest.mock import patch
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy import create_engine, text
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from govoplan_access.backend.db.models import Account, Group, User
|
||||
from govoplan_core.core.access import PrincipalRef
|
||||
from govoplan_core.core.change_sequence import ChangeSequenceEntry
|
||||
from govoplan_core.db.base import Base
|
||||
from govoplan_files.backend.capabilities import FilesAccessService, virtual_folder_resource_id
|
||||
from govoplan_files.backend.db.models import FileAsset, FileFolder, FileShare
|
||||
from govoplan_files.backend.storage.files import (
|
||||
count_assets_for_user,
|
||||
list_assets_for_user,
|
||||
list_recent_assets_for_user,
|
||||
)
|
||||
|
||||
|
||||
TENANT_ID = "tenant-1"
|
||||
@@ -19,6 +27,54 @@ GROUP_ID = "group-1"
|
||||
|
||||
|
||||
class FilesAccessProviderTests(unittest.TestCase):
|
||||
def test_recent_projection_is_bounded_ordered_and_access_filtered(self) -> None:
|
||||
session = _session()
|
||||
self.addCleanup(_close_session, session)
|
||||
_seed_access_subjects(session)
|
||||
now = datetime.now(timezone.utc)
|
||||
session.add_all(
|
||||
[
|
||||
FileAsset(
|
||||
id="owned-older",
|
||||
tenant_id=TENANT_ID,
|
||||
owner_type="user",
|
||||
owner_user_id=USER_ID,
|
||||
display_path="older.pdf",
|
||||
filename="older.pdf",
|
||||
updated_at=now - timedelta(hours=2),
|
||||
),
|
||||
FileAsset(
|
||||
id="owned-newer",
|
||||
tenant_id=TENANT_ID,
|
||||
owner_type="user",
|
||||
owner_user_id=USER_ID,
|
||||
display_path="newer.pdf",
|
||||
filename="newer.pdf",
|
||||
updated_at=now - timedelta(hours=1),
|
||||
),
|
||||
FileAsset(
|
||||
id="other-newest",
|
||||
tenant_id=TENANT_ID,
|
||||
owner_type="user",
|
||||
owner_user_id=OTHER_USER_ID,
|
||||
display_path="private.pdf",
|
||||
filename="private.pdf",
|
||||
updated_at=now,
|
||||
),
|
||||
]
|
||||
)
|
||||
session.commit()
|
||||
|
||||
with patch("govoplan_files.backend.storage.files.user_group_ids", return_value=[]):
|
||||
recent = list_recent_assets_for_user(
|
||||
session,
|
||||
tenant_id=TENANT_ID,
|
||||
user_id=USER_ID,
|
||||
limit=1,
|
||||
)
|
||||
|
||||
self.assertEqual(["owned-newer"], [asset.id for asset in recent])
|
||||
|
||||
def test_file_access_provider_explains_owner_share_admin_and_missing_resources(self) -> None:
|
||||
session = _session()
|
||||
self.addCleanup(_close_session, session)
|
||||
@@ -75,10 +131,123 @@ class FilesAccessProviderTests(unittest.TestCase):
|
||||
self.assertTrue(any(item.kind == "owner" and item.id == GROUP_ID for item in virtual_items))
|
||||
self.assertEqual("files.not_found", missing_items[0].source)
|
||||
|
||||
def test_file_property_filters_cover_campaign_and_audit_usage(self) -> None:
|
||||
session = _session()
|
||||
self.addCleanup(_close_session, session)
|
||||
_seed_access_subjects(session)
|
||||
assets = [
|
||||
FileAsset(
|
||||
id=f"file-{index}",
|
||||
tenant_id=TENANT_ID,
|
||||
owner_type="user",
|
||||
owner_user_id=USER_ID,
|
||||
display_path=f"{index}.pdf",
|
||||
filename=f"{index}.pdf",
|
||||
)
|
||||
for index in range(1, 4)
|
||||
]
|
||||
session.add_all([
|
||||
*assets,
|
||||
FileShare(
|
||||
id="share-campaign",
|
||||
tenant_id=TENANT_ID,
|
||||
file_asset_id=assets[0].id,
|
||||
target_type="campaign",
|
||||
target_id="campaign-1",
|
||||
permission="read",
|
||||
),
|
||||
FileShare(
|
||||
id="share-campaign-expired",
|
||||
tenant_id=TENANT_ID,
|
||||
file_asset_id=assets[2].id,
|
||||
target_type="campaign",
|
||||
target_id="campaign-1",
|
||||
permission="read",
|
||||
expires_at=datetime.now(timezone.utc) - timedelta(seconds=1),
|
||||
),
|
||||
])
|
||||
session.commit()
|
||||
session.execute(
|
||||
text(
|
||||
"INSERT INTO campaign_attachment_uses "
|
||||
"(id, tenant_id, file_asset_id, use_stage) "
|
||||
"VALUES (:id, :tenant_id, :file_asset_id, :use_stage)"
|
||||
),
|
||||
{
|
||||
"id": "attachment-use-1",
|
||||
"tenant_id": TENANT_ID,
|
||||
"file_asset_id": assets[1].id,
|
||||
"use_stage": "sent",
|
||||
},
|
||||
)
|
||||
session.commit()
|
||||
|
||||
linked = list_assets_for_user(
|
||||
session,
|
||||
tenant_id=TENANT_ID,
|
||||
user_id=USER_ID,
|
||||
owner_type="user",
|
||||
owner_id=USER_ID,
|
||||
campaign_usage="linked",
|
||||
)
|
||||
unlinked = list_assets_for_user(
|
||||
session,
|
||||
tenant_id=TENANT_ID,
|
||||
user_id=USER_ID,
|
||||
owner_type="user",
|
||||
owner_id=USER_ID,
|
||||
campaign_usage="unlinked",
|
||||
)
|
||||
audit_relevant = list_assets_for_user(
|
||||
session,
|
||||
tenant_id=TENANT_ID,
|
||||
user_id=USER_ID,
|
||||
owner_type="user",
|
||||
owner_id=USER_ID,
|
||||
audit_relevant=True,
|
||||
)
|
||||
|
||||
self.assertEqual([asset.id for asset in linked], ["file-1", "file-2"])
|
||||
self.assertEqual([asset.id for asset in unlinked], ["file-3"])
|
||||
self.assertEqual([asset.id for asset in audit_relevant], ["file-2"])
|
||||
self.assertEqual(
|
||||
count_assets_for_user(
|
||||
session,
|
||||
tenant_id=TENANT_ID,
|
||||
user_id=USER_ID,
|
||||
owner_type="user",
|
||||
owner_id=USER_ID,
|
||||
campaign_usage="unlinked",
|
||||
),
|
||||
1,
|
||||
)
|
||||
|
||||
|
||||
def _session():
|
||||
engine = create_engine("sqlite:///:memory:", future=True)
|
||||
Base.metadata.create_all(bind=engine, tables=[Account.__table__, User.__table__, Group.__table__, FileAsset.__table__, FileFolder.__table__, FileShare.__table__])
|
||||
Base.metadata.create_all(
|
||||
bind=engine,
|
||||
tables=[
|
||||
Account.__table__,
|
||||
User.__table__,
|
||||
Group.__table__,
|
||||
ChangeSequenceEntry.__table__,
|
||||
FileAsset.__table__,
|
||||
FileFolder.__table__,
|
||||
FileShare.__table__,
|
||||
],
|
||||
)
|
||||
with engine.begin() as connection:
|
||||
connection.execute(
|
||||
text(
|
||||
"CREATE TABLE campaign_attachment_uses ("
|
||||
"id VARCHAR(36) PRIMARY KEY, "
|
||||
"tenant_id VARCHAR(36) NOT NULL, "
|
||||
"file_asset_id VARCHAR(36) NOT NULL, "
|
||||
"use_stage VARCHAR(20) NOT NULL"
|
||||
")"
|
||||
)
|
||||
)
|
||||
return sessionmaker(bind=engine, future=True)()
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import hashlib
|
||||
import unittest
|
||||
import zipfile
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
from fastapi import UploadFile
|
||||
|
||||
from govoplan_files.backend.routes.uploads import (
|
||||
_validate_archive_preview_token,
|
||||
preview_archive_upload,
|
||||
)
|
||||
from govoplan_files.backend.storage.common import FileStorageError
|
||||
|
||||
|
||||
def _archive_bytes() -> bytes:
|
||||
target = io.BytesIO()
|
||||
with zipfile.ZipFile(target, "w", compression=zipfile.ZIP_DEFLATED) as archive:
|
||||
archive.writestr("folder/report.txt", b"report")
|
||||
return target.getvalue()
|
||||
|
||||
|
||||
class ArchivePreviewTokenTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.settings = SimpleNamespace(
|
||||
file_upload_zip_max_bytes=10 * 1024 * 1024,
|
||||
file_archive_max_entries=10_000,
|
||||
file_archive_max_expanded_bytes=2 * 1024 * 1024 * 1024,
|
||||
file_archive_max_expansion_ratio=100,
|
||||
file_archive_preview_ttl_seconds=30 * 60,
|
||||
)
|
||||
self.principal = SimpleNamespace(
|
||||
tenant_id="tenant-1",
|
||||
user=SimpleNamespace(id="user-1"),
|
||||
)
|
||||
|
||||
def test_preview_token_is_bound_to_actor_archive_and_destination(self) -> None:
|
||||
archive_data = _archive_bytes()
|
||||
upload = UploadFile(
|
||||
file=io.BytesIO(archive_data),
|
||||
filename="monthly.zip",
|
||||
)
|
||||
with patch(
|
||||
"govoplan_files.backend.routes.uploads.settings",
|
||||
self.settings,
|
||||
):
|
||||
preview = preview_archive_upload(
|
||||
file=upload,
|
||||
owner_type="user",
|
||||
owner_id="user-1",
|
||||
path="incoming",
|
||||
campaign_id=None,
|
||||
password=None,
|
||||
principal=self.principal,
|
||||
)
|
||||
_validate_archive_preview_token(
|
||||
preview.preview_token,
|
||||
tenant_id="tenant-1",
|
||||
user_id="user-1",
|
||||
owner_type="user",
|
||||
owner_id="user-1",
|
||||
path="incoming",
|
||||
campaign_id=None,
|
||||
archive_format="zip",
|
||||
archive_sha256=hashlib.sha256(archive_data).hexdigest(),
|
||||
)
|
||||
with self.assertRaisesRegex(
|
||||
FileStorageError, "does not match this upload destination"
|
||||
):
|
||||
_validate_archive_preview_token(
|
||||
preview.preview_token,
|
||||
tenant_id="tenant-1",
|
||||
user_id="user-1",
|
||||
owner_type="user",
|
||||
owner_id="user-1",
|
||||
path="elsewhere",
|
||||
campaign_id=None,
|
||||
archive_format="zip",
|
||||
archive_sha256=hashlib.sha256(archive_data).hexdigest(),
|
||||
)
|
||||
with self.assertRaisesRegex(
|
||||
FileStorageError, "contents changed"
|
||||
):
|
||||
_validate_archive_preview_token(
|
||||
preview.preview_token,
|
||||
tenant_id="tenant-1",
|
||||
user_id="user-1",
|
||||
owner_type="user",
|
||||
owner_id="user-1",
|
||||
path="incoming",
|
||||
campaign_id=None,
|
||||
archive_format="zip",
|
||||
archive_sha256="0" * 64,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,170 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import tarfile
|
||||
import unittest
|
||||
import zipfile
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import pyzipper
|
||||
|
||||
from govoplan_files.backend.storage.archives import (
|
||||
ArchivePasswordError,
|
||||
extract_archive_upload,
|
||||
inspect_archive,
|
||||
)
|
||||
from govoplan_files.backend.storage.common import FileStorageError
|
||||
|
||||
|
||||
def _zip_bytes(entries: dict[str, bytes]) -> bytes:
|
||||
target = io.BytesIO()
|
||||
with zipfile.ZipFile(target, "w", compression=zipfile.ZIP_DEFLATED) as archive:
|
||||
for path, data in entries.items():
|
||||
archive.writestr(path, data)
|
||||
return target.getvalue()
|
||||
|
||||
|
||||
def _tar_bytes(entries: dict[str, bytes], mode: str = "w:gz") -> bytes:
|
||||
target = io.BytesIO()
|
||||
with tarfile.open(fileobj=target, mode=mode) as archive:
|
||||
for path, data in entries.items():
|
||||
info = tarfile.TarInfo(path)
|
||||
info.size = len(data)
|
||||
archive.addfile(info, io.BytesIO(data))
|
||||
return target.getvalue()
|
||||
|
||||
|
||||
def _encrypted_zip_bytes(password: str) -> bytes:
|
||||
target = io.BytesIO()
|
||||
with pyzipper.AESZipFile(
|
||||
target,
|
||||
"w",
|
||||
compression=zipfile.ZIP_DEFLATED,
|
||||
encryption=pyzipper.WZ_AES,
|
||||
) as archive:
|
||||
archive.setpassword(password.encode("utf-8"))
|
||||
archive.writestr("secure/report.txt", b"classified")
|
||||
return target.getvalue()
|
||||
|
||||
|
||||
class ArchiveInspectionTests(unittest.TestCase):
|
||||
def test_zip_preview_derives_folders_and_sizes(self) -> None:
|
||||
inspection = inspect_archive(
|
||||
_zip_bytes(
|
||||
{
|
||||
"department/one.txt": b"one",
|
||||
"department/nested/two.csv": b"two",
|
||||
}
|
||||
),
|
||||
filename="monthly.zip",
|
||||
)
|
||||
|
||||
self.assertEqual("zip", inspection.archive_format)
|
||||
self.assertEqual(2, inspection.file_count)
|
||||
self.assertEqual(2, inspection.directory_count)
|
||||
self.assertEqual(6, inspection.expanded_size_bytes)
|
||||
self.assertEqual(
|
||||
[
|
||||
"department",
|
||||
"department/nested",
|
||||
"department/nested/two.csv",
|
||||
"department/one.txt",
|
||||
],
|
||||
[entry.path for entry in inspection.entries],
|
||||
)
|
||||
|
||||
def test_tar_compression_variants_are_inspected(self) -> None:
|
||||
for filename, mode in (
|
||||
("monthly.tar", "w"),
|
||||
("monthly.tar.gz", "w:gz"),
|
||||
("monthly.tar.bz2", "w:bz2"),
|
||||
("monthly.tar.xz", "w:xz"),
|
||||
):
|
||||
with self.subTest(filename=filename):
|
||||
inspection = inspect_archive(
|
||||
_tar_bytes({"report.txt": b"report"}, mode=mode),
|
||||
filename=filename,
|
||||
)
|
||||
self.assertEqual(1, inspection.file_count)
|
||||
self.assertEqual(filename.removeprefix("monthly."), inspection.archive_format)
|
||||
|
||||
def test_unsafe_member_path_is_rejected(self) -> None:
|
||||
with self.assertRaisesRegex(FileStorageError, "Unsafe archive member"):
|
||||
inspect_archive(
|
||||
_zip_bytes({"../escape.txt": b"no"}),
|
||||
filename="unsafe.zip",
|
||||
)
|
||||
|
||||
def test_archive_bomb_ratio_is_rejected(self) -> None:
|
||||
with self.assertRaisesRegex(FileStorageError, "expansion ratio"):
|
||||
inspect_archive(
|
||||
_zip_bytes({"zeros.bin": b"\0" * 50_000}),
|
||||
filename="bomb.zip",
|
||||
max_expansion_ratio=2,
|
||||
)
|
||||
|
||||
def test_encrypted_zip_reports_and_verifies_password(self) -> None:
|
||||
archive = _encrypted_zip_bytes("correct horse")
|
||||
|
||||
missing = inspect_archive(archive, filename="secure.zip")
|
||||
self.assertTrue(missing.requires_password)
|
||||
self.assertFalse(missing.password_verified)
|
||||
|
||||
verified = inspect_archive(
|
||||
archive,
|
||||
filename="secure.zip",
|
||||
password="correct horse",
|
||||
)
|
||||
self.assertTrue(verified.requires_password)
|
||||
self.assertTrue(verified.password_verified)
|
||||
|
||||
with self.assertRaisesRegex(ArchivePasswordError, "incorrect"):
|
||||
inspect_archive(
|
||||
archive,
|
||||
filename="secure.zip",
|
||||
password="wrong",
|
||||
)
|
||||
|
||||
def test_folder_selection_extracts_only_descendants(self) -> None:
|
||||
archive = _zip_bytes(
|
||||
{
|
||||
"selected/one.txt": b"one",
|
||||
"selected/two.txt": b"two",
|
||||
"other/three.txt": b"three",
|
||||
}
|
||||
)
|
||||
stored = SimpleNamespace(
|
||||
asset=SimpleNamespace(id="asset"),
|
||||
version=SimpleNamespace(id="version"),
|
||||
blob=SimpleNamespace(id="blob"),
|
||||
)
|
||||
with patch(
|
||||
"govoplan_files.backend.storage.archives.create_file_asset",
|
||||
return_value=stored,
|
||||
) as create_asset:
|
||||
result = extract_archive_upload(
|
||||
object(), # type: ignore[arg-type]
|
||||
tenant_id="tenant-1",
|
||||
owner_type="user",
|
||||
owner_id="user-1",
|
||||
user_id="user-1",
|
||||
archive_data=archive,
|
||||
filename="selection.zip",
|
||||
folder="imports",
|
||||
campaign_id=None,
|
||||
selected_paths=["selected"],
|
||||
)
|
||||
|
||||
self.assertEqual(2, len(result))
|
||||
self.assertEqual(
|
||||
{"imports/selected/one.txt", "imports/selected/two.txt"},
|
||||
{
|
||||
call.kwargs["display_path"]
|
||||
for call in create_asset.call_args_list
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,89 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from govoplan_core.auth import ApiPrincipal
|
||||
from govoplan_core.core.access import PrincipalRef
|
||||
from govoplan_core.core.files import ManagedArtifactStore, ManagedArtifactWriteRequest
|
||||
from govoplan_files.backend.capabilities import FilesArtifactStore
|
||||
|
||||
|
||||
def principal() -> ApiPrincipal:
|
||||
return ApiPrincipal(
|
||||
principal=PrincipalRef(
|
||||
account_id="account-1",
|
||||
membership_id="membership-1",
|
||||
tenant_id="tenant-1",
|
||||
scopes=frozenset({"files:file:upload"}),
|
||||
),
|
||||
account=object(),
|
||||
user=SimpleNamespace(id="user-1"),
|
||||
)
|
||||
|
||||
|
||||
class _Session:
|
||||
def query(self):
|
||||
raise AssertionError("not used")
|
||||
|
||||
def flush(self):
|
||||
raise AssertionError("not used")
|
||||
|
||||
|
||||
def stored():
|
||||
return SimpleNamespace(
|
||||
asset=SimpleNamespace(
|
||||
id="file-1",
|
||||
display_path="Generated/Templates/result.html",
|
||||
owner_type="user",
|
||||
),
|
||||
version=SimpleNamespace(
|
||||
id="version-1",
|
||||
filename_at_upload="result.html",
|
||||
content_type="text/html",
|
||||
size_bytes=6,
|
||||
checksum_sha256="0" * 64,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class FilesArtifactStoreTests(unittest.TestCase):
|
||||
def test_plain_write_uses_files_authority_and_returns_provider_neutral_ref(self) -> None:
|
||||
service = FilesArtifactStore()
|
||||
self.assertIsInstance(service, ManagedArtifactStore)
|
||||
request = ManagedArtifactWriteRequest(
|
||||
filename="result.html",
|
||||
payload=b"result",
|
||||
content_type="text/html",
|
||||
folder="Generated/Templates",
|
||||
metadata={"producer_module": "templates"},
|
||||
)
|
||||
with patch("govoplan_files.backend.capabilities.create_file_asset", return_value=stored()) as create:
|
||||
result = service.store_artifact(_Session(), principal(), request=request)
|
||||
self.assertEqual("file-1", result.file_asset_id)
|
||||
self.assertEqual("version-1", result.file_version_id)
|
||||
self.assertEqual("tenant-1", create.call_args.kwargs["tenant_id"])
|
||||
self.assertEqual("user-1", create.call_args.kwargs["owner_id"])
|
||||
self.assertEqual(b"result", create.call_args.kwargs["data"])
|
||||
|
||||
def test_idempotent_write_uses_source_provenance_without_payload_metadata(self) -> None:
|
||||
request = ManagedArtifactWriteRequest(
|
||||
filename="result.html",
|
||||
payload=b"secret body",
|
||||
content_type="text/html",
|
||||
idempotency_key="render-1",
|
||||
metadata={"producer_module": "templates", "output_sha256": "a" * 64},
|
||||
)
|
||||
with patch(
|
||||
"govoplan_files.backend.capabilities.sync_file_asset_from_source",
|
||||
return_value=(stored(), "unchanged", None),
|
||||
) as sync:
|
||||
FilesArtifactStore().store_artifact(_Session(), principal(), request=request)
|
||||
metadata = sync.call_args.kwargs["metadata"]
|
||||
self.assertEqual("render-1", metadata["source_provenance"]["external_id"])
|
||||
self.assertNotIn("secret body", str(metadata))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,219 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
import unittest
|
||||
|
||||
from govoplan_core.core.configuration_packages import (
|
||||
ConfigurationPackageFragment,
|
||||
ConfigurationPreflightContext,
|
||||
)
|
||||
from govoplan_core.core.infrastructure_capabilities import (
|
||||
infrastructure_capability_receipt_from_mapping,
|
||||
)
|
||||
from govoplan_files.backend.configuration_provider import (
|
||||
FILES_CONFIGURATION_CAPABILITY,
|
||||
FilesConfigurationProvider,
|
||||
)
|
||||
from govoplan_files.backend.manifest import manifest
|
||||
|
||||
|
||||
def _receipt(*, source: str = "host-local", state: str = "configured"):
|
||||
endpoint = (
|
||||
{"kind": "filesystem", "reference": "volume:files-data"}
|
||||
if source == "host-local"
|
||||
else {"scheme": "http", "host": "garage", "port": 3900}
|
||||
)
|
||||
secret_refs = (
|
||||
[]
|
||||
if source == "host-local"
|
||||
else [
|
||||
"env:FILE_STORAGE_S3_ACCESS_KEY_ID",
|
||||
"env:FILE_STORAGE_S3_SECRET_ACCESS_KEY",
|
||||
"env:GARAGE_RPC_SECRET",
|
||||
]
|
||||
)
|
||||
return infrastructure_capability_receipt_from_mapping(
|
||||
{
|
||||
"schema_version": 1,
|
||||
"installation_id": "files-provider-test",
|
||||
"profile": "evaluation",
|
||||
"capabilities": [
|
||||
{
|
||||
"id": "files.storage",
|
||||
"label": "Managed file content storage",
|
||||
"state": state,
|
||||
"source": source,
|
||||
"detail": "Deployment-owned storage binding.",
|
||||
"endpoint": endpoint,
|
||||
"secret_refs": secret_refs,
|
||||
"dependent_modules": ["files"],
|
||||
}
|
||||
],
|
||||
"post_install_tasks": [],
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _local_settings(**overrides):
|
||||
values = {
|
||||
"file_storage_backend": "local",
|
||||
"file_storage_local_root": "/var/lib/govoplan/files",
|
||||
}
|
||||
values.update(overrides)
|
||||
return SimpleNamespace(**values)
|
||||
|
||||
|
||||
def _s3_settings(**overrides):
|
||||
values = {
|
||||
"file_storage_backend": "s3",
|
||||
"file_storage_s3_endpoint_url": "http://garage:3900",
|
||||
"file_storage_s3_bucket": "files",
|
||||
"file_storage_s3_deployment_managed": True,
|
||||
"file_storage_s3_endpoint_trusted": False,
|
||||
}
|
||||
values.update(overrides)
|
||||
return SimpleNamespace(**values)
|
||||
|
||||
|
||||
class FilesConfigurationProviderTests(unittest.TestCase):
|
||||
def test_provider_is_registered(self) -> None:
|
||||
self.assertIn(FILES_CONFIGURATION_CAPABILITY, manifest.capability_factories)
|
||||
|
||||
def test_matching_local_storage_is_an_idempotent_noop(self) -> None:
|
||||
provider = FilesConfigurationProvider(
|
||||
settings=_local_settings(),
|
||||
environment={},
|
||||
)
|
||||
context = ConfigurationPreflightContext(
|
||||
infrastructure_receipt=_receipt(),
|
||||
)
|
||||
fragment = ConfigurationPackageFragment(
|
||||
module_id="files",
|
||||
fragment_type="managed_storage",
|
||||
fragment_id="files-storage",
|
||||
payload={
|
||||
"expected_backend": "local",
|
||||
"expected_source": "host-local",
|
||||
},
|
||||
)
|
||||
|
||||
first = provider.preflight(fragment, context)
|
||||
applied = provider.apply(fragment, {}, context)
|
||||
second = provider.preflight(fragment, context)
|
||||
|
||||
self.assertEqual("skip", first.plan[0].action)
|
||||
self.assertEqual("skip", second.plan[0].action)
|
||||
self.assertEqual((), applied.diagnostics)
|
||||
self.assertEqual({}, applied.created_refs)
|
||||
self.assertEqual({}, applied.updated_refs)
|
||||
|
||||
def test_runtime_backend_mismatch_blocks_without_rewriting_settings(self) -> None:
|
||||
settings = _local_settings()
|
||||
provider = FilesConfigurationProvider(settings=settings, environment={})
|
||||
context = ConfigurationPreflightContext(
|
||||
infrastructure_receipt=_receipt(source="installer-managed-garage"),
|
||||
)
|
||||
fragment = ConfigurationPackageFragment(
|
||||
module_id="files",
|
||||
fragment_type="managed_storage",
|
||||
payload={},
|
||||
)
|
||||
|
||||
result = provider.preflight(fragment, context)
|
||||
|
||||
self.assertEqual("blocked", result.plan[0].action)
|
||||
self.assertIn(
|
||||
"files_storage_runtime_mismatch",
|
||||
{item.code for item in result.diagnostics},
|
||||
)
|
||||
self.assertEqual("local", settings.file_storage_backend)
|
||||
|
||||
def test_garage_binding_requires_only_files_secret_references(self) -> None:
|
||||
fragment = ConfigurationPackageFragment(
|
||||
module_id="files",
|
||||
fragment_type="managed_storage",
|
||||
payload={},
|
||||
)
|
||||
context = ConfigurationPreflightContext(
|
||||
infrastructure_receipt=_receipt(source="installer-managed-garage"),
|
||||
)
|
||||
missing = FilesConfigurationProvider(
|
||||
settings=_s3_settings(),
|
||||
environment={},
|
||||
).preflight(fragment, context)
|
||||
available = FilesConfigurationProvider(
|
||||
settings=_s3_settings(),
|
||||
environment={
|
||||
"FILE_STORAGE_S3_ACCESS_KEY_ID": "reference-resolved",
|
||||
"FILE_STORAGE_S3_SECRET_ACCESS_KEY": "reference-resolved",
|
||||
},
|
||||
).preflight(fragment, context)
|
||||
|
||||
self.assertEqual("blocked", missing.plan[0].action)
|
||||
self.assertEqual(
|
||||
2,
|
||||
sum(
|
||||
item.code == "files_storage_secret_reference_unresolved"
|
||||
for item in missing.diagnostics
|
||||
),
|
||||
)
|
||||
self.assertEqual("skip", available.plan[0].action)
|
||||
|
||||
def test_operator_supplied_s3_requires_explicit_endpoint_trust(self) -> None:
|
||||
receipt = _receipt(source="operator-supplied-s3", state="externally_supplied")
|
||||
fragment = ConfigurationPackageFragment(
|
||||
module_id="files",
|
||||
fragment_type="managed_storage",
|
||||
payload={},
|
||||
)
|
||||
context = ConfigurationPreflightContext(infrastructure_receipt=receipt)
|
||||
environment = {
|
||||
"FILE_STORAGE_S3_ACCESS_KEY_ID": "reference-resolved",
|
||||
"FILE_STORAGE_S3_SECRET_ACCESS_KEY": "reference-resolved",
|
||||
}
|
||||
|
||||
blocked = FilesConfigurationProvider(
|
||||
settings=_s3_settings(
|
||||
file_storage_s3_deployment_managed=False,
|
||||
file_storage_s3_endpoint_trusted=False,
|
||||
),
|
||||
environment=environment,
|
||||
).preflight(fragment, context)
|
||||
ready = FilesConfigurationProvider(
|
||||
settings=_s3_settings(
|
||||
file_storage_s3_deployment_managed=False,
|
||||
file_storage_s3_endpoint_trusted=True,
|
||||
),
|
||||
environment=environment,
|
||||
).preflight(fragment, context)
|
||||
|
||||
self.assertEqual("blocked", blocked.plan[0].action)
|
||||
self.assertIn(
|
||||
"files_storage_trust_boundary_missing",
|
||||
{item.code for item in blocked.diagnostics},
|
||||
)
|
||||
self.assertEqual("skip", ready.plan[0].action)
|
||||
|
||||
def test_inline_storage_secret_is_rejected(self) -> None:
|
||||
provider = FilesConfigurationProvider(
|
||||
settings=_local_settings(),
|
||||
environment={},
|
||||
)
|
||||
result = provider.preflight(
|
||||
ConfigurationPackageFragment(
|
||||
module_id="files",
|
||||
fragment_type="managed_storage",
|
||||
payload={"secret_access_key": "inline"},
|
||||
),
|
||||
ConfigurationPreflightContext(infrastructure_receipt=_receipt()),
|
||||
)
|
||||
|
||||
self.assertEqual("blocked", result.plan[0].action)
|
||||
self.assertIn(
|
||||
"files_configuration_secret_forbidden",
|
||||
{item.code for item in result.diagnostics},
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -12,7 +12,8 @@ from govoplan_core.core.change_sequence import ChangeSequenceEntry
|
||||
from govoplan_core.db.base import Base
|
||||
from govoplan_core.security.secrets import encrypt_secret
|
||||
from govoplan_files.backend.db.models import FileConnectorCredential, FileConnectorProfile
|
||||
from govoplan_files.backend.router import deactivate_connector_credential, deactivate_connector_profile
|
||||
from govoplan_files.backend.routes.connector_profiles import deactivate_connector_profile
|
||||
from govoplan_files.backend.routes.connector_settings import deactivate_connector_credential
|
||||
|
||||
|
||||
class Principal:
|
||||
|
||||
@@ -10,7 +10,7 @@ from unittest.mock import MagicMock
|
||||
|
||||
from fastapi import HTTPException
|
||||
|
||||
from govoplan_files.backend.router import discover_connector_endpoint
|
||||
from govoplan_files.backend.routes.connector_settings import discover_connector_endpoint
|
||||
from govoplan_files.backend.schemas import FileConnectorDiscoveryRequest
|
||||
from govoplan_files.backend.storage.connector_browse import ConnectorBrowseError, _profile_password, _s3_verify
|
||||
from govoplan_files.backend.storage.connector_deployment import (
|
||||
@@ -182,7 +182,7 @@ class ConnectorDiscoveryBoundaryTests(unittest.TestCase):
|
||||
credential_mode="basic",
|
||||
credentials={"username": "admin", "password_env": "MASTER_KEY_B64"},
|
||||
)
|
||||
with patch("govoplan_files.backend.router.browse_connector_profile") as browse, self.assertRaises(
|
||||
with patch("govoplan_files.backend.routes.connector_settings.browse_connector_profile") as browse, self.assertRaises(
|
||||
HTTPException
|
||||
) as raised:
|
||||
discover_connector_endpoint(payload, session=object(), principal=self._principal()) # type: ignore[arg-type]
|
||||
@@ -212,10 +212,10 @@ class ConnectorDiscoveryBoundaryTests(unittest.TestCase):
|
||||
events.append("io")
|
||||
return []
|
||||
|
||||
with patch("govoplan_files.backend.router._ensure_connector_configuration_allowed", side_effect=ensure), patch(
|
||||
"govoplan_files.backend.router._audit_connector_discovery_attempt",
|
||||
with patch("govoplan_files.backend.routes.connector_settings._ensure_connector_configuration_allowed", side_effect=ensure), patch(
|
||||
"govoplan_files.backend.routes.connector_settings._audit_connector_discovery_attempt",
|
||||
side_effect=audit,
|
||||
), patch("govoplan_files.backend.router.browse_connector_profile", side_effect=browse):
|
||||
), patch("govoplan_files.backend.routes.connector_settings.browse_connector_profile", side_effect=browse):
|
||||
response = discover_connector_endpoint(payload, session=object(), principal=self._principal()) # type: ignore[arg-type]
|
||||
|
||||
self.assertEqual("usable", response.status)
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
|
||||
from govoplan_files.backend.storage.connector_policy import (
|
||||
ConnectorAccessRequest,
|
||||
ConnectorPolicySource,
|
||||
connector_policy_decision,
|
||||
)
|
||||
|
||||
|
||||
class ConnectorPolicyPatternTests(unittest.TestCase):
|
||||
def test_resource_reference_allow_patterns_match(self) -> None:
|
||||
decision = connector_policy_decision(
|
||||
ConnectorAccessRequest(
|
||||
connector_id="dev-smb",
|
||||
credential_id="credential-primary",
|
||||
provider="smb",
|
||||
),
|
||||
(
|
||||
ConnectorPolicySource(
|
||||
scope_type="tenant",
|
||||
scope_id="tenant-1",
|
||||
label="Tenant",
|
||||
policy={
|
||||
"allow": {
|
||||
"connectors": ["dev-*"],
|
||||
"credentials": ["credential-*"],
|
||||
"providers": ["s?b"],
|
||||
}
|
||||
},
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
self.assertTrue(decision.allowed)
|
||||
|
||||
def test_resource_reference_deny_patterns_take_precedence(self) -> None:
|
||||
decision = connector_policy_decision(
|
||||
ConnectorAccessRequest(
|
||||
connector_id="dev-smb",
|
||||
credential_id="credential-legacy",
|
||||
provider="smb",
|
||||
),
|
||||
(
|
||||
ConnectorPolicySource(
|
||||
scope_type="tenant",
|
||||
scope_id="tenant-1",
|
||||
label="Tenant",
|
||||
policy={
|
||||
"allow": {"connectors": ["dev-*"]},
|
||||
"deny": {"credentials": ["*-legacy"]},
|
||||
},
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
self.assertFalse(decision.allowed)
|
||||
self.assertEqual(
|
||||
("connector_policy_denylist",),
|
||||
decision.requirements,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,5 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
@@ -8,6 +9,7 @@ from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from govoplan_access.backend.db import models as access_models # noqa: F401 - resolve Files user foreign keys
|
||||
from govoplan_core.db.base import Base
|
||||
from govoplan_core.security.credential_envelopes import CredentialEnvelope
|
||||
from govoplan_core.security.secrets import decrypt_secret, encrypt_secret
|
||||
from govoplan_files.backend.db.models import FileConnectorProfile
|
||||
from govoplan_files.backend.storage.common import FileStorageError
|
||||
@@ -20,7 +22,13 @@ from govoplan_files.backend.storage.connector_profile_store import (
|
||||
class ConnectorProfileStoreUpdateTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.engine = create_engine("sqlite:///:memory:")
|
||||
Base.metadata.create_all(self.engine, tables=[FileConnectorProfile.__table__])
|
||||
Base.metadata.create_all(
|
||||
self.engine,
|
||||
tables=[
|
||||
FileConnectorProfile.__table__,
|
||||
CredentialEnvelope.__table__,
|
||||
],
|
||||
)
|
||||
self.session = sessionmaker(bind=self.engine)()
|
||||
self.row = FileConnectorProfile(
|
||||
id="profile-1",
|
||||
@@ -171,6 +179,50 @@ class ConnectorProfileStoreUpdateTests(unittest.TestCase):
|
||||
self.assertEqual([self.row.id], [profile.id for profile in profiles])
|
||||
self.assertEqual("original-password", profiles[0].password_value)
|
||||
|
||||
def test_reusable_credential_resolves_and_revocation_keeps_profile_visible(self) -> None:
|
||||
credential = CredentialEnvelope(
|
||||
id="shared-credential",
|
||||
tenant_id="tenant-1",
|
||||
scope_type="tenant",
|
||||
scope_id="tenant-1",
|
||||
name="Shared WebDAV login",
|
||||
credential_kind="username_password",
|
||||
public_data={"username": "ada"},
|
||||
secret_data_encrypted=encrypt_secret(json.dumps({"password": "secret"})),
|
||||
secret_keys=["password"],
|
||||
allowed_modules=["files"],
|
||||
allowed_server_refs=[],
|
||||
inherit_to_lower_scopes=True,
|
||||
is_active=True,
|
||||
revision="revision-1",
|
||||
)
|
||||
self.row.credential_profile_id = "credential-envelope:shared-credential"
|
||||
self.row.username = None
|
||||
self.row.password_encrypted = None
|
||||
self.row.token_encrypted = None
|
||||
self.session.add(credential)
|
||||
self.session.commit()
|
||||
|
||||
profile = list_database_connector_profiles(
|
||||
self.session,
|
||||
tenant_id="tenant-1",
|
||||
)[0]
|
||||
|
||||
self.assertEqual("ada", profile.username)
|
||||
self.assertEqual("secret", profile.password_value)
|
||||
self.assertTrue(profile.credentials_configured)
|
||||
|
||||
credential.is_active = False
|
||||
self.session.commit()
|
||||
profile = list_database_connector_profiles(
|
||||
self.session,
|
||||
tenant_id="tenant-1",
|
||||
)[0]
|
||||
|
||||
self.assertIsNone(profile.password_value)
|
||||
self.assertFalse(profile.credentials_configured)
|
||||
self.assertTrue(profile.metadata["credential_unavailable"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -5,7 +5,7 @@ from datetime import UTC, datetime
|
||||
from unittest.mock import patch
|
||||
|
||||
from govoplan_files.backend.storage.connector_browse import ConnectorBrowseError, _smb_location, browse_connector_profile
|
||||
from govoplan_files.backend.storage.connector_imports import ConnectorImportError, read_connector_file
|
||||
from govoplan_files.backend.storage.connector_imports import read_connector_file
|
||||
from govoplan_files.backend.storage.connector_profiles import ConnectorProfile, connector_profiles_from_payload
|
||||
from govoplan_files.backend.storage.connector_providers import connector_provider_descriptors
|
||||
|
||||
@@ -23,6 +23,10 @@ class FakeS3Client:
|
||||
self.list_objects_request: dict[str, object] | None = None
|
||||
self.head_request: dict[str, object] | None = None
|
||||
self.get_request: dict[str, object] | None = None
|
||||
self.closed = False
|
||||
|
||||
def close(self) -> None:
|
||||
self.closed = True
|
||||
|
||||
def list_objects_v2(self, **kwargs: object) -> dict[str, object]:
|
||||
self.list_objects_request = dict(kwargs)
|
||||
@@ -75,42 +79,39 @@ def s3_profile(**overrides: object) -> ConnectorProfile:
|
||||
|
||||
|
||||
class ConnectorProviderTests(unittest.TestCase):
|
||||
def test_smb_sdk_transport_fails_closed_before_client_creation_in_all_modes(self) -> None:
|
||||
def test_smb_browse_uses_the_files_owned_pinned_connection_cache(self) -> None:
|
||||
profile = ConnectorProfile(
|
||||
id="smb",
|
||||
label="SMB",
|
||||
provider="smb",
|
||||
endpoint_url="smb://files.example.test/share",
|
||||
)
|
||||
for allow_private, address in ((False, "93.184.216.34"), (True, "10.0.0.5")):
|
||||
with self.subTest(allow_private=allow_private), patch.dict(
|
||||
"os.environ",
|
||||
{
|
||||
"APP_ENV": "production",
|
||||
"GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS": str(allow_private).lower(),
|
||||
},
|
||||
), patch(
|
||||
"govoplan_core.security.outbound_http.socket.getaddrinfo",
|
||||
return_value=[(2, 1, 6, "", (address, 445))],
|
||||
), patch("govoplan_files.backend.storage.connector_browse._smbclient_module") as sdk, self.assertRaisesRegex(
|
||||
ConnectorBrowseError,
|
||||
"redirects/referrals.*DNS/IP pinning",
|
||||
):
|
||||
browse_connector_profile(profile, path="")
|
||||
sdk.assert_not_called()
|
||||
with patch.dict(
|
||||
"os.environ",
|
||||
{"APP_ENV": "production", "GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS": "false"},
|
||||
), patch(
|
||||
"govoplan_core.security.outbound_http.socket.getaddrinfo",
|
||||
return_value=[(2, 1, 6, "", ("93.184.216.34", 445))],
|
||||
), patch("govoplan_files.backend.storage.connector_browse._smbclient_module") as sdk:
|
||||
sdk.return_value.scandir.return_value.__enter__.return_value = iter(())
|
||||
self.assertEqual([], browse_connector_profile(profile, path=""))
|
||||
|
||||
def test_smb_explicit_ip_still_fails_closed_because_the_sdk_may_follow_referrals(self) -> None:
|
||||
kwargs = sdk.return_value.scandir.call_args.kwargs
|
||||
self.assertIsInstance(kwargs["connection_cache"], dict)
|
||||
self.assertTrue(kwargs["require_signing"])
|
||||
|
||||
def test_smb_endpoint_preflight_applies_private_network_policy(self) -> None:
|
||||
profile = ConnectorProfile(id="smb", label="SMB", provider="smb", endpoint_url="smb://10.0.0.5/share")
|
||||
with patch.dict(
|
||||
"os.environ",
|
||||
{"APP_ENV": "production", "GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS": "true"},
|
||||
{"APP_ENV": "production", "GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS": "false"},
|
||||
), patch(
|
||||
"govoplan_core.security.outbound_http.socket.getaddrinfo",
|
||||
return_value=[(2, 1, 6, "", ("10.0.0.5", 445))],
|
||||
), self.assertRaisesRegex(ConnectorBrowseError, "redirects/referrals.*DNS/IP pinning"):
|
||||
), self.assertRaisesRegex(ConnectorBrowseError, "non-public network"):
|
||||
_smb_location(profile)
|
||||
|
||||
def test_smb_import_surfaces_fail_closed_policy_as_an_import_error(self) -> None:
|
||||
def test_smb_import_uses_the_same_pinned_connection_cache(self) -> None:
|
||||
profile = ConnectorProfile(id="smb", label="SMB", provider="smb", endpoint_url="smb://10.0.0.5/share")
|
||||
with patch.dict(
|
||||
"os.environ",
|
||||
@@ -118,12 +119,13 @@ class ConnectorProviderTests(unittest.TestCase):
|
||||
), patch(
|
||||
"govoplan_core.security.outbound_http.socket.getaddrinfo",
|
||||
return_value=[(2, 1, 6, "", ("10.0.0.5", 445))],
|
||||
), patch("govoplan_files.backend.storage.connector_imports._smbclient_module") as sdk, self.assertRaisesRegex(
|
||||
ConnectorImportError,
|
||||
"redirects/referrals.*DNS/IP pinning",
|
||||
):
|
||||
read_connector_file(profile, library_id="", path="notice.txt", max_bytes=1024)
|
||||
sdk.assert_not_called()
|
||||
), patch("govoplan_files.backend.storage.connector_imports._smbclient_module") as sdk:
|
||||
sdk.return_value.stat.return_value.st_size = 4
|
||||
sdk.return_value.open_file.return_value.__enter__.return_value.read.return_value = b"test"
|
||||
downloaded = read_connector_file(profile, library_id="", path="notice.txt", max_bytes=1024)
|
||||
|
||||
self.assertEqual(b"test", downloaded.data)
|
||||
self.assertIsInstance(sdk.return_value.stat.call_args.kwargs["connection_cache"], dict)
|
||||
|
||||
def test_provider_descriptors_include_s3_and_reserved_microsoft_providers(self) -> None:
|
||||
descriptors = {descriptor.provider: descriptor for descriptor in connector_provider_descriptors()}
|
||||
@@ -172,6 +174,7 @@ class ConnectorProviderTests(unittest.TestCase):
|
||||
self.assertEqual("govoplan:root/report.xlsx", items[1].external_id)
|
||||
self.assertEqual("root/report.xlsx", items[1].metadata["key"])
|
||||
self.assertEqual("next-page", items[1].metadata["next_continuation_token"])
|
||||
self.assertTrue(client.closed)
|
||||
|
||||
def test_s3_browse_lists_buckets_when_profile_has_no_bucket(self) -> None:
|
||||
client = FakeS3Client()
|
||||
@@ -181,27 +184,33 @@ class ConnectorProviderTests(unittest.TestCase):
|
||||
|
||||
self.assertEqual(["archive"], [item.path for item in items])
|
||||
|
||||
def test_s3_sdk_transport_fails_closed_before_client_creation_in_private_mode(self) -> None:
|
||||
def test_s3_client_is_constructed_through_the_pinned_transport(self) -> None:
|
||||
with patch.dict(
|
||||
"os.environ",
|
||||
{"APP_ENV": "production", "GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS": "true"},
|
||||
), patch(
|
||||
"govoplan_core.security.outbound_http.socket.getaddrinfo",
|
||||
return_value=[(2, 1, 6, "", ("127.0.0.1", 9000))],
|
||||
), patch("govoplan_files.backend.storage.connector_browse.import_module") as importer, self.assertRaisesRegex(
|
||||
ConnectorBrowseError,
|
||||
"until that transport supports.*DNS/IP pinning",
|
||||
):
|
||||
), patch(
|
||||
"govoplan_files.backend.storage.connector_browse.create_pinned_s3_client",
|
||||
return_value=FakeS3Client(),
|
||||
) as factory:
|
||||
browse_connector_profile(s3_profile(), path="")
|
||||
importer.assert_not_called()
|
||||
|
||||
def test_s3_sdk_endpoint_discovery_fails_closed(self) -> None:
|
||||
with patch("govoplan_files.backend.storage.connector_browse.import_module") as importer, self.assertRaisesRegex(
|
||||
ConnectorBrowseError,
|
||||
"endpoint discovery.*cannot guarantee.*DNS/IP pinning",
|
||||
):
|
||||
kwargs = factory.call_args.kwargs
|
||||
self.assertEqual("http://127.0.0.1:9000", kwargs["endpoint_url"])
|
||||
self.assertEqual("access-key", kwargs["aws_access_key_id"])
|
||||
self.assertEqual("secret-key", kwargs["aws_secret_access_key"])
|
||||
self.assertEqual({}, kwargs["config"].proxies)
|
||||
|
||||
def test_s3_endpoint_discovery_uses_the_same_pinned_transport(self) -> None:
|
||||
with patch(
|
||||
"govoplan_files.backend.storage.connector_browse.create_pinned_s3_client",
|
||||
return_value=FakeS3Client(),
|
||||
) as factory:
|
||||
browse_connector_profile(s3_profile(endpoint_url=None), path="")
|
||||
importer.assert_not_called()
|
||||
|
||||
self.assertNotIn("endpoint_url", factory.call_args.kwargs)
|
||||
|
||||
def test_s3_import_downloads_object_and_preserves_remote_identity(self) -> None:
|
||||
client = FakeS3Client()
|
||||
@@ -217,6 +226,7 @@ class ConnectorProviderTests(unittest.TestCase):
|
||||
self.assertEqual("govoplan:root/report.txt", downloaded.external_id)
|
||||
self.assertEqual("s3://govoplan/root/report.txt", downloaded.external_url)
|
||||
self.assertEqual("checksum", downloaded.metadata["checksum_sha256"])
|
||||
self.assertTrue(client.closed)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
from govoplan_files.backend.storage.common import FileStorageError
|
||||
from govoplan_files.backend.storage.connector_profiles import ConnectorProfile
|
||||
from govoplan_files.backend.storage.connector_spaces import (
|
||||
soft_delete_connector_space,
|
||||
validate_connector_space_write_mode,
|
||||
)
|
||||
|
||||
|
||||
class ConnectorSpaceDeletionTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.session = Mock()
|
||||
self.space = SimpleNamespace(
|
||||
tenant_id="tenant-1",
|
||||
owner_type="user",
|
||||
owner_user_id="owner-1",
|
||||
owner_group_id=None,
|
||||
deleted_at=None,
|
||||
is_active=True,
|
||||
)
|
||||
|
||||
@patch("govoplan_files.backend.storage.connector_spaces.utcnow")
|
||||
@patch("govoplan_files.backend.storage.connector_spaces.ensure_owner_access")
|
||||
def test_owner_can_soft_delete_local_space_definition(
|
||||
self, ensure_owner_access: Mock, utcnow: Mock
|
||||
) -> None:
|
||||
deleted_at = object()
|
||||
utcnow.return_value = deleted_at
|
||||
|
||||
result = soft_delete_connector_space(
|
||||
self.session, self.space, user_id="owner-1"
|
||||
)
|
||||
|
||||
self.assertIs(result, self.space)
|
||||
self.assertIs(deleted_at, self.space.deleted_at)
|
||||
self.assertFalse(self.space.is_active)
|
||||
ensure_owner_access.assert_called_once_with(
|
||||
self.session,
|
||||
tenant_id="tenant-1",
|
||||
owner_type="user",
|
||||
owner_id="owner-1",
|
||||
user_id="owner-1",
|
||||
is_admin=False,
|
||||
)
|
||||
self.session.add.assert_called_once_with(self.space)
|
||||
self.session.flush.assert_called_once_with()
|
||||
|
||||
@patch(
|
||||
"govoplan_files.backend.storage.connector_spaces.ensure_owner_access",
|
||||
side_effect=FileStorageError("File space access denied"),
|
||||
)
|
||||
def test_inaccessible_space_is_blocked_without_mutation(
|
||||
self, _ensure_owner_access: Mock
|
||||
) -> None:
|
||||
with self.assertRaisesRegex(FileStorageError, "access denied"):
|
||||
soft_delete_connector_space(
|
||||
self.session, self.space, user_id="other-user"
|
||||
)
|
||||
|
||||
self.assertIsNone(self.space.deleted_at)
|
||||
self.assertTrue(self.space.is_active)
|
||||
self.session.add.assert_not_called()
|
||||
self.session.flush.assert_not_called()
|
||||
|
||||
def test_read_only_mode_does_not_require_remote_write_capability(self) -> None:
|
||||
profile = ConnectorProfile(
|
||||
id="webdav-read",
|
||||
label="Read only",
|
||||
provider="webdav",
|
||||
capabilities=("browse", "import"),
|
||||
)
|
||||
|
||||
validate_connector_space_write_mode(profile, read_only=True)
|
||||
|
||||
def test_two_way_mode_requires_explicit_supported_write_capability(self) -> None:
|
||||
missing_capability = ConnectorProfile(
|
||||
id="s3-read",
|
||||
label="S3 read only",
|
||||
provider="s3",
|
||||
capabilities=("browse",),
|
||||
)
|
||||
unsupported_provider = ConnectorProfile(
|
||||
id="webdav-write",
|
||||
label="WebDAV write",
|
||||
provider="webdav",
|
||||
capabilities=("write",),
|
||||
)
|
||||
|
||||
for profile in (missing_capability, unsupported_provider):
|
||||
with self.subTest(profile=profile.id):
|
||||
with self.assertRaisesRegex(FileStorageError, "requires an S3"):
|
||||
validate_connector_space_write_mode(profile, read_only=False)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -10,6 +10,9 @@ from govoplan_files.backend.storage.connector_visibility import (
|
||||
connector_profile_usable_for_import,
|
||||
visible_connector_profiles_for_actor,
|
||||
)
|
||||
from govoplan_files.backend.storage.connector_providers import (
|
||||
connector_provider_descriptors,
|
||||
)
|
||||
|
||||
|
||||
def _profile(
|
||||
@@ -168,12 +171,16 @@ class ConnectorVisibilityTests(unittest.TestCase):
|
||||
self,
|
||||
) -> None:
|
||||
self.assertTrue(connector_profile_usable_for_import(_profile("webdav")))
|
||||
self.assertFalse(
|
||||
connector_profile_usable_for_import(_profile("s3", provider="s3"))
|
||||
)
|
||||
self.assertFalse(
|
||||
connector_profile_usable_for_import(_profile("smb", provider="smb"))
|
||||
)
|
||||
descriptors = {
|
||||
item.provider: item for item in connector_provider_descriptors()
|
||||
}
|
||||
for provider in ("s3", "smb"):
|
||||
self.assertEqual(
|
||||
descriptors[provider].installed,
|
||||
connector_profile_usable_for_import(
|
||||
_profile(provider, provider=provider)
|
||||
),
|
||||
)
|
||||
self.assertFalse(
|
||||
connector_profile_usable_for_import(
|
||||
_profile("sharepoint", provider="sharepoint")
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
import hashlib
|
||||
import unittest
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from govoplan_core.core.encryption import (
|
||||
CAPABILITY_ENCRYPTION_CONTENT_CIPHER,
|
||||
ContentProtectionEnvelope,
|
||||
ProtectedContent,
|
||||
)
|
||||
from govoplan_files.backend.db.models import FileBlob
|
||||
from govoplan_files.backend.runtime import configure_runtime
|
||||
from govoplan_files.backend.storage.common import FileStorageError
|
||||
from govoplan_files.backend.storage.content_protection import protect_blob_content
|
||||
from govoplan_files.backend.storage.integrity import read_verified_blob_bytes
|
||||
|
||||
|
||||
class Registry:
|
||||
def __init__(self, capability=None) -> None:
|
||||
self.capability_value = capability
|
||||
|
||||
def has_capability(self, name: str) -> bool:
|
||||
return (
|
||||
name == CAPABILITY_ENCRYPTION_CONTENT_CIPHER
|
||||
and self.capability_value is not None
|
||||
)
|
||||
|
||||
def capability(self, _name: str):
|
||||
return self.capability_value
|
||||
|
||||
|
||||
class Cipher:
|
||||
def protect_content(self, _session, *, request):
|
||||
ciphertext = b"protected:" + request.plaintext
|
||||
envelope = ContentProtectionEnvelope(
|
||||
envelope_id="envelope-file-1",
|
||||
tenant_id=request.tenant_id,
|
||||
owner_module=request.owner_module,
|
||||
resource_type=request.resource_type,
|
||||
resource_id=request.resource_id,
|
||||
profile_kind="server_envelope",
|
||||
profile_id=request.profile_id,
|
||||
provider_id="test",
|
||||
vault_id=request.vault_id,
|
||||
key_version=1,
|
||||
algorithm_suite="AES-256-GCM",
|
||||
ciphertext_ref=request.ciphertext_ref,
|
||||
ciphertext_digest=f"sha256:{hashlib.sha256(ciphertext).hexdigest()}",
|
||||
authenticated_context_digest="sha256:context",
|
||||
state="active",
|
||||
created_at=datetime.now(tz=UTC),
|
||||
wrapped_key_refs=("wrapped:1",),
|
||||
)
|
||||
return ProtectedContent(envelope=envelope, ciphertext=ciphertext)
|
||||
|
||||
def unprotect_content(self, _session, *, request):
|
||||
if not request.ciphertext.startswith(b"protected:"):
|
||||
raise ValueError("invalid fixture ciphertext")
|
||||
return request.ciphertext.removeprefix(b"protected:")
|
||||
|
||||
def execute_rewrap(self, *_args, **_kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
def prepare_reencryption(self, *_args, **_kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class Backend:
|
||||
def __init__(self, data: bytes) -> None:
|
||||
self.data = data
|
||||
|
||||
def get_bytes(self, _key: str) -> bytes:
|
||||
return self.data
|
||||
|
||||
|
||||
class FileContentProtectionTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.engine = create_engine("sqlite+pysqlite:///:memory:")
|
||||
FileBlob.__table__.create(self.engine)
|
||||
self.session = Session(self.engine)
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self.session.close()
|
||||
self.engine.dispose()
|
||||
|
||||
def test_encrypted_blob_is_opened_after_stored_object_integrity_check(self) -> None:
|
||||
configure_runtime(registry=Registry(Cipher()), settings=object())
|
||||
plaintext = b"monthly records"
|
||||
protected = protect_blob_content(
|
||||
self.session,
|
||||
tenant_id="tenant-1",
|
||||
blob_id="blob-1",
|
||||
vault_id="vault-1",
|
||||
ciphertext_ref="tenants/tenant-1/files/blob-1",
|
||||
plaintext=plaintext,
|
||||
actor_id="account-1",
|
||||
content_type="text/plain",
|
||||
)
|
||||
blob = FileBlob(
|
||||
id="blob-1",
|
||||
tenant_id="tenant-1",
|
||||
storage_backend="test",
|
||||
storage_key="tenants/tenant-1/files/blob-1",
|
||||
checksum_sha256=hashlib.sha256(plaintext).hexdigest(),
|
||||
size_bytes=len(plaintext),
|
||||
protection_discriminator="vault:vault-1",
|
||||
encryption_envelope_id=protected.envelope.envelope_id,
|
||||
storage_checksum_sha256=hashlib.sha256(protected.ciphertext).hexdigest(),
|
||||
storage_size_bytes=len(protected.ciphertext),
|
||||
content_type="text/plain",
|
||||
ref_count=1,
|
||||
integrity_status="verified",
|
||||
)
|
||||
self.session.add(blob)
|
||||
self.session.flush()
|
||||
self.assertEqual(
|
||||
plaintext,
|
||||
read_verified_blob_bytes(blob, backend=Backend(protected.ciphertext)),
|
||||
)
|
||||
|
||||
def test_encrypted_blob_fails_closed_without_encryption_capability(self) -> None:
|
||||
configure_runtime(registry=Registry(), settings=object())
|
||||
ciphertext = b"protected:monthly records"
|
||||
blob = FileBlob(
|
||||
id="blob-1",
|
||||
tenant_id="tenant-1",
|
||||
storage_backend="test",
|
||||
storage_key="tenants/tenant-1/files/blob-1",
|
||||
checksum_sha256=hashlib.sha256(b"monthly records").hexdigest(),
|
||||
size_bytes=len(b"monthly records"),
|
||||
protection_discriminator="vault:vault-1",
|
||||
encryption_envelope_id="envelope-file-1",
|
||||
storage_checksum_sha256=hashlib.sha256(ciphertext).hexdigest(),
|
||||
storage_size_bytes=len(ciphertext),
|
||||
ref_count=1,
|
||||
integrity_status="verified",
|
||||
)
|
||||
self.session.add(blob)
|
||||
self.session.flush()
|
||||
with self.assertRaisesRegex(FileStorageError, "Encryption is unavailable"):
|
||||
read_verified_blob_bytes(blob, backend=Backend(ciphertext))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -49,6 +49,10 @@ class FilesRuntimeDocumentationTests(unittest.TestCase):
|
||||
settings = SimpleNamespace(
|
||||
file_upload_max_bytes=7 * 1024 * 1024,
|
||||
file_upload_zip_max_bytes=19 * 1024 * 1024,
|
||||
file_archive_max_expanded_bytes=31 * 1024 * 1024,
|
||||
file_archive_max_entries=321,
|
||||
file_archive_max_expansion_ratio=42,
|
||||
file_archive_preview_ttl_seconds=15 * 60,
|
||||
)
|
||||
with patch(
|
||||
"govoplan_files.backend.documentation.visible_connector_profiles_for_actor",
|
||||
@@ -73,8 +77,12 @@ class FilesRuntimeDocumentationTests(unittest.TestCase):
|
||||
|
||||
archive = topics["files.workflow.upload-and-unpack-zip"]
|
||||
self.assertIn("19 MiB (19,922,944 bytes)", archive.body)
|
||||
self.assertIn("31 MiB (32,505,856 bytes)", archive.body)
|
||||
self.assertIn("7 MiB (7,340,032 bytes)", archive.body)
|
||||
self.assertIn("1,000", archive.body)
|
||||
self.assertIn("321", archive.body)
|
||||
self.assertIn("42:1", archive.body)
|
||||
self.assertIn("15 minutes", archive.body)
|
||||
self.assertIn("passwords remain request-only", archive.body)
|
||||
self.assertIn("Actual extracted bytes are counted", archive.body)
|
||||
|
||||
def test_connector_task_requires_authority_and_a_visible_usable_profile(
|
||||
@@ -134,6 +142,7 @@ class FilesRuntimeDocumentationTests(unittest.TestCase):
|
||||
base_path="/classified",
|
||||
credential_mode="basic",
|
||||
password_value="credential-secret",
|
||||
secret_ref="runtime/connector-secret",
|
||||
)
|
||||
with patch(
|
||||
"govoplan_files.backend.documentation.visible_connector_profiles_for_actor",
|
||||
|
||||
@@ -0,0 +1,468 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from govoplan_access.backend.db.models import Account, Group, User
|
||||
from govoplan_core.core.dsar import DsarProvider, DsarSubjectRef
|
||||
from govoplan_core.core.change_sequence import ChangeSequenceEntry
|
||||
from govoplan_core.db.base import Base
|
||||
from govoplan_core.privacy.dsar_workflow import (
|
||||
DataSubjectRequest,
|
||||
create_data_subject_request,
|
||||
execute_data_subject_erasure,
|
||||
plan_data_subject_erasure,
|
||||
search_data_subject_request,
|
||||
)
|
||||
from govoplan_files.backend.db.models import (
|
||||
FileAsset,
|
||||
FileBlob,
|
||||
FileConnectorCredential,
|
||||
FileConnectorPolicy,
|
||||
FileConnectorProfile,
|
||||
FileConnectorSpace,
|
||||
FileFolder,
|
||||
FileFormEvidenceGrant,
|
||||
FileIntegrityFinding,
|
||||
FileIntegrityScan,
|
||||
FileShare,
|
||||
FileVersion,
|
||||
)
|
||||
from govoplan_files.backend.dsar_provider import (
|
||||
FILES_DSAR_CAPABILITY,
|
||||
FilesDsarProvider,
|
||||
)
|
||||
from govoplan_files.backend.manifest import manifest
|
||||
|
||||
|
||||
class _Registry:
|
||||
def __init__(self, provider: FilesDsarProvider, *, files_active: bool = True) -> None:
|
||||
self.provider = provider
|
||||
self.files_active = files_active
|
||||
|
||||
def capability_names(self):
|
||||
return (FILES_DSAR_CAPABILITY,)
|
||||
|
||||
def capability_owner(self, name):
|
||||
self._assert_capability(name)
|
||||
return "files"
|
||||
|
||||
def tenant_entitlement_resolver(self):
|
||||
files_active = self.files_active
|
||||
|
||||
class _Resolver:
|
||||
@staticmethod
|
||||
def resolve(session, tenant_id):
|
||||
del session, tenant_id
|
||||
return type(
|
||||
"State",
|
||||
(),
|
||||
{"effective_modules": ("files",) if files_active else ()},
|
||||
)()
|
||||
|
||||
return _Resolver()
|
||||
|
||||
def require_tenant_capability(self, name, session, **kwargs):
|
||||
del session, kwargs
|
||||
self._assert_capability(name)
|
||||
return self.provider
|
||||
|
||||
def manifests(self):
|
||||
return (type("Manifest", (), {"id": "files"})(),)
|
||||
|
||||
@staticmethod
|
||||
def _assert_capability(name: str) -> None:
|
||||
if name != FILES_DSAR_CAPABILITY:
|
||||
raise KeyError(name)
|
||||
|
||||
|
||||
class FilesDsarProviderTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.engine = create_engine("sqlite:///:memory:", future=True)
|
||||
Base.metadata.create_all(
|
||||
bind=self.engine,
|
||||
tables=[
|
||||
Account.__table__,
|
||||
User.__table__,
|
||||
Group.__table__,
|
||||
ChangeSequenceEntry.__table__,
|
||||
DataSubjectRequest.__table__,
|
||||
FileBlob.__table__,
|
||||
FileAsset.__table__,
|
||||
FileVersion.__table__,
|
||||
FileFolder.__table__,
|
||||
FileFormEvidenceGrant.__table__,
|
||||
FileShare.__table__,
|
||||
FileConnectorCredential.__table__,
|
||||
FileConnectorPolicy.__table__,
|
||||
FileConnectorProfile.__table__,
|
||||
FileConnectorSpace.__table__,
|
||||
FileIntegrityScan.__table__,
|
||||
FileIntegrityFinding.__table__,
|
||||
],
|
||||
)
|
||||
self.session = sessionmaker(bind=self.engine, future=True)()
|
||||
self.account = Account(
|
||||
id="account-1",
|
||||
email="subject@example.test",
|
||||
normalized_email="subject@example.test",
|
||||
display_name="Subject",
|
||||
password_hash="not-exported",
|
||||
)
|
||||
self.user = User(
|
||||
id="membership-1",
|
||||
tenant_id="tenant-1",
|
||||
account_id=self.account.id,
|
||||
email="subject@example.test",
|
||||
display_name="Subject",
|
||||
)
|
||||
self.blob = FileBlob(
|
||||
id="blob-1",
|
||||
tenant_id="tenant-1",
|
||||
storage_backend="local",
|
||||
storage_key="private/storage/key-do-not-export",
|
||||
checksum_sha256="a" * 64,
|
||||
size_bytes=12,
|
||||
ref_count=1,
|
||||
)
|
||||
self.asset = FileAsset(
|
||||
id="asset-1",
|
||||
tenant_id="tenant-1",
|
||||
owner_type="user",
|
||||
owner_user_id=self.user.id,
|
||||
created_by_user_id=self.user.id,
|
||||
current_version_id="version-1",
|
||||
display_path="subjects/private.txt",
|
||||
filename="private.txt",
|
||||
description="Subject-provided document",
|
||||
retained_until=datetime.now(timezone.utc) + timedelta(days=30),
|
||||
lifecycle_reason="Pending proceeding",
|
||||
metadata_={"password": "metadata-secret-do-not-export"},
|
||||
)
|
||||
self.version = FileVersion(
|
||||
id="version-1",
|
||||
tenant_id="tenant-1",
|
||||
file_asset_id=self.asset.id,
|
||||
blob_id=self.blob.id,
|
||||
version_number=1,
|
||||
filename_at_upload=self.asset.filename,
|
||||
display_path_at_upload=self.asset.display_path,
|
||||
content_type="text/plain",
|
||||
size_bytes=12,
|
||||
checksum_sha256="a" * 64,
|
||||
created_by_user_id=self.user.id,
|
||||
)
|
||||
self.folder = FileFolder(
|
||||
id="folder-1",
|
||||
tenant_id="tenant-1",
|
||||
owner_type="user",
|
||||
owner_user_id=self.user.id,
|
||||
path="subjects",
|
||||
created_by_user_id=self.user.id,
|
||||
)
|
||||
self.share = FileShare(
|
||||
id="share-1",
|
||||
tenant_id="tenant-1",
|
||||
file_asset_id=self.asset.id,
|
||||
target_type="user",
|
||||
target_id=self.user.id,
|
||||
permission="read",
|
||||
created_by_user_id=self.user.id,
|
||||
)
|
||||
self.form_evidence = FileFormEvidenceGrant(
|
||||
id="evidence-1",
|
||||
tenant_id="tenant-1",
|
||||
form_instance_id="form-instance-1",
|
||||
form_definition_id="application",
|
||||
form_definition_revision="7",
|
||||
token_sha256="b" * 64,
|
||||
idempotency_key="evidence-key-1",
|
||||
request_sha256="c" * 64,
|
||||
custodian_user_id=self.user.id,
|
||||
evidence_kind="attachment",
|
||||
purpose="Submitted application evidence",
|
||||
status="uploaded",
|
||||
expires_at=datetime.now(timezone.utc) + timedelta(days=1),
|
||||
max_size_bytes=1024,
|
||||
allowed_content_types=["text/plain"],
|
||||
file_asset_id=self.asset.id,
|
||||
file_version_id=self.version.id,
|
||||
metadata_={},
|
||||
)
|
||||
self.credential = FileConnectorCredential(
|
||||
id="credential-1",
|
||||
tenant_id="tenant-1",
|
||||
scope_type="tenant",
|
||||
scope_id="tenant-1",
|
||||
label="Subject-created credential",
|
||||
provider="s3",
|
||||
credential_mode="database",
|
||||
username="subject-user",
|
||||
password_encrypted="encrypted-password-do-not-export",
|
||||
token_encrypted="encrypted-token-do-not-export",
|
||||
password_env="PASSWORD_ENV_DO_NOT_EXPORT",
|
||||
secret_ref="vault://do-not-export",
|
||||
created_by_user_id=self.user.id,
|
||||
updated_by_user_id=self.user.id,
|
||||
)
|
||||
self.integrity_scan = FileIntegrityScan(
|
||||
id="scan-1",
|
||||
tenant_id="tenant-1",
|
||||
storage_backend="local",
|
||||
storage_prefix="private-prefix-do-not-export",
|
||||
created_by_user_id=self.user.id,
|
||||
)
|
||||
tenant_two_asset = FileAsset(
|
||||
id="asset-tenant-2",
|
||||
tenant_id="tenant-2",
|
||||
owner_type="user",
|
||||
owner_user_id=self.user.id,
|
||||
display_path="other-tenant.txt",
|
||||
filename="other-tenant.txt",
|
||||
)
|
||||
self.session.add_all(
|
||||
[
|
||||
self.account,
|
||||
self.user,
|
||||
self.blob,
|
||||
self.asset,
|
||||
self.version,
|
||||
self.folder,
|
||||
self.share,
|
||||
self.form_evidence,
|
||||
self.credential,
|
||||
self.integrity_scan,
|
||||
tenant_two_asset,
|
||||
]
|
||||
)
|
||||
self.session.commit()
|
||||
self.provider = FilesDsarProvider()
|
||||
self.subject = DsarSubjectRef(membership_id=self.user.id)
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self.session.close()
|
||||
self.engine.dispose()
|
||||
|
||||
def test_manifest_publishes_protocol_conforming_provider(self) -> None:
|
||||
provided_names = {item.name for item in manifest.provides_interfaces}
|
||||
self.assertIn(FILES_DSAR_CAPABILITY, provided_names)
|
||||
provider = manifest.capability_factories[FILES_DSAR_CAPABILITY](None)
|
||||
self.assertIsInstance(provider, DsarProvider)
|
||||
|
||||
def test_search_is_tenant_scoped_and_excludes_bytes_and_secrets(self) -> None:
|
||||
records = self.provider.search_subject(
|
||||
self.session,
|
||||
tenant_id="tenant-1",
|
||||
subject=self.subject,
|
||||
)
|
||||
resource_types = {record.resource_type for record in records}
|
||||
self.assertTrue(
|
||||
{
|
||||
"file_asset",
|
||||
"file_version",
|
||||
"file_folder",
|
||||
"file_share",
|
||||
"file_form_evidence",
|
||||
"connector_credential",
|
||||
"file_integrity_scan",
|
||||
}.issubset(resource_types)
|
||||
)
|
||||
serialized = repr([record.to_dict() for record in records])
|
||||
self.assertNotIn("asset-tenant-2", serialized)
|
||||
self.assertNotIn("private/storage/key-do-not-export", serialized)
|
||||
self.assertNotIn("metadata-secret-do-not-export", serialized)
|
||||
self.assertNotIn("encrypted-password-do-not-export", serialized)
|
||||
self.assertNotIn("encrypted-token-do-not-export", serialized)
|
||||
self.assertNotIn("subject-user", serialized)
|
||||
self.assertNotIn("PASSWORD_ENV_DO_NOT_EXPORT", serialized)
|
||||
self.assertNotIn("vault://do-not-export", serialized)
|
||||
self.assertNotIn("private-prefix-do-not-export", serialized)
|
||||
|
||||
def test_conflicting_direct_subject_references_fail_closed(self) -> None:
|
||||
records = self.provider.search_subject(
|
||||
self.session,
|
||||
tenant_id="tenant-1",
|
||||
subject=DsarSubjectRef(
|
||||
membership_id=self.user.id,
|
||||
external_references={"files.user": "another-membership"},
|
||||
),
|
||||
)
|
||||
|
||||
self.assertEqual((), records)
|
||||
|
||||
def test_plan_classifies_retention_and_manual_review(self) -> None:
|
||||
records = self.provider.search_subject(
|
||||
self.session,
|
||||
tenant_id="tenant-1",
|
||||
subject=self.subject,
|
||||
)
|
||||
actions = self.provider.plan_erasure(
|
||||
self.session,
|
||||
tenant_id="tenant-1",
|
||||
subject=self.subject,
|
||||
records=records,
|
||||
)
|
||||
|
||||
kinds = {action.kind for action in actions}
|
||||
self.assertTrue({"retain", "manual_review", "revoke", "detach"}.issubset(kinds))
|
||||
asset_retention = next(
|
||||
action
|
||||
for action in actions
|
||||
if action.action_id == "files:retain:file_asset:asset-1"
|
||||
)
|
||||
self.assertIn("retained until", asset_retention.rationale)
|
||||
self.assertTrue(
|
||||
any(action.action_id == "files:revoke:file_share:share-1" for action in actions)
|
||||
)
|
||||
self.assertFalse(
|
||||
any(action.kind == "delete" and action.executable for action in actions)
|
||||
)
|
||||
|
||||
def test_execution_is_revalidated_and_idempotent(self) -> None:
|
||||
records = self.provider.search_subject(
|
||||
self.session,
|
||||
tenant_id="tenant-1",
|
||||
subject=self.subject,
|
||||
)
|
||||
actions = self.provider.plan_erasure(
|
||||
self.session,
|
||||
tenant_id="tenant-1",
|
||||
subject=self.subject,
|
||||
records=records,
|
||||
)
|
||||
executable = tuple(action for action in actions if action.executable)
|
||||
|
||||
first = self.provider.execute_erasure(
|
||||
self.session,
|
||||
tenant_id="tenant-1",
|
||||
subject=self.subject,
|
||||
actions=executable,
|
||||
request_id="dsar-1",
|
||||
)
|
||||
self.assertEqual({"executed"}, {result.status for result in first})
|
||||
self.assertIsNotNone(self.share.revoked_at)
|
||||
self.assertIsNone(self.asset.created_by_user_id)
|
||||
self.assertIsNone(self.version.created_by_user_id)
|
||||
self.assertIsNone(self.folder.created_by_user_id)
|
||||
self.assertIsNone(self.credential.created_by_user_id)
|
||||
self.assertIsNone(self.credential.updated_by_user_id)
|
||||
|
||||
repeated = self.provider.execute_erasure(
|
||||
self.session,
|
||||
tenant_id="tenant-1",
|
||||
subject=self.subject,
|
||||
actions=executable,
|
||||
request_id="dsar-1",
|
||||
)
|
||||
self.assertEqual({"unchanged"}, {result.status for result in repeated})
|
||||
|
||||
def test_execution_blocks_when_reference_changed_after_planning(self) -> None:
|
||||
records = self.provider.search_subject(
|
||||
self.session,
|
||||
tenant_id="tenant-1",
|
||||
subject=self.subject,
|
||||
)
|
||||
actions = self.provider.plan_erasure(
|
||||
self.session,
|
||||
tenant_id="tenant-1",
|
||||
subject=self.subject,
|
||||
records=records,
|
||||
)
|
||||
detach = next(
|
||||
action
|
||||
for action in actions
|
||||
if action.action_id
|
||||
== "files:detach:file_version:created_by_user_id:version-1"
|
||||
)
|
||||
self.version.created_by_user_id = "replacement-user"
|
||||
self.session.flush()
|
||||
|
||||
result = self.provider.execute_erasure(
|
||||
self.session,
|
||||
tenant_id="tenant-1",
|
||||
subject=self.subject,
|
||||
actions=(detach,),
|
||||
request_id="dsar-2",
|
||||
)
|
||||
|
||||
self.assertEqual("blocked", result[0].status)
|
||||
self.assertEqual("replacement-user", self.version.created_by_user_id)
|
||||
|
||||
def test_core_workflow_discovers_active_provider_and_skips_it_when_disabled(
|
||||
self,
|
||||
) -> None:
|
||||
request = create_data_subject_request(
|
||||
self.session,
|
||||
tenant_id="tenant-1",
|
||||
reference="DSAR-FILES-1",
|
||||
request_kind="access_and_erasure",
|
||||
subject=self.subject,
|
||||
purpose="Respond to an authorized privacy request.",
|
||||
legal_basis="Article 15 and 17 GDPR",
|
||||
due_at=None,
|
||||
requested_by_account_id="privacy-officer",
|
||||
)
|
||||
self.session.commit()
|
||||
registry = _Registry(self.provider)
|
||||
|
||||
search_data_subject_request(
|
||||
self.session,
|
||||
registry=registry,
|
||||
row=request,
|
||||
expected_revision=1,
|
||||
)
|
||||
self.assertEqual(
|
||||
"searched", request.status, request.search_result["provider_runs"]
|
||||
)
|
||||
self.assertEqual(["files"], request.coverage["covered_modules"])
|
||||
self.assertEqual([], request.coverage["modules_without_provider"])
|
||||
plan_data_subject_erasure(
|
||||
self.session,
|
||||
registry=registry,
|
||||
row=request,
|
||||
expected_revision=2,
|
||||
)
|
||||
executable_ids = [
|
||||
action["action_id"]
|
||||
for action in request.erasure_plan["actions"]
|
||||
if action["executable"]
|
||||
]
|
||||
execute_data_subject_erasure(
|
||||
self.session,
|
||||
registry=registry,
|
||||
row=request,
|
||||
expected_revision=3,
|
||||
action_ids=executable_ids,
|
||||
)
|
||||
self.assertEqual("completed", request.status)
|
||||
|
||||
disabled = create_data_subject_request(
|
||||
self.session,
|
||||
tenant_id="tenant-1",
|
||||
reference="DSAR-FILES-DISABLED",
|
||||
request_kind="access",
|
||||
subject=self.subject,
|
||||
purpose="Verify disabled-module coverage.",
|
||||
legal_basis="Article 15 GDPR",
|
||||
due_at=None,
|
||||
requested_by_account_id="privacy-officer",
|
||||
)
|
||||
search_data_subject_request(
|
||||
self.session,
|
||||
registry=_Registry(self.provider, files_active=False),
|
||||
row=disabled,
|
||||
expected_revision=1,
|
||||
)
|
||||
|
||||
self.assertEqual(0, disabled.search_result["record_count"])
|
||||
self.assertEqual(
|
||||
[FILES_DSAR_CAPABILITY],
|
||||
disabled.coverage["inactive_provider_capabilities"],
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,164 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import timedelta
|
||||
import unittest
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from govoplan_access.backend.db.models import Account, Group, User
|
||||
from govoplan_core.db.base import Base
|
||||
from govoplan_core.core.change_sequence import ChangeSequenceEntry
|
||||
from govoplan_files.backend.db.models import (
|
||||
FileAsset,
|
||||
FileBlob,
|
||||
FileFolder,
|
||||
FileFormEvidenceGrant,
|
||||
FileShare,
|
||||
FileVersion,
|
||||
)
|
||||
from govoplan_files.backend.storage.common import FileStorageError, utcnow
|
||||
from govoplan_files.backend.storage.lifecycle import (
|
||||
preview_asset_purge,
|
||||
restore_asset,
|
||||
set_asset_lifecycle,
|
||||
)
|
||||
|
||||
|
||||
class FileLifecycleTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.engine = create_engine("sqlite:///:memory:", future=True)
|
||||
Base.metadata.create_all(
|
||||
bind=self.engine,
|
||||
tables=[
|
||||
Account.__table__,
|
||||
User.__table__,
|
||||
Group.__table__,
|
||||
ChangeSequenceEntry.__table__,
|
||||
FileBlob.__table__,
|
||||
FileAsset.__table__,
|
||||
FileVersion.__table__,
|
||||
FileFolder.__table__,
|
||||
FileFormEvidenceGrant.__table__,
|
||||
FileShare.__table__,
|
||||
],
|
||||
)
|
||||
self.session = sessionmaker(bind=self.engine, future=True)()
|
||||
self.blob = FileBlob(
|
||||
id="blob-1",
|
||||
tenant_id="tenant-1",
|
||||
storage_backend="local",
|
||||
storage_key="tenants/tenant-1/files/blob-1",
|
||||
checksum_sha256="a" * 64,
|
||||
size_bytes=1,
|
||||
ref_count=1,
|
||||
)
|
||||
self.asset = FileAsset(
|
||||
id="file-1",
|
||||
tenant_id="tenant-1",
|
||||
owner_type="user",
|
||||
owner_user_id="user-1",
|
||||
current_version_id="version-1",
|
||||
display_path="records/report.txt",
|
||||
filename="report.txt",
|
||||
deleted_at=utcnow(),
|
||||
)
|
||||
self.version = FileVersion(
|
||||
id="version-1",
|
||||
tenant_id="tenant-1",
|
||||
file_asset_id="file-1",
|
||||
blob_id="blob-1",
|
||||
version_number=1,
|
||||
filename_at_upload="report.txt",
|
||||
display_path_at_upload="records/report.txt",
|
||||
size_bytes=1,
|
||||
checksum_sha256="a" * 64,
|
||||
)
|
||||
self.session.add_all([self.blob, self.asset, self.version])
|
||||
self.session.commit()
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self.session.close()
|
||||
self.engine.dispose()
|
||||
|
||||
def test_preview_hash_changes_with_retention_and_blocks_active_policy(self) -> None:
|
||||
first = preview_asset_purge(
|
||||
self.session, tenant_id="tenant-1", file_ids=[self.asset.id]
|
||||
)
|
||||
self.assertTrue(first.eligible)
|
||||
|
||||
set_asset_lifecycle(
|
||||
self.session,
|
||||
self.asset,
|
||||
retained_until=utcnow() + timedelta(days=7),
|
||||
legal_hold=True,
|
||||
reason="Active proceeding",
|
||||
expected_revision=1,
|
||||
)
|
||||
self.session.commit()
|
||||
second = preview_asset_purge(
|
||||
self.session, tenant_id="tenant-1", file_ids=[self.asset.id]
|
||||
)
|
||||
|
||||
self.assertFalse(second.eligible)
|
||||
self.assertNotEqual(first.preview_sha256, second.preview_sha256)
|
||||
self.assertEqual(
|
||||
{"legal_hold", "retention_active"}, set(second.items[0].blockers)
|
||||
)
|
||||
|
||||
def test_restore_preserves_version_and_provenance_and_rejects_collision(self) -> None:
|
||||
self.asset.metadata_ = {"source": {"provider": "s3", "revision": "v1"}}
|
||||
self.session.commit()
|
||||
|
||||
self.assertTrue(restore_asset(self.session, self.asset))
|
||||
self.assertEqual("version-1", self.asset.current_version_id)
|
||||
self.assertEqual("v1", self.asset.metadata_["source"]["revision"])
|
||||
self.session.commit()
|
||||
|
||||
self.asset.deleted_at = utcnow()
|
||||
collision = FileAsset(
|
||||
id="file-2",
|
||||
tenant_id="tenant-1",
|
||||
owner_type="user",
|
||||
owner_user_id="user-1",
|
||||
display_path=self.asset.display_path,
|
||||
filename=self.asset.filename,
|
||||
)
|
||||
self.session.add(collision)
|
||||
self.session.commit()
|
||||
with self.assertRaisesRegex(FileStorageError, "path is already in use"):
|
||||
restore_asset(self.session, self.asset)
|
||||
|
||||
def test_active_share_blocks_purge_until_revoked(self) -> None:
|
||||
share = FileShare(
|
||||
id="share-1",
|
||||
tenant_id="tenant-1",
|
||||
file_asset_id=self.asset.id,
|
||||
target_type="tenant",
|
||||
target_id="tenant-1",
|
||||
permission="read",
|
||||
)
|
||||
self.session.add(share)
|
||||
self.session.commit()
|
||||
|
||||
blocked = preview_asset_purge(
|
||||
self.session, tenant_id="tenant-1", file_ids=[self.asset.id]
|
||||
)
|
||||
self.assertIn("active_share", blocked.items[0].blockers)
|
||||
|
||||
share.revoked_at = utcnow()
|
||||
self.session.commit()
|
||||
allowed = preview_asset_purge(
|
||||
self.session, tenant_id="tenant-1", file_ids=[self.asset.id]
|
||||
)
|
||||
self.assertTrue(allowed.eligible)
|
||||
|
||||
def test_missing_campaign_table_is_an_optional_module_safe_path(self) -> None:
|
||||
preview = preview_asset_purge(
|
||||
self.session, tenant_id="tenant-1", file_ids=[self.asset.id]
|
||||
)
|
||||
self.assertNotIn("campaign_evidence", preview.items[0].blockers)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,333 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from io import BytesIO
|
||||
from types import SimpleNamespace
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from fastapi import UploadFile
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import Session
|
||||
from starlette.datastructures import Headers
|
||||
|
||||
from govoplan_access.backend.db.models import Account, User
|
||||
from govoplan_core.core.access import (
|
||||
CAPABILITY_ACCESS_DIRECTORY,
|
||||
AccessSubjectRef,
|
||||
UserRef,
|
||||
)
|
||||
from govoplan_core.core.change_sequence import ChangeSequenceEntry
|
||||
from govoplan_core.core.form_evidence import (
|
||||
FormEvidenceContractError,
|
||||
FormEvidenceGrantRequest,
|
||||
FormEvidenceInspectionRequest,
|
||||
)
|
||||
from govoplan_core.core.institutional import EvidenceReference, InstitutionalReference
|
||||
from govoplan_core.db.base import Base
|
||||
from govoplan_files.backend.db.models import (
|
||||
FileAsset,
|
||||
FileBlob,
|
||||
FileFormEvidenceGrant,
|
||||
FileVersion,
|
||||
)
|
||||
from govoplan_files.backend.form_evidence import FilesFormEvidenceProvider
|
||||
from govoplan_files.backend.routes.form_evidence import upload_form_evidence
|
||||
|
||||
|
||||
class _Directory:
|
||||
def __init__(self, user: UserRef) -> None:
|
||||
self._user = user
|
||||
|
||||
def get_account(self, account_id: str):
|
||||
del account_id
|
||||
return None
|
||||
|
||||
def get_user(self, user_id: str):
|
||||
return self._user if user_id == self._user.id else None
|
||||
|
||||
def get_users(self, user_ids):
|
||||
return {user_id: self._user for user_id in user_ids if user_id == self._user.id}
|
||||
|
||||
def users_for_tenant(self, tenant_id: str):
|
||||
return (self._user,) if tenant_id == self._user.tenant_id else ()
|
||||
|
||||
def get_group(self, group_id: str):
|
||||
del group_id
|
||||
return None
|
||||
|
||||
def get_groups(self, group_ids):
|
||||
del group_ids
|
||||
return {}
|
||||
|
||||
def groups_for_tenant(self, tenant_id: str):
|
||||
del tenant_id
|
||||
return ()
|
||||
|
||||
def groups_for_user(self, user_id: str, *, tenant_id: str):
|
||||
del user_id, tenant_id
|
||||
return ()
|
||||
|
||||
def display_label(self, subject: AccessSubjectRef):
|
||||
del subject
|
||||
return self._user.display_name
|
||||
|
||||
|
||||
class _Registry:
|
||||
def __init__(self, directory: _Directory) -> None:
|
||||
self._directory = directory
|
||||
|
||||
def has_capability(self, name: str) -> bool:
|
||||
return name == CAPABILITY_ACCESS_DIRECTORY
|
||||
|
||||
def require_capability(self, name: str):
|
||||
if not self.has_capability(name):
|
||||
raise KeyError(name)
|
||||
return self._directory
|
||||
|
||||
|
||||
class FilesFormEvidenceTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.engine = create_engine("sqlite+pysqlite:///:memory:")
|
||||
Base.metadata.create_all(
|
||||
self.engine,
|
||||
tables=[
|
||||
Account.__table__,
|
||||
User.__table__,
|
||||
FileBlob.__table__,
|
||||
FileAsset.__table__,
|
||||
FileVersion.__table__,
|
||||
FileFormEvidenceGrant.__table__,
|
||||
ChangeSequenceEntry.__table__,
|
||||
],
|
||||
)
|
||||
self.session = Session(self.engine)
|
||||
user_ref = UserRef(
|
||||
id="user-1",
|
||||
account_id="account-1",
|
||||
tenant_id="tenant-1",
|
||||
email="user@example.test",
|
||||
display_name="Evidence Custodian",
|
||||
)
|
||||
self.provider = FilesFormEvidenceProvider(
|
||||
_Registry(_Directory(user_ref)),
|
||||
SimpleNamespace(file_upload_max_bytes=2_000_000),
|
||||
)
|
||||
self.principal = SimpleNamespace(tenant_id="tenant-1")
|
||||
self.definition_ref = InstitutionalReference(
|
||||
kind="form",
|
||||
owner_module="forms",
|
||||
object_id="form-1",
|
||||
tenant_id="tenant-1",
|
||||
version="3",
|
||||
)
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self.session.close()
|
||||
self.engine.dispose()
|
||||
|
||||
def request(self, **overrides: object) -> FormEvidenceGrantRequest:
|
||||
values: dict[str, object] = {
|
||||
"tenant_id": "tenant-1",
|
||||
"instance_id": "instance-1",
|
||||
"definition_ref": self.definition_ref,
|
||||
"evidence_kind": "document",
|
||||
"purpose": "supporting document",
|
||||
"idempotency_key": "grant-1",
|
||||
"expires_at": datetime.now(UTC) + timedelta(hours=1),
|
||||
"custodian_ref": "user:user-1",
|
||||
"max_size_bytes": 1_000_000,
|
||||
"allowed_content_types": ("application/pdf",),
|
||||
}
|
||||
values.update(overrides)
|
||||
return FormEvidenceGrantRequest(**values) # type: ignore[arg-type]
|
||||
|
||||
def test_grant_persists_only_token_hash_and_replays_without_secret(self) -> None:
|
||||
request = self.request()
|
||||
issued = self.provider.create_upload_grant(
|
||||
self.session,
|
||||
self.principal,
|
||||
request=request,
|
||||
)
|
||||
row = self.session.query(FileFormEvidenceGrant).one()
|
||||
|
||||
self.assertIsNotNone(issued.upload_token)
|
||||
self.assertNotEqual(issued.upload_token, row.token_sha256)
|
||||
self.assertLessEqual(
|
||||
issued.expires_at,
|
||||
datetime.now(UTC) + timedelta(minutes=15, seconds=1),
|
||||
)
|
||||
replay = self.provider.create_upload_grant(
|
||||
self.session,
|
||||
self.principal,
|
||||
request=request,
|
||||
)
|
||||
self.assertTrue(replay.replayed)
|
||||
self.assertIsNone(replay.upload_token)
|
||||
self.assertEqual(issued.grant_id, replay.grant_id)
|
||||
|
||||
def test_grant_capacity_counts_outstanding_uploads_but_allows_replay(self) -> None:
|
||||
request = self.request(metadata={"remaining_attachments": 1})
|
||||
self.provider.create_upload_grant(self.session, self.principal, request=request)
|
||||
|
||||
replay = self.provider.create_upload_grant(
|
||||
self.session,
|
||||
self.principal,
|
||||
request=request,
|
||||
)
|
||||
self.assertTrue(replay.replayed)
|
||||
with self.assertRaisesRegex(
|
||||
FormEvidenceContractError,
|
||||
"maximum number of active attachment uploads",
|
||||
):
|
||||
self.provider.create_upload_grant(
|
||||
self.session,
|
||||
self.principal,
|
||||
request=self.request(
|
||||
idempotency_key="grant-2",
|
||||
metadata={"remaining_attachments": 1},
|
||||
),
|
||||
)
|
||||
|
||||
def test_idempotency_key_cannot_be_reused_for_another_request(self) -> None:
|
||||
self.provider.create_upload_grant(
|
||||
self.session,
|
||||
self.principal,
|
||||
request=self.request(),
|
||||
)
|
||||
with self.assertRaisesRegex(
|
||||
FormEvidenceContractError,
|
||||
"idempotency conflict",
|
||||
):
|
||||
self.provider.create_upload_grant(
|
||||
self.session,
|
||||
self.principal,
|
||||
request=self.request(purpose="another purpose"),
|
||||
)
|
||||
|
||||
def test_inspection_accepts_only_exact_verified_managed_version(self) -> None:
|
||||
issued = self.provider.create_upload_grant(
|
||||
self.session,
|
||||
self.principal,
|
||||
request=self.request(),
|
||||
)
|
||||
grant = self.session.get(FileFormEvidenceGrant, issued.grant_id)
|
||||
assert grant is not None
|
||||
blob = FileBlob(
|
||||
id="blob-1",
|
||||
tenant_id="tenant-1",
|
||||
storage_backend="local",
|
||||
storage_key="tenant-1/blob-1",
|
||||
checksum_sha256="a" * 64,
|
||||
size_bytes=100,
|
||||
integrity_status="verified",
|
||||
)
|
||||
asset = FileAsset(
|
||||
id="asset-1",
|
||||
tenant_id="tenant-1",
|
||||
owner_type="user",
|
||||
owner_user_id="user-1",
|
||||
current_version_id="version-1",
|
||||
display_path="Form submissions/instance-1/evidence.pdf",
|
||||
filename="evidence.pdf",
|
||||
)
|
||||
version = FileVersion(
|
||||
id="version-1",
|
||||
tenant_id="tenant-1",
|
||||
file_asset_id="asset-1",
|
||||
blob_id="blob-1",
|
||||
version_number=1,
|
||||
filename_at_upload="evidence.pdf",
|
||||
display_path_at_upload=asset.display_path,
|
||||
content_type="application/pdf",
|
||||
size_bytes=100,
|
||||
checksum_sha256="a" * 64,
|
||||
)
|
||||
self.session.add_all((blob, asset, version))
|
||||
grant.status = "uploaded"
|
||||
grant.file_asset_id = asset.id
|
||||
grant.file_version_id = version.id
|
||||
self.session.flush()
|
||||
reference = EvidenceReference(
|
||||
kind="document",
|
||||
owner_module="files",
|
||||
evidence_id=asset.id,
|
||||
tenant_id="tenant-1",
|
||||
version=version.id,
|
||||
checksum="a" * 64,
|
||||
)
|
||||
|
||||
accepted = self.provider.inspect_evidence(
|
||||
self.session,
|
||||
self.principal,
|
||||
request=FormEvidenceInspectionRequest(
|
||||
tenant_id="tenant-1",
|
||||
instance_id="instance-1",
|
||||
definition_ref=self.definition_ref,
|
||||
evidence=reference,
|
||||
purpose="final submission",
|
||||
final=True,
|
||||
),
|
||||
)
|
||||
self.assertTrue(accepted.accepted)
|
||||
|
||||
rejected = self.provider.inspect_evidence(
|
||||
self.session,
|
||||
self.principal,
|
||||
request=FormEvidenceInspectionRequest(
|
||||
tenant_id="tenant-1",
|
||||
instance_id="another-instance",
|
||||
definition_ref=self.definition_ref,
|
||||
evidence=reference,
|
||||
purpose="final submission",
|
||||
final=True,
|
||||
),
|
||||
)
|
||||
self.assertEqual("rejected", rejected.state)
|
||||
|
||||
def test_public_upload_consumes_grant_and_returns_exact_evidence(self) -> None:
|
||||
issued = self.provider.create_upload_grant(
|
||||
self.session,
|
||||
self.principal,
|
||||
request=self.request(),
|
||||
)
|
||||
assert issued.upload_token is not None
|
||||
stored = SimpleNamespace(
|
||||
asset=SimpleNamespace(id="asset-uploaded"),
|
||||
version=SimpleNamespace(
|
||||
id="version-uploaded",
|
||||
checksum_sha256="c" * 64,
|
||||
size_bytes=8,
|
||||
),
|
||||
)
|
||||
upload = UploadFile(
|
||||
filename="evidence.pdf",
|
||||
file=BytesIO(b"evidence"),
|
||||
headers=Headers({"content-type": "application/pdf"}),
|
||||
)
|
||||
|
||||
with (
|
||||
patch(
|
||||
"govoplan_files.backend.routes.form_evidence.create_file_asset",
|
||||
return_value=stored,
|
||||
) as create,
|
||||
patch("govoplan_files.backend.routes.form_evidence.emit_platform_event"),
|
||||
):
|
||||
response = upload_form_evidence(
|
||||
file=upload,
|
||||
x_form_evidence_token=issued.upload_token,
|
||||
session=self.session,
|
||||
)
|
||||
|
||||
grant = self.session.get(FileFormEvidenceGrant, issued.grant_id)
|
||||
assert grant is not None
|
||||
self.assertEqual("uploaded", grant.status)
|
||||
self.assertEqual("asset-uploaded", grant.file_asset_id)
|
||||
self.assertEqual("version-uploaded", grant.file_version_id)
|
||||
self.assertEqual("asset-uploaded", response["evidence"]["evidence_id"])
|
||||
self.assertEqual(b"evidence", create.call_args.kwargs["data"])
|
||||
self.assertEqual("user-1", create.call_args.kwargs["owner_id"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,211 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from fastapi import HTTPException
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from govoplan_access.backend.db.models import Account, Group, User
|
||||
from govoplan_core.db.base import Base
|
||||
from govoplan_files.backend.db.models import (
|
||||
FileBlob,
|
||||
FileIntegrityFinding,
|
||||
FileIntegrityScan,
|
||||
)
|
||||
from govoplan_files.backend.routes.integrity import _assert_expected_revision
|
||||
from govoplan_files.backend.storage.backends import LocalFilesystemStorageBackend
|
||||
from govoplan_files.backend.storage.common import FileStorageError
|
||||
from govoplan_files.backend.storage.integrity import (
|
||||
cleanup_orphan_finding,
|
||||
create_integrity_scan,
|
||||
read_verified_blob_bytes,
|
||||
recheck_integrity_finding,
|
||||
run_integrity_scan_batch,
|
||||
)
|
||||
|
||||
|
||||
TENANT_ID = "tenant-1"
|
||||
USER_ID = "user-1"
|
||||
|
||||
|
||||
class IntegrityReconciliationTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.temporary_directory = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.temporary_directory.cleanup)
|
||||
self.backend = LocalFilesystemStorageBackend(
|
||||
Path(self.temporary_directory.name)
|
||||
)
|
||||
self.engine = create_engine("sqlite:///:memory:", future=True)
|
||||
Base.metadata.create_all(
|
||||
bind=self.engine,
|
||||
tables=[
|
||||
Account.__table__,
|
||||
User.__table__,
|
||||
Group.__table__,
|
||||
FileBlob.__table__,
|
||||
FileIntegrityScan.__table__,
|
||||
FileIntegrityFinding.__table__,
|
||||
],
|
||||
)
|
||||
self.session = sessionmaker(bind=self.engine, future=True)()
|
||||
self.addCleanup(self._close)
|
||||
|
||||
def _close(self) -> None:
|
||||
self.session.close()
|
||||
self.engine.dispose()
|
||||
|
||||
def test_scan_is_bounded_resumable_and_reconciles_both_orphan_directions(
|
||||
self,
|
||||
) -> None:
|
||||
valid_data = b"valid"
|
||||
restored_data = b"restore-me"
|
||||
expected_corrupt_data = b"expected"
|
||||
stored_corrupt_data = b"corrupt!"
|
||||
valid = _blob("blob-1", "valid.bin", valid_data)
|
||||
missing = _blob("blob-2", "missing.bin", restored_data)
|
||||
corrupt = _blob("blob-3", "corrupt.bin", expected_corrupt_data)
|
||||
self.session.add_all([valid, missing, corrupt])
|
||||
self.session.commit()
|
||||
self.backend.put_bytes(valid.storage_key, valid_data)
|
||||
self.backend.put_bytes(corrupt.storage_key, stored_corrupt_data)
|
||||
orphan_key = f"tenants/{TENANT_ID}/files/orphan.bin"
|
||||
self.backend.put_bytes(orphan_key, b"orphan")
|
||||
|
||||
scan = create_integrity_scan(
|
||||
self.session,
|
||||
tenant_id=TENANT_ID,
|
||||
user_id=USER_ID,
|
||||
batch_size=1,
|
||||
backend=self.backend,
|
||||
)
|
||||
self.session.commit()
|
||||
self.assertEqual(1, scan.revision)
|
||||
|
||||
invocations = 0
|
||||
while scan.status != "completed":
|
||||
run_integrity_scan_batch(
|
||||
self.session,
|
||||
scan,
|
||||
backend=self.backend,
|
||||
)
|
||||
self.session.commit()
|
||||
invocations += 1
|
||||
scan = self.session.get(FileIntegrityScan, scan.id)
|
||||
self.assertIsNotNone(scan)
|
||||
self.session.expire_all()
|
||||
if invocations > 12:
|
||||
self.fail("Integrity scan did not complete")
|
||||
|
||||
self.assertGreater(invocations, 3)
|
||||
self.assertEqual(1 + invocations, scan.revision)
|
||||
self.assertEqual(3, scan.scanned_blob_count)
|
||||
self.assertEqual(1, scan.verified_blob_count)
|
||||
self.assertEqual(2, scan.quarantined_blob_count)
|
||||
self.assertEqual(3, scan.scanned_object_count)
|
||||
self.assertEqual(1, scan.orphan_object_count)
|
||||
findings = (
|
||||
self.session.query(FileIntegrityFinding)
|
||||
.filter(FileIntegrityFinding.scan_id == scan.id)
|
||||
.all()
|
||||
)
|
||||
self.assertEqual(
|
||||
{"missing", "checksum_mismatch", "orphan_object"},
|
||||
{finding.kind for finding in findings},
|
||||
)
|
||||
|
||||
missing = self.session.get(FileBlob, missing.id)
|
||||
corrupt = self.session.get(FileBlob, corrupt.id)
|
||||
self.assertEqual("missing", missing.integrity_status)
|
||||
self.assertEqual("checksum_mismatch", corrupt.integrity_status)
|
||||
with self.assertRaisesRegex(FileStorageError, "quarantined"):
|
||||
read_verified_blob_bytes(missing, backend=self.backend)
|
||||
|
||||
missing_finding = next(
|
||||
finding for finding in findings if finding.kind == "missing"
|
||||
)
|
||||
self.backend.put_bytes(missing.storage_key, restored_data)
|
||||
preview = recheck_integrity_finding(
|
||||
self.session,
|
||||
missing_finding,
|
||||
user_id=USER_ID,
|
||||
dry_run=True,
|
||||
backend=self.backend,
|
||||
)
|
||||
self.assertTrue(preview.inspection.valid)
|
||||
self.assertEqual("open", missing_finding.state)
|
||||
repaired = recheck_integrity_finding(
|
||||
self.session,
|
||||
missing_finding,
|
||||
user_id=USER_ID,
|
||||
dry_run=False,
|
||||
backend=self.backend,
|
||||
)
|
||||
self.session.commit()
|
||||
self.assertTrue(repaired.changed)
|
||||
self.assertEqual("resolved", missing_finding.state)
|
||||
self.assertEqual(
|
||||
restored_data,
|
||||
read_verified_blob_bytes(missing, backend=self.backend),
|
||||
)
|
||||
|
||||
orphan_finding = next(
|
||||
finding for finding in findings if finding.kind == "orphan_object"
|
||||
)
|
||||
preview_cleanup = cleanup_orphan_finding(
|
||||
self.session,
|
||||
orphan_finding,
|
||||
user_id=USER_ID,
|
||||
dry_run=True,
|
||||
backend=self.backend,
|
||||
)
|
||||
self.assertEqual("would_delete", preview_cleanup.action)
|
||||
self.assertTrue(self.backend.exists(orphan_key))
|
||||
with patch(
|
||||
"govoplan_files.backend.storage.integrity.begin_orphan_cleanup_recovery"
|
||||
):
|
||||
cleanup = cleanup_orphan_finding(
|
||||
self.session,
|
||||
orphan_finding,
|
||||
user_id=USER_ID,
|
||||
dry_run=False,
|
||||
backend=self.backend,
|
||||
)
|
||||
repeated = cleanup_orphan_finding(
|
||||
self.session,
|
||||
orphan_finding,
|
||||
user_id=USER_ID,
|
||||
dry_run=False,
|
||||
backend=self.backend,
|
||||
)
|
||||
self.assertEqual("deleted", cleanup.action)
|
||||
self.assertFalse(self.backend.exists(orphan_key))
|
||||
self.assertEqual("already_deleted", repeated.action)
|
||||
self.assertFalse(repeated.changed)
|
||||
|
||||
def test_stale_integrity_action_revision_is_rejected(self) -> None:
|
||||
with self.assertRaises(HTTPException) as captured:
|
||||
_assert_expected_revision(3, 2)
|
||||
|
||||
self.assertEqual(409, captured.exception.status_code)
|
||||
self.assertIn("reload", str(captured.exception.detail).lower())
|
||||
|
||||
|
||||
def _blob(blob_id: str, filename: str, expected_data: bytes) -> FileBlob:
|
||||
return FileBlob(
|
||||
id=blob_id,
|
||||
tenant_id=TENANT_ID,
|
||||
storage_backend="local",
|
||||
storage_key=f"tenants/{TENANT_ID}/files/{filename}",
|
||||
checksum_sha256=hashlib.sha256(expected_data).hexdigest(),
|
||||
size_bytes=len(expected_data),
|
||||
ref_count=1,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -4,13 +4,24 @@ import unittest
|
||||
|
||||
|
||||
STATIC_TOPIC_IDS = {
|
||||
"files.configuration-package.managed-storage",
|
||||
"files.quick-access-and-product-area",
|
||||
"files.search.managed-content",
|
||||
"files.workflow.organize-managed-files",
|
||||
"files.workflow.find-and-download-files",
|
||||
"files.workflow.share-managed-files",
|
||||
"files.workflow.delete-managed-files",
|
||||
"files.workflow.restore-retain-and-purge",
|
||||
"files.privacy.data-subject-requests",
|
||||
"files.governed-connectors-and-provenance",
|
||||
"files.reference.integrity-recovery-and-fail-closed-transports",
|
||||
"files.reference.shared-storage-profile",
|
||||
"files.reference.generated-artifact-store",
|
||||
"files.reference.snapshot-provenance-and-capabilities",
|
||||
"files.records.exact-version-source",
|
||||
"files.forms-runtime.managed-evidence",
|
||||
"files.postbox.exact-version-references",
|
||||
"files.tabular-content",
|
||||
"files.assurance.process-and-release-readiness",
|
||||
}
|
||||
RUNTIME_TOPIC_IDS = {
|
||||
@@ -103,17 +114,30 @@ class FilesManifestDocumentationTests(unittest.TestCase):
|
||||
def test_share_and_delete_tasks_state_current_boundaries(self) -> None:
|
||||
share = self.topic("files.workflow.share-managed-files")
|
||||
self.assertEqual("available", share.layer)
|
||||
self.assertIn("does not yet provide a general share editor", share.body)
|
||||
self.assertIn("no share-revocation route", share.body)
|
||||
self.assertIn("Expired and revoked grants stop authorizing", share.body)
|
||||
self.assertIn("revocation is idempotent", share.body)
|
||||
self.assertIn(
|
||||
"/api/v1/files/{file_id}/shares", {link.href for link in share.links}
|
||||
)
|
||||
|
||||
delete = self.topic("files.workflow.delete-managed-files")
|
||||
self.assertIn("Soft-delete", delete.summary)
|
||||
self.assertIn("no self-service restore or hard-purge", delete.body)
|
||||
self.assertIn("Authorized restoration", delete.body)
|
||||
self.assertTrue(
|
||||
any("not a hard purge" in item for item in delete.metadata["limitations"])
|
||||
any("soft deletion" in item.casefold() for item in delete.metadata["limitations"])
|
||||
)
|
||||
|
||||
lifecycle = self.topic("files.workflow.restore-retain-and-purge")
|
||||
self.assertIn("legal hold", lifecycle.body)
|
||||
self.assertIn("preview hash", lifecycle.body)
|
||||
self.assertIn("recovery ledger", lifecycle.body)
|
||||
|
||||
privacy = self.topic("files.privacy.data-subject-requests")
|
||||
self.assertEqual(("admin",), privacy.documentation_types)
|
||||
self.assertIn("without raw file bytes", privacy.body)
|
||||
self.assertIn("separately authorized Files purge", privacy.body)
|
||||
self.assertTrue(
|
||||
any("membership" in item for item in privacy.metadata["limitations"])
|
||||
)
|
||||
|
||||
def test_admin_topic_covers_policy_redaction_and_atomic_credential_deletion(
|
||||
@@ -125,9 +149,20 @@ class FilesManifestDocumentationTests(unittest.TestCase):
|
||||
self.assertEqual("reference", topic.metadata["kind"])
|
||||
self.assertEqual("File connections", topic.metadata["screen"])
|
||||
self.assertTrue(topic.metadata["section"])
|
||||
self.assertEqual(
|
||||
{
|
||||
"files.connectors",
|
||||
"files.connector.credentials",
|
||||
"files.connector.policy",
|
||||
},
|
||||
set(topic.metadata["help_contexts"]),
|
||||
)
|
||||
self.assertIn("deny rules win", topic.body)
|
||||
self.assertIn("redact secret values", topic.body)
|
||||
self.assertIn("same transaction", topic.body)
|
||||
self.assertIn("read-only by default", topic.body)
|
||||
self.assertIn("S3", topic.body)
|
||||
self.assertIn("Automatic remote deletion", topic.body)
|
||||
self.assertTrue(
|
||||
any(
|
||||
"non-owned external references" in item
|
||||
@@ -142,7 +177,7 @@ class FilesManifestDocumentationTests(unittest.TestCase):
|
||||
"/api/v1/files/connectors/credentials", {link.href for link in topic.links}
|
||||
)
|
||||
|
||||
def test_operator_topic_covers_recovery_and_fail_closed_s3_smb(self) -> None:
|
||||
def test_operator_topic_covers_recovery_and_pinned_s3_smb(self) -> None:
|
||||
topic = self.topic(
|
||||
"files.reference.integrity-recovery-and-fail-closed-transports"
|
||||
)
|
||||
@@ -153,9 +188,31 @@ class FilesManifestDocumentationTests(unittest.TestCase):
|
||||
self.assertIn("SMB", topic.body)
|
||||
self.assertIn("fail closed", topic.body)
|
||||
self.assertIn("DFS referrals", topic.body)
|
||||
self.assertIn("ambient credential discovery", topic.body)
|
||||
self.assertIn("does not remove backend blob objects", topic.body)
|
||||
self.assertIn("bounded resumable integrity scan", topic.body)
|
||||
self.assertIn("quarantined", topic.body)
|
||||
self.assertIn("MASTER_KEY_B64", topic.metadata["recovery_unit"])
|
||||
self.assertIn(
|
||||
"Encryption envelope and wrapped-key rows", topic.metadata["recovery_unit"]
|
||||
)
|
||||
self.assertIn("lease-fenced Core recovery", topic.body)
|
||||
self.assertIn("Ops", topic.body)
|
||||
self.assertIn("Hard purge", topic.body)
|
||||
self.assertIn("S3 connector write-back", topic.body)
|
||||
self.assertTrue(topic.metadata["verification"])
|
||||
self.assertIn(
|
||||
"/api/v1/files/integrity/scans",
|
||||
{link.href for link in topic.links},
|
||||
)
|
||||
self.assertIn(
|
||||
"/admin?section=tenant-file-integrity",
|
||||
{link.href for link in topic.links},
|
||||
)
|
||||
self.assertIn(
|
||||
"files.admin.tenant-integrity",
|
||||
{surface.id for surface in self.manifest.frontend.view_surfaces},
|
||||
)
|
||||
self.assertIn(
|
||||
"GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS", topic.configuration_keys
|
||||
)
|
||||
@@ -166,7 +223,11 @@ class FilesManifestDocumentationTests(unittest.TestCase):
|
||||
self.assertEqual(("admin", "user"), topic.documentation_types)
|
||||
self.assertEqual("reference", topic.metadata["kind"])
|
||||
self.assertEqual(
|
||||
["files.access@0.1.6", "files.campaign_attachments@0.1.6"],
|
||||
[
|
||||
"files.access@0.1.6",
|
||||
"files.campaign_attachments@0.1.6",
|
||||
"forms_runtime.evidence.files@1.0.0",
|
||||
],
|
||||
topic.metadata["provided_interfaces"],
|
||||
)
|
||||
self.assertIn("revision", topic.metadata["provenance_fields"])
|
||||
@@ -184,8 +245,9 @@ class FilesManifestDocumentationTests(unittest.TestCase):
|
||||
self.assertIn("process_owner", topic.audience)
|
||||
self.assertIn("release_manager", topic.audience)
|
||||
self.assertIn("versions align", topic.body)
|
||||
self.assertIn("no general share-management UI or share revocation", topic.body)
|
||||
self.assertIn("no enforced retention or legal hold", topic.body)
|
||||
self.assertIn("share lifecycle", topic.body)
|
||||
self.assertIn("legal hold", topic.body)
|
||||
self.assertIn("S3 write-back", topic.body)
|
||||
for key in ("prerequisites", "steps", "outcome", "verification"):
|
||||
self.assertTrue(topic.metadata[key])
|
||||
self.assertTrue(
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
from alembic.runtime.migration import MigrationContext
|
||||
from sqlalchemy import create_engine, inspect
|
||||
|
||||
from govoplan_access.backend.manifest import get_manifest as get_access_manifest
|
||||
from govoplan_core.db.migrations import migrate_database
|
||||
from govoplan_files.backend.manifest import get_manifest as get_files_manifest
|
||||
|
||||
|
||||
class FilesMigrationTests(unittest.TestCase):
|
||||
def test_fresh_migration_creates_form_evidence_grants_and_head(self) -> None:
|
||||
with tempfile.TemporaryDirectory(
|
||||
prefix="govoplan-files-migration-"
|
||||
) as directory:
|
||||
url = f"sqlite:///{Path(directory) / 'files.db'}"
|
||||
migrate_database(
|
||||
database_url=url,
|
||||
enabled_modules=("access", "files"),
|
||||
manifest_factories=(get_access_manifest, get_files_manifest),
|
||||
)
|
||||
engine = create_engine(url)
|
||||
try:
|
||||
self.assertIn(
|
||||
"file_form_evidence_grants",
|
||||
inspect(engine).get_table_names(),
|
||||
)
|
||||
with engine.connect() as connection:
|
||||
self.assertIn(
|
||||
"a2b3c4d5e6f9",
|
||||
set(MigrationContext.configure(connection).get_current_heads()),
|
||||
)
|
||||
finally:
|
||||
engine.dispose()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,42 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
from govoplan_files.backend.operational_checks import managed_storage_roundtrip_check
|
||||
from govoplan_files.backend.storage.backends import LocalFilesystemStorageBackend
|
||||
|
||||
|
||||
def test_managed_storage_roundtrip_removes_probe(tmp_path) -> None:
|
||||
backend = LocalFilesystemStorageBackend(tmp_path)
|
||||
|
||||
with patch(
|
||||
"govoplan_files.backend.operational_checks.get_storage_backend",
|
||||
return_value=backend,
|
||||
):
|
||||
result = managed_storage_roundtrip_check()
|
||||
|
||||
assert result.state == "ok"
|
||||
assert result.metrics["backend"] == "local"
|
||||
assert list(tmp_path.rglob("*.bin")) == []
|
||||
|
||||
|
||||
def test_managed_storage_roundtrip_fails_closed() -> None:
|
||||
class BrokenBackend:
|
||||
name = "broken"
|
||||
|
||||
def put_bytes(self, *_args, **_kwargs):
|
||||
raise OSError("private provider detail")
|
||||
|
||||
def delete(self, *_args, **_kwargs):
|
||||
return None
|
||||
|
||||
with patch(
|
||||
"govoplan_files.backend.operational_checks.get_storage_backend",
|
||||
return_value=BrokenBackend(),
|
||||
):
|
||||
result = managed_storage_roundtrip_check()
|
||||
|
||||
assert result.state == "error"
|
||||
assert result.readiness_critical is True
|
||||
assert "private provider detail" not in result.detail
|
||||
|
||||
@@ -0,0 +1,160 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from govoplan_core.core.files import PostboxFileReferenceRequest
|
||||
from govoplan_files.backend.capabilities import FilesPostboxReferenceService
|
||||
from govoplan_files.backend.db.models import FileAsset, FileBlob, FileVersion
|
||||
from govoplan_files.backend.storage.common import FileStorageError
|
||||
|
||||
|
||||
class _Principal:
|
||||
def __init__(self, *, tenant_id: str = "tenant-1", scopes: set[str] | None = None):
|
||||
self.tenant_id = tenant_id
|
||||
self.user = SimpleNamespace(id="user-1")
|
||||
self._scopes = {"files:file:download"} if scopes is None else scopes
|
||||
|
||||
def has(self, scope: str) -> bool:
|
||||
return scope in self._scopes
|
||||
|
||||
|
||||
class _Session:
|
||||
def __init__(self, values: dict[tuple[type[object], str], object]):
|
||||
self.values = values
|
||||
|
||||
def get(self, model: type[object], resource_id: str):
|
||||
return self.values.get((model, resource_id))
|
||||
|
||||
|
||||
def _request(reference_type: str = "file_version", reference_id: str = "version-1"):
|
||||
return PostboxFileReferenceRequest(
|
||||
postbox_id="postbox-1",
|
||||
message_id="message-1",
|
||||
reference_type=reference_type,
|
||||
reference_id=reference_id,
|
||||
)
|
||||
|
||||
|
||||
def _objects():
|
||||
asset = SimpleNamespace(
|
||||
id="file-1",
|
||||
tenant_id="tenant-1",
|
||||
current_version_id="version-2",
|
||||
deleted_at=None,
|
||||
filename="current.txt",
|
||||
display_path="Evidence/current.txt",
|
||||
)
|
||||
version = SimpleNamespace(
|
||||
id="version-1",
|
||||
tenant_id="tenant-1",
|
||||
file_asset_id="file-1",
|
||||
blob_id="blob-1",
|
||||
filename_at_upload="evidence.txt",
|
||||
content_type="text/plain",
|
||||
size_bytes=8,
|
||||
checksum_sha256="a" * 64,
|
||||
)
|
||||
blob = SimpleNamespace(
|
||||
id="blob-1",
|
||||
tenant_id="tenant-1",
|
||||
content_type="text/plain",
|
||||
)
|
||||
return asset, version, blob
|
||||
|
||||
|
||||
class FilesPostboxReferenceTests(unittest.TestCase):
|
||||
def test_resolves_the_referenced_version_instead_of_the_current_version(self) -> None:
|
||||
asset, version, blob = _objects()
|
||||
session = _Session(
|
||||
{
|
||||
(FileAsset, asset.id): asset,
|
||||
(FileVersion, version.id): version,
|
||||
(FileBlob, blob.id): blob,
|
||||
}
|
||||
)
|
||||
|
||||
with patch(
|
||||
"govoplan_files.backend.capabilities.get_asset_for_user",
|
||||
return_value=asset,
|
||||
):
|
||||
result = FilesPostboxReferenceService().resolve_postbox_references(
|
||||
session,
|
||||
_Principal(),
|
||||
tenant_id="tenant-1",
|
||||
requests=(_request(),),
|
||||
)[0]
|
||||
|
||||
self.assertTrue(result.available)
|
||||
self.assertEqual("version-1", result.file_version_id)
|
||||
self.assertEqual(
|
||||
"/api/v1/files/file-1/versions/version-1/download",
|
||||
result.download_path,
|
||||
)
|
||||
self.assertTrue(result.provenance["exact_version"])
|
||||
|
||||
def test_fails_closed_for_tenant_permission_and_file_access_mismatches(self) -> None:
|
||||
service = FilesPostboxReferenceService()
|
||||
empty = _Session({})
|
||||
|
||||
tenant_result = service.resolve_postbox_references(
|
||||
empty,
|
||||
_Principal(tenant_id="tenant-2"),
|
||||
tenant_id="tenant-1",
|
||||
requests=(_request(),),
|
||||
)[0]
|
||||
permission_result = service.resolve_postbox_references(
|
||||
empty,
|
||||
_Principal(scopes=set()),
|
||||
tenant_id="tenant-1",
|
||||
requests=(_request(),),
|
||||
)[0]
|
||||
|
||||
asset, version, blob = _objects()
|
||||
populated = _Session(
|
||||
{
|
||||
(FileAsset, asset.id): asset,
|
||||
(FileVersion, version.id): version,
|
||||
(FileBlob, blob.id): blob,
|
||||
}
|
||||
)
|
||||
with patch(
|
||||
"govoplan_files.backend.capabilities.get_asset_for_user",
|
||||
side_effect=FileStorageError("denied"),
|
||||
):
|
||||
access_result = service.resolve_postbox_references(
|
||||
populated,
|
||||
_Principal(),
|
||||
tenant_id="tenant-1",
|
||||
requests=(_request(),),
|
||||
)[0]
|
||||
|
||||
self.assertEqual("tenant_mismatch", tenant_result.reason_code)
|
||||
self.assertEqual("download_permission_missing", permission_result.reason_code)
|
||||
self.assertEqual("file_access_denied", access_result.reason_code)
|
||||
self.assertFalse(access_result.available)
|
||||
|
||||
def test_asset_reference_does_not_drift_to_a_later_current_version(self) -> None:
|
||||
asset, version, blob = _objects()
|
||||
session = _Session(
|
||||
{
|
||||
(FileAsset, asset.id): asset,
|
||||
(FileVersion, version.id): version,
|
||||
(FileBlob, blob.id): blob,
|
||||
}
|
||||
)
|
||||
|
||||
result = FilesPostboxReferenceService().resolve_postbox_references(
|
||||
session,
|
||||
_Principal(),
|
||||
tenant_id="tenant-1",
|
||||
requests=(_request("file", asset.id),),
|
||||
)[0]
|
||||
|
||||
self.assertFalse(result.available)
|
||||
self.assertEqual("exact_version_required", result.reason_code)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,97 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from govoplan_access.backend.db import models as access_models # noqa: F401
|
||||
from govoplan_core.core.provider_governance import ExternalProviderStateContext
|
||||
from govoplan_core.db.base import Base
|
||||
from govoplan_files.backend.db.models import FileConnectorProfile, FileConnectorSpace
|
||||
from govoplan_files.backend.manifest import manifest
|
||||
from govoplan_files.backend.provider_state import (
|
||||
REMOTE_STORAGE_PROVIDER_ID,
|
||||
remote_storage_provider_states,
|
||||
)
|
||||
|
||||
|
||||
class FilesProviderStateTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.engine = create_engine("sqlite+pysqlite:///:memory:", future=True)
|
||||
Base.metadata.create_all(
|
||||
self.engine,
|
||||
tables=(FileConnectorProfile.__table__, FileConnectorSpace.__table__),
|
||||
)
|
||||
self.session = sessionmaker(bind=self.engine, expire_on_commit=False)()
|
||||
self.profile = FileConnectorProfile(
|
||||
id="profile-1",
|
||||
tenant_id="tenant-1",
|
||||
scope_type="tenant",
|
||||
scope_id="tenant-1",
|
||||
label="Remote WebDAV",
|
||||
provider="webdav",
|
||||
endpoint_url="https://files.example.test/dav/",
|
||||
enabled=True,
|
||||
)
|
||||
self.session.add_all(
|
||||
(
|
||||
self.profile,
|
||||
FileConnectorSpace(
|
||||
id="space-1",
|
||||
tenant_id="tenant-1",
|
||||
owner_type="tenant",
|
||||
label="Documents",
|
||||
connector_profile_id=self.profile.id,
|
||||
provider="webdav",
|
||||
remote_path="/documents",
|
||||
read_only=True,
|
||||
is_active=True,
|
||||
),
|
||||
)
|
||||
)
|
||||
self.session.commit()
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self.session.close()
|
||||
self.engine.dispose()
|
||||
|
||||
def test_state_does_not_overstate_live_health_or_expose_endpoint(self) -> None:
|
||||
state = remote_storage_provider_states(
|
||||
ExternalProviderStateContext(session=self.session, tenant_id="tenant-1")
|
||||
)[0]
|
||||
|
||||
self.assertEqual("external_mirror", state.authority_mode)
|
||||
self.assertEqual("unknown", state.health)
|
||||
self.assertEqual("attention", state.recovery)
|
||||
self.assertEqual(1, state.metrics["active_spaces"])
|
||||
self.assertNotIn("files.example.test", str(state.to_dict()))
|
||||
self.assertEqual(
|
||||
(),
|
||||
remote_storage_provider_states(
|
||||
ExternalProviderStateContext(
|
||||
session=self.session,
|
||||
tenant_id="tenant-2",
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
def test_unimplemented_profile_is_fail_closed_and_manifest_registered(self) -> None:
|
||||
self.profile.provider = "sharepoint"
|
||||
self.session.flush()
|
||||
|
||||
state = remote_storage_provider_states(
|
||||
ExternalProviderStateContext(session=self.session, tenant_id="tenant-1")
|
||||
)[0]
|
||||
|
||||
self.assertEqual("error", state.health)
|
||||
self.assertEqual("unsupported", state.recovery)
|
||||
self.assertEqual(REMOTE_STORAGE_PROVIDER_ID, manifest.external_providers[0].id)
|
||||
self.assertEqual(
|
||||
REMOTE_STORAGE_PROVIDER_ID,
|
||||
manifest.external_provider_state_providers[0].provider_id,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,38 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
import unittest
|
||||
|
||||
from govoplan_files.backend.manifest import manifest
|
||||
|
||||
|
||||
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
class FilesQuickAccessContractTests(unittest.TestCase):
|
||||
def test_manifest_declares_exact_version_result_and_fallback(self) -> None:
|
||||
tool = manifest.frontend.quick_access_tools[0]
|
||||
|
||||
self.assertEqual("files.recent", tool.id)
|
||||
self.assertEqual(("files.file-version",), tool.returned_reference_kinds)
|
||||
self.assertEqual("files.quick_access.files", tool.help_context_id)
|
||||
self.assertEqual("/files", tool.full_page_path)
|
||||
|
||||
def test_renderer_and_page_keep_recent_selection_and_upload_bounded(self) -> None:
|
||||
renderer = (
|
||||
REPOSITORY_ROOT / "webui/src/features/files/FileQuickAccess.tsx"
|
||||
).read_text(encoding="utf-8")
|
||||
page = (
|
||||
REPOSITORY_ROOT / "webui/src/features/files/FilesPage.tsx"
|
||||
).read_text(encoding="utf-8")
|
||||
|
||||
self.assertIn('sort: "recent", page_size: 7', renderer)
|
||||
self.assertIn('kind: "file-version"', renderer)
|
||||
self.assertIn("quickAccessLaunchState(launchContext)", renderer)
|
||||
self.assertIn('to="/files?quickAction=upload"', renderer)
|
||||
self.assertIn('parameters.get("quickAction") !== "upload"', page)
|
||||
self.assertIn('openDialog("upload"', page)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,108 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime
|
||||
from types import SimpleNamespace
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from govoplan_core.core.records import RecordContractError, RecordSourceLocator
|
||||
from govoplan_files.backend.db.models import FileAsset, FileBlob, FileVersion
|
||||
from govoplan_files.backend.record_source import FilesRecordSource
|
||||
|
||||
|
||||
@dataclass
|
||||
class Principal:
|
||||
tenant_id: str = "tenant-1"
|
||||
membership_id: str = "user-1"
|
||||
user: object = field(default_factory=lambda: SimpleNamespace(id="user-1"))
|
||||
|
||||
def has(self, scope: str) -> bool:
|
||||
return scope in {"files:file:read"}
|
||||
|
||||
|
||||
class FilesRecordSourceTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.session = MagicMock(spec=Session)
|
||||
self.asset = FileAsset(
|
||||
id="asset-1",
|
||||
tenant_id="tenant-1",
|
||||
owner_type="user",
|
||||
owner_user_id="user-1",
|
||||
current_version_id="version-1",
|
||||
display_path="Evidence/Decision.pdf",
|
||||
filename="Decision.pdf",
|
||||
)
|
||||
self.blob = FileBlob(
|
||||
id="blob-1",
|
||||
tenant_id="tenant-1",
|
||||
storage_backend="local",
|
||||
storage_key="tenant-1/blob-1",
|
||||
checksum_sha256="a" * 64,
|
||||
size_bytes=1024,
|
||||
integrity_status="verified",
|
||||
)
|
||||
self.version = FileVersion(
|
||||
id="version-1",
|
||||
tenant_id="tenant-1",
|
||||
file_asset_id="asset-1",
|
||||
blob_id="blob-1",
|
||||
version_number=1,
|
||||
filename_at_upload="Decision.pdf",
|
||||
display_path_at_upload="Evidence/Decision.pdf",
|
||||
content_type="application/pdf",
|
||||
size_bytes=1024,
|
||||
checksum_sha256="a" * 64,
|
||||
)
|
||||
self.version.created_at = datetime.now(UTC)
|
||||
version_query = self.session.query.return_value.filter.return_value
|
||||
version_query.filter.return_value.one_or_none.return_value = self.version
|
||||
self.session.get.return_value = self.blob
|
||||
|
||||
def locator(self) -> RecordSourceLocator:
|
||||
return RecordSourceLocator(
|
||||
tenant_id="tenant-1",
|
||||
source_module="files",
|
||||
resource_type="file_version",
|
||||
resource_id="asset-1",
|
||||
source_revision="version-1",
|
||||
)
|
||||
|
||||
def test_resolves_exact_authorized_version_and_digest(self) -> None:
|
||||
with patch(
|
||||
"govoplan_files.backend.record_source.get_asset_for_user",
|
||||
return_value=self.asset,
|
||||
):
|
||||
result = FilesRecordSource().resolve(
|
||||
self.session,
|
||||
Principal(),
|
||||
locator=self.locator(),
|
||||
purpose="document decision basis",
|
||||
)
|
||||
|
||||
self.assertEqual("version-1", result.locator.source_revision)
|
||||
self.assertEqual("a" * 64, result.content_sha256)
|
||||
self.assertEqual("verified", result.metadata["integrity_status"])
|
||||
|
||||
def test_quarantined_version_fails_closed(self) -> None:
|
||||
self.blob.quarantined_at = datetime.now(UTC)
|
||||
self.session.flush()
|
||||
with (
|
||||
patch(
|
||||
"govoplan_files.backend.record_source.get_asset_for_user",
|
||||
return_value=self.asset,
|
||||
),
|
||||
self.assertRaisesRegex(RecordContractError, "integrity gate"),
|
||||
):
|
||||
FilesRecordSource().resolve(
|
||||
self.session,
|
||||
Principal(),
|
||||
locator=self.locator(),
|
||||
purpose="document decision basis",
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,13 +1,79 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from collections import Counter
|
||||
from inspect import signature
|
||||
|
||||
from govoplan_files.backend.router import router
|
||||
from govoplan_files.backend.routes.assets import router as assets_router
|
||||
from govoplan_files.backend.routes.connector_io import router as connector_io_router
|
||||
from govoplan_files.backend.routes.connector_profiles import (
|
||||
router as connector_profiles_router,
|
||||
)
|
||||
from govoplan_files.backend.routes.connector_settings import (
|
||||
router as connector_settings_router,
|
||||
)
|
||||
from govoplan_files.backend.routes.folders import router as folders_router
|
||||
from govoplan_files.backend.routes.form_evidence import router as form_evidence_router
|
||||
from govoplan_files.backend.routes.integrity import router as integrity_router
|
||||
from govoplan_files.backend.routes.listing import router as listing_router
|
||||
from govoplan_files.backend.routes.lifecycle import router as lifecycle_router
|
||||
from govoplan_files.backend.routes.shares import router as shares_router
|
||||
from govoplan_files.backend.routes.spaces import router as spaces_router
|
||||
from govoplan_files.backend.routes.transfers import router as transfers_router
|
||||
from govoplan_files.backend.routes.uploads import router as uploads_router
|
||||
|
||||
|
||||
class FilesRouterContractTests(unittest.TestCase):
|
||||
@staticmethod
|
||||
def _operation_keys(candidate_router) -> list[tuple[str, str]]:
|
||||
return [
|
||||
(method, route.path)
|
||||
for route in candidate_router.routes
|
||||
for method in sorted(route.methods or ())
|
||||
]
|
||||
|
||||
def test_composed_router_contains_every_workflow_operation_once(self) -> None:
|
||||
workflow_routers = (
|
||||
spaces_router,
|
||||
folders_router,
|
||||
form_evidence_router,
|
||||
integrity_router,
|
||||
lifecycle_router,
|
||||
listing_router,
|
||||
uploads_router,
|
||||
connector_settings_router,
|
||||
connector_io_router,
|
||||
connector_profiles_router,
|
||||
assets_router,
|
||||
shares_router,
|
||||
transfers_router,
|
||||
)
|
||||
expected = [
|
||||
operation
|
||||
for workflow_router in workflow_routers
|
||||
for operation in self._operation_keys(workflow_router)
|
||||
]
|
||||
actual = self._operation_keys(router)
|
||||
|
||||
self.assertEqual(expected, actual)
|
||||
self.assertEqual(62, len(actual))
|
||||
self.assertFalse(
|
||||
[operation for operation, count in Counter(actual).items() if count > 1]
|
||||
)
|
||||
|
||||
def test_archive_preview_and_confirmation_routes_are_exposed(self) -> None:
|
||||
routes = {
|
||||
(tuple(sorted(route.methods or ())), route.path) for route in router.routes
|
||||
}
|
||||
|
||||
self.assertIn((("POST",), "/files/archive-preview"), routes)
|
||||
self.assertIn((("POST",), "/files/archive-confirm"), routes)
|
||||
|
||||
def test_connector_routes_keep_existing_api_paths(self) -> None:
|
||||
routes = {(tuple(sorted(route.methods or ())), route.path) for route in router.routes}
|
||||
routes = {
|
||||
(tuple(sorted(route.methods or ())), route.path) for route in router.routes
|
||||
}
|
||||
|
||||
expected = {
|
||||
(("GET",), "/files/connectors/providers"),
|
||||
@@ -17,20 +83,69 @@ class FilesRouterContractTests(unittest.TestCase):
|
||||
(("GET",), "/files/connectors/profiles/{profile_id}/browse"),
|
||||
(("POST",), "/files/connectors/profiles/{profile_id}/import"),
|
||||
(("POST",), "/files/connectors/profiles/{profile_id}/sync"),
|
||||
(("POST",), "/files/connector-spaces/{space_id}/write-back"),
|
||||
(("GET",), "/files/connectors/credentials"),
|
||||
(("POST",), "/files/connectors/credentials"),
|
||||
(("GET",), "/files/connector-spaces"),
|
||||
(("POST",), "/files/connector-spaces"),
|
||||
(("PATCH",), "/files/connector-spaces/{space_id}"),
|
||||
(("DELETE",), "/files/connector-spaces/{space_id}"),
|
||||
}
|
||||
|
||||
self.assertTrue(expected.issubset(routes))
|
||||
|
||||
def test_bulk_organize_routes_keep_existing_api_paths(self) -> None:
|
||||
routes = {(tuple(sorted(route.methods or ())), route.path) for route in router.routes}
|
||||
routes = {
|
||||
(tuple(sorted(route.methods or ())), route.path) for route in router.routes
|
||||
}
|
||||
|
||||
self.assertIn((("POST",), "/files/bulk-rename"), routes)
|
||||
self.assertIn((("POST",), "/files/transfer"), routes)
|
||||
|
||||
def test_share_lifecycle_routes_are_exposed(self) -> None:
|
||||
routes = {
|
||||
(tuple(sorted(route.methods or ())), route.path) for route in router.routes
|
||||
}
|
||||
|
||||
self.assertIn((("GET",), "/files/{file_id}/shares"), routes)
|
||||
self.assertIn((("POST",), "/files/{file_id}/shares"), routes)
|
||||
self.assertIn((("DELETE",), "/files/{file_id}/shares/{share_id}"), routes)
|
||||
self.assertIn((("GET",), "/files/{file_id}/share-target-options"), routes)
|
||||
|
||||
def test_exact_version_download_route_is_exposed(self) -> None:
|
||||
routes = {
|
||||
(tuple(sorted(route.methods or ())), route.path) for route in router.routes
|
||||
}
|
||||
|
||||
self.assertIn(
|
||||
(("GET",), "/files/{file_id}/versions/{version_id}/download"),
|
||||
routes,
|
||||
)
|
||||
|
||||
def test_governed_lifecycle_routes_are_exposed(self) -> None:
|
||||
routes = {
|
||||
(tuple(sorted(route.methods or ())), route.path) for route in router.routes
|
||||
}
|
||||
|
||||
self.assertIn((("PATCH",), "/files/{file_id}/lifecycle"), routes)
|
||||
self.assertIn((("POST",), "/files/assets/{file_id}/restore"), routes)
|
||||
self.assertIn((("POST",), "/files/purge/preview"), routes)
|
||||
self.assertIn((("POST",), "/files/purge/execute"), routes)
|
||||
self.assertIn((("POST",), "/files/purge/blobs"), routes)
|
||||
|
||||
def test_file_listing_exposes_structured_property_filters(self) -> None:
|
||||
route = next(
|
||||
route
|
||||
for route in listing_router.routes
|
||||
if route.path == "/files" and "GET" in (route.methods or ())
|
||||
)
|
||||
parameters = signature(route.endpoint).parameters
|
||||
|
||||
self.assertIn("campaign_usage", parameters)
|
||||
self.assertIn("audit_relevant", parameters)
|
||||
self.assertIn("sort", parameters)
|
||||
self.assertIn("total", route.response_model.model_fields)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -0,0 +1,294 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import socket
|
||||
import sys
|
||||
import threading
|
||||
import types
|
||||
import unittest
|
||||
from importlib.util import find_spec
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from unittest.mock import patch
|
||||
|
||||
from govoplan_core.security.outbound_http import (
|
||||
OutboundHttpBlocked,
|
||||
create_outbound_connection as core_create_outbound_connection,
|
||||
)
|
||||
from govoplan_files.backend.storage.sdk_peer_pinning import (
|
||||
SdkPeerPinningError,
|
||||
_botocore_transport_types,
|
||||
create_pinned_s3_client,
|
||||
install_pinned_smb_transport,
|
||||
pinned_smb_connection_cache,
|
||||
)
|
||||
|
||||
|
||||
class _Socket:
|
||||
def __init__(self) -> None:
|
||||
self.connected_to: object | None = None
|
||||
|
||||
def settimeout(self, _value: object) -> None:
|
||||
return None
|
||||
|
||||
def setsockopt(self, *_args: object) -> None:
|
||||
return None
|
||||
|
||||
def connect(self, value: object) -> None:
|
||||
self.connected_to = value
|
||||
|
||||
def close(self) -> None:
|
||||
return None
|
||||
|
||||
|
||||
class _S3Handler(BaseHTTPRequestHandler):
|
||||
def do_GET(self) -> None: # noqa: N802 - stdlib handler contract
|
||||
body = (
|
||||
b'<?xml version="1.0" encoding="UTF-8"?>'
|
||||
b'<ListAllMyBucketsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">'
|
||||
b'<Owner><ID>govoplan</ID><DisplayName>GovOPlaN</DisplayName></Owner>'
|
||||
b'<Buckets><Bucket><Name>evidence</Name><CreationDate>2026-08-04T00:00:00Z</CreationDate>'
|
||||
b'</Bucket></Buckets></ListAllMyBucketsResult>'
|
||||
)
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/xml")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def log_message(self, _format: str, *_args: object) -> None:
|
||||
return None
|
||||
|
||||
|
||||
@unittest.skipUnless(find_spec("botocore") and find_spec("boto3"), "boto3 extra is not installed")
|
||||
class S3PeerPinningTests(unittest.TestCase):
|
||||
def test_client_is_born_with_the_pinned_http_session(self) -> None:
|
||||
client = create_pinned_s3_client(
|
||||
endpoint_url="http://127.0.0.1:9000",
|
||||
region_name="eu-central-1",
|
||||
aws_access_key_id="access",
|
||||
aws_secret_access_key="secret",
|
||||
)
|
||||
try:
|
||||
session_type, _, _ = _botocore_transport_types()
|
||||
self.assertIsInstance(client._endpoint.http_session, session_type)
|
||||
self.assertEqual({}, client._endpoint.http_session._proxy_config._proxies)
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
def test_each_connection_attempt_resolves_and_pins_again(self) -> None:
|
||||
_, http_connection, _ = _botocore_transport_types()
|
||||
connection = http_connection(host="objects.example.test", port=80)
|
||||
sockets = [_Socket(), _Socket()]
|
||||
with patch(
|
||||
"govoplan_files.backend.storage.sdk_peer_pinning.create_outbound_connection",
|
||||
side_effect=sockets,
|
||||
) as connector:
|
||||
self.assertIs(sockets[0], connection._new_conn())
|
||||
self.assertIs(sockets[1], connection._new_conn())
|
||||
|
||||
self.assertEqual(2, connector.call_count)
|
||||
self.assertEqual("objects.example.test", connector.call_args.args[0])
|
||||
|
||||
def test_real_sdk_request_uses_the_pinned_socket_path(self) -> None:
|
||||
server = ThreadingHTTPServer(("127.0.0.1", 0), _S3Handler)
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
endpoint = f"http://127.0.0.1:{server.server_port}"
|
||||
try:
|
||||
with patch.dict(
|
||||
"os.environ",
|
||||
{"APP_ENV": "production", "GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS": "true"},
|
||||
), patch(
|
||||
"govoplan_files.backend.storage.sdk_peer_pinning.create_outbound_connection",
|
||||
wraps=core_create_outbound_connection,
|
||||
) as connector:
|
||||
client = create_pinned_s3_client(
|
||||
endpoint_url=endpoint,
|
||||
region_name="eu-central-1",
|
||||
aws_access_key_id="access",
|
||||
aws_secret_access_key="secret",
|
||||
)
|
||||
try:
|
||||
response = client.list_buckets()
|
||||
finally:
|
||||
client.close()
|
||||
finally:
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
thread.join(timeout=2)
|
||||
|
||||
self.assertEqual("evidence", response["Buckets"][0]["Name"])
|
||||
self.assertEqual("127.0.0.1", connector.call_args.args[0])
|
||||
|
||||
def test_every_sdk_selected_origin_uses_a_pinned_pool(self) -> None:
|
||||
session_type, http_connection, https_connection = _botocore_transport_types()
|
||||
session = session_type(proxies={})
|
||||
try:
|
||||
first = session._manager.connection_from_url("http://one.example.test/root")
|
||||
redirected = session._manager.connection_from_url("https://two.example.test/root")
|
||||
self.assertIs(first.ConnectionCls, http_connection)
|
||||
self.assertIs(redirected.ConnectionCls, https_connection)
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
def test_tls_connection_retains_the_configured_authority_for_sni(self) -> None:
|
||||
_, _, https_connection = _botocore_transport_types()
|
||||
connection = https_connection(host="objects.example.test", port=443)
|
||||
with patch(
|
||||
"govoplan_files.backend.storage.sdk_peer_pinning.create_outbound_connection",
|
||||
return_value=_Socket(),
|
||||
) as connector:
|
||||
connection._new_conn()
|
||||
|
||||
self.assertEqual("objects.example.test", connection.host)
|
||||
self.assertEqual("objects.example.test", connection._dns_host)
|
||||
self.assertEqual("objects.example.test", connector.call_args.args[0])
|
||||
|
||||
def test_mixed_answer_and_peer_change_fail_closed(self) -> None:
|
||||
_, http_connection, _ = _botocore_transport_types()
|
||||
connection = http_connection(host="objects.example.test", port=80)
|
||||
records = [
|
||||
[(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("93.184.216.34", 80))],
|
||||
[
|
||||
(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("93.184.216.34", 80)),
|
||||
(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("10.0.0.5", 80)),
|
||||
],
|
||||
]
|
||||
with patch.dict(
|
||||
"os.environ",
|
||||
{"APP_ENV": "production", "GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS": "false"},
|
||||
), patch(
|
||||
"govoplan_core.security.outbound_http.socket.getaddrinfo",
|
||||
side_effect=records,
|
||||
), patch(
|
||||
"govoplan_core.security.outbound_http.socket.socket",
|
||||
return_value=_Socket(),
|
||||
):
|
||||
connection._new_conn()
|
||||
with self.assertRaisesRegex(Exception, "non-public network"):
|
||||
connection._new_conn()
|
||||
|
||||
|
||||
class _FakeTcp:
|
||||
def __init__(self, server: str, port: int, timeout: float | None = None) -> None:
|
||||
self.server = server
|
||||
self.port = port
|
||||
self.timeout = timeout
|
||||
self.connected = False
|
||||
self._sock = None
|
||||
self._sock_lock = threading.Lock()
|
||||
|
||||
|
||||
class _FakeConnection:
|
||||
def __init__(self, _guid: object, server: str, port: int, *, require_signing: bool) -> None:
|
||||
self.server = server
|
||||
self.port = port
|
||||
self.require_signing = require_signing
|
||||
self.transport = _FakeTcp(server, port)
|
||||
self.session_table: dict[str, object] = {}
|
||||
|
||||
def connect(self, *, timeout: float) -> None:
|
||||
self.transport.timeout = timeout
|
||||
self.transport.connect()
|
||||
|
||||
|
||||
class _FakeSession:
|
||||
def __init__(
|
||||
self,
|
||||
connection: _FakeConnection,
|
||||
*,
|
||||
username: str | None,
|
||||
password: str | None,
|
||||
require_encryption: bool,
|
||||
auth_protocol: str,
|
||||
) -> None:
|
||||
self.connection = connection
|
||||
self.username = username
|
||||
self.password = password
|
||||
self.encrypt_data = require_encryption
|
||||
self.auth_protocol = auth_protocol
|
||||
self.encrypt = require_encryption
|
||||
|
||||
def connect(self) -> None:
|
||||
self.connection.session_table[self.username or "anonymous"] = self
|
||||
|
||||
|
||||
class SmbPeerPinningTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
pinned_smb_connection_cache.cache_clear()
|
||||
|
||||
def _modules(self) -> tuple[types.ModuleType, dict[str, types.ModuleType]]:
|
||||
smbclient = types.ModuleType("smbclient")
|
||||
pool = types.ModuleType("smbclient._pool")
|
||||
|
||||
def original_register_session(
|
||||
server: str,
|
||||
username: str | None = None,
|
||||
password: str | None = None,
|
||||
port: int = 445,
|
||||
encrypt: bool | None = None,
|
||||
connection_timeout: float = 60,
|
||||
connection_cache: dict[str, object] | None = None,
|
||||
auth_protocol: str = "negotiate",
|
||||
require_signing: bool = True,
|
||||
) -> None:
|
||||
del server, username, password, port, encrypt, connection_timeout
|
||||
del connection_cache, auth_protocol, require_signing
|
||||
|
||||
pool.register_session = original_register_session
|
||||
pool.ClientConfig = lambda: types.SimpleNamespace(client_guid="guid")
|
||||
connection = types.ModuleType("smbprotocol.connection")
|
||||
connection.Connection = _FakeConnection
|
||||
connection.Tcp = _FakeTcp
|
||||
session = types.ModuleType("smbprotocol.session")
|
||||
session.Session = _FakeSession
|
||||
transport = types.ModuleType("smbprotocol.transport")
|
||||
transport.Tcp = _FakeTcp
|
||||
return smbclient, {
|
||||
"smbclient._pool": pool,
|
||||
"smbprotocol.connection": connection,
|
||||
"smbprotocol.session": session,
|
||||
"smbprotocol.transport": transport,
|
||||
}
|
||||
|
||||
def test_initial_reconnect_and_referral_hosts_are_each_pinned(self) -> None:
|
||||
smbclient, modules = self._modules()
|
||||
sockets = [_Socket(), _Socket(), _Socket()]
|
||||
with patch.dict(sys.modules, modules), patch(
|
||||
"govoplan_files.backend.storage.sdk_peer_pinning.create_outbound_connection",
|
||||
side_effect=sockets,
|
||||
) as connector:
|
||||
install_pinned_smb_transport(smbclient)
|
||||
register = modules["smbclient._pool"].register_session
|
||||
cache: dict[str, object] = {}
|
||||
first = register("files.example.test", connection_cache=cache)
|
||||
first.connection.transport.connected = False
|
||||
register("files.example.test", connection_cache=cache)
|
||||
register("dfs-target.example.test", connection_cache=cache)
|
||||
|
||||
self.assertEqual(
|
||||
["files.example.test", "files.example.test", "dfs-target.example.test"],
|
||||
[call.args[0] for call in connector.call_args_list],
|
||||
)
|
||||
|
||||
def test_referral_to_disallowed_peer_fails_before_session_creation(self) -> None:
|
||||
smbclient, modules = self._modules()
|
||||
with patch.dict(sys.modules, modules), patch(
|
||||
"govoplan_files.backend.storage.sdk_peer_pinning.create_outbound_connection",
|
||||
side_effect=OutboundHttpBlocked("non-public network"),
|
||||
):
|
||||
install_pinned_smb_transport(smbclient)
|
||||
register = modules["smbclient._pool"].register_session
|
||||
with self.assertRaisesRegex(ValueError, "non-public network"):
|
||||
register("private-referral.example.test", connection_cache={})
|
||||
|
||||
def test_transport_tampering_after_installation_fails_closed(self) -> None:
|
||||
smbclient, modules = self._modules()
|
||||
with patch.dict(sys.modules, modules):
|
||||
install_pinned_smb_transport(smbclient)
|
||||
modules["smbprotocol.connection"].Tcp = _FakeTcp
|
||||
with self.assertRaisesRegex(SdkPeerPinningError, "changed after peer pinning"):
|
||||
install_pinned_smb_transport(smbclient)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,145 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
import unittest
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from govoplan_access.backend.db.models import Account, Group, User
|
||||
from govoplan_core.auth import ApiPrincipal
|
||||
from govoplan_core.core.access import PrincipalRef
|
||||
from govoplan_core.core.change_sequence import ChangeSequenceEntry
|
||||
from govoplan_core.core.events import EventObjectRef, EventTenantRef, PlatformEvent
|
||||
from govoplan_core.core.search import (
|
||||
SearchAuthorizationRequest,
|
||||
SearchBackfillRequest,
|
||||
SearchResourceReference,
|
||||
)
|
||||
from govoplan_core.db.base import Base
|
||||
from govoplan_files.backend.db.models import FileAsset, FileFolder, FileShare
|
||||
from govoplan_files.backend.search_source import (
|
||||
FilesSearchSource,
|
||||
PROVIDER_ID,
|
||||
)
|
||||
|
||||
|
||||
class FilesSearchSourceTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.engine = create_engine("sqlite://")
|
||||
Base.metadata.create_all(
|
||||
self.engine,
|
||||
tables=(
|
||||
Account.__table__,
|
||||
User.__table__,
|
||||
Group.__table__,
|
||||
FileAsset.__table__,
|
||||
FileFolder.__table__,
|
||||
FileShare.__table__,
|
||||
ChangeSequenceEntry.__table__,
|
||||
),
|
||||
)
|
||||
self.session = Session(self.engine)
|
||||
self.session.add_all(
|
||||
(
|
||||
Account(
|
||||
id="account-1",
|
||||
email="one@example.test",
|
||||
normalized_email="one@example.test",
|
||||
),
|
||||
User(
|
||||
id="user-1",
|
||||
tenant_id="tenant-1",
|
||||
account_id="account-1",
|
||||
email="one@example.test",
|
||||
),
|
||||
FileAsset(
|
||||
id="file-1",
|
||||
tenant_id="tenant-1",
|
||||
owner_type="user",
|
||||
owner_user_id="user-1",
|
||||
display_path="records/permit.pdf",
|
||||
filename="permit.pdf",
|
||||
description="Monthly permit evidence",
|
||||
),
|
||||
FileAsset(
|
||||
id="file-other",
|
||||
tenant_id="tenant-2",
|
||||
owner_type="user",
|
||||
display_path="other.pdf",
|
||||
filename="other.pdf",
|
||||
),
|
||||
)
|
||||
)
|
||||
self.session.commit()
|
||||
self.source = FilesSearchSource()
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self.session.close()
|
||||
self.engine.dispose()
|
||||
|
||||
def test_backfill_and_authorization_are_tenant_bounded(self) -> None:
|
||||
page = self.source.backfill(
|
||||
self.session,
|
||||
request=SearchBackfillRequest(
|
||||
tenant_id="tenant-1",
|
||||
provider_id=PROVIDER_ID,
|
||||
resource_type="file",
|
||||
rebuild_id="rebuild-1",
|
||||
),
|
||||
)
|
||||
self.assertEqual(("file-1",), tuple(doc.resource_id for doc in page.documents))
|
||||
reference = SearchResourceReference(
|
||||
tenant_id="tenant-1",
|
||||
module_id="files",
|
||||
resource_type="file",
|
||||
resource_id="file-1",
|
||||
)
|
||||
request = SearchAuthorizationRequest(reference=reference, source_revision="1")
|
||||
self.assertTrue(
|
||||
self.source.authorize(
|
||||
self.session,
|
||||
_principal({"files:file:read"}),
|
||||
requests=(request,),
|
||||
)[reference.key]
|
||||
)
|
||||
self.assertFalse(
|
||||
self.source.authorize(
|
||||
self.session,
|
||||
_principal(set()),
|
||||
requests=(request,),
|
||||
)[reference.key]
|
||||
)
|
||||
|
||||
def test_committed_file_event_produces_authoritative_upsert(self) -> None:
|
||||
event = PlatformEvent(
|
||||
type="files.file.updated",
|
||||
module_id="files",
|
||||
tenant=EventTenantRef(id="tenant-1"),
|
||||
resource=EventObjectRef(type="file", id="file-1"),
|
||||
)
|
||||
changes = self.source.index_changes_for_event(
|
||||
self.session,
|
||||
event=event,
|
||||
delivery_key="delivery-1",
|
||||
)
|
||||
self.assertEqual(1, len(changes))
|
||||
self.assertEqual("upsert", changes[0].kind)
|
||||
self.assertEqual(event.event_id, changes[0].cursor)
|
||||
|
||||
|
||||
def _principal(scopes: set[str]) -> ApiPrincipal:
|
||||
return ApiPrincipal(
|
||||
principal=PrincipalRef(
|
||||
account_id="account-1",
|
||||
membership_id="user-1",
|
||||
tenant_id="tenant-1",
|
||||
scopes=frozenset(scopes),
|
||||
),
|
||||
account=SimpleNamespace(id="account-1"),
|
||||
user=SimpleNamespace(id="user-1"),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,237 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from datetime import timedelta
|
||||
from unittest.mock import patch
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.dialects import postgresql
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from govoplan_access.backend.db.models import Account, Group, User
|
||||
from govoplan_core.core.change_sequence import ChangeSequenceEntry
|
||||
from govoplan_core.db.base import Base
|
||||
from govoplan_files.backend.db.models import FileAsset, FileShare
|
||||
from govoplan_files.backend.storage.common import FileStorageError, utcnow
|
||||
from govoplan_files.backend.storage.files import (
|
||||
_asset_visibility_query_for_user,
|
||||
get_asset_for_user,
|
||||
list_file_shares,
|
||||
revoke_file_share,
|
||||
share_file,
|
||||
)
|
||||
|
||||
|
||||
TENANT_ID = "tenant-1"
|
||||
OWNER_ID = "owner-1"
|
||||
RECIPIENT_ID = "recipient-1"
|
||||
GROUP_ID = "group-1"
|
||||
|
||||
|
||||
class FileShareLifecycleTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.engine = create_engine("sqlite:///:memory:", future=True)
|
||||
Base.metadata.create_all(
|
||||
bind=self.engine,
|
||||
tables=[
|
||||
Account.__table__,
|
||||
User.__table__,
|
||||
Group.__table__,
|
||||
ChangeSequenceEntry.__table__,
|
||||
FileAsset.__table__,
|
||||
FileShare.__table__,
|
||||
],
|
||||
)
|
||||
self.session = sessionmaker(bind=self.engine, future=True)()
|
||||
self.asset = FileAsset(
|
||||
id="file-1",
|
||||
tenant_id=TENANT_ID,
|
||||
owner_type="user",
|
||||
owner_user_id=OWNER_ID,
|
||||
display_path="shared.pdf",
|
||||
filename="shared.pdf",
|
||||
)
|
||||
self.session.add(self.asset)
|
||||
self.session.commit()
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self.session.close()
|
||||
self.engine.dispose()
|
||||
|
||||
@patch(
|
||||
"govoplan_files.backend.storage.files.user_group_ids",
|
||||
return_value=[],
|
||||
)
|
||||
def test_expired_share_stops_access_immediately(self, _groups) -> None:
|
||||
self.session.add(
|
||||
FileShare(
|
||||
id="expired-share",
|
||||
tenant_id=TENANT_ID,
|
||||
file_asset_id=self.asset.id,
|
||||
target_type="user",
|
||||
target_id=RECIPIENT_ID,
|
||||
permission="read",
|
||||
expires_at=utcnow() - timedelta(seconds=1),
|
||||
)
|
||||
)
|
||||
self.session.commit()
|
||||
|
||||
with self.assertRaisesRegex(FileStorageError, "No access"):
|
||||
get_asset_for_user(
|
||||
self.session,
|
||||
tenant_id=TENANT_ID,
|
||||
user_id=RECIPIENT_ID,
|
||||
asset_id=self.asset.id,
|
||||
)
|
||||
self.assertEqual(
|
||||
[],
|
||||
list_file_shares(
|
||||
self.session,
|
||||
tenant_id=TENANT_ID,
|
||||
asset_id=self.asset.id,
|
||||
),
|
||||
)
|
||||
|
||||
@patch(
|
||||
"govoplan_files.backend.storage.files.user_group_ids",
|
||||
return_value=[GROUP_ID],
|
||||
)
|
||||
def test_independent_active_grant_survives_other_expiry(self, _groups) -> None:
|
||||
self.session.add_all(
|
||||
[
|
||||
FileShare(
|
||||
id="expired-user-share",
|
||||
tenant_id=TENANT_ID,
|
||||
file_asset_id=self.asset.id,
|
||||
target_type="user",
|
||||
target_id=RECIPIENT_ID,
|
||||
permission="read",
|
||||
expires_at=utcnow() - timedelta(seconds=1),
|
||||
),
|
||||
FileShare(
|
||||
id="active-group-share",
|
||||
tenant_id=TENANT_ID,
|
||||
file_asset_id=self.asset.id,
|
||||
target_type="group",
|
||||
target_id=GROUP_ID,
|
||||
permission="read",
|
||||
expires_at=utcnow() + timedelta(hours=1),
|
||||
),
|
||||
]
|
||||
)
|
||||
self.session.commit()
|
||||
|
||||
result = get_asset_for_user(
|
||||
self.session,
|
||||
tenant_id=TENANT_ID,
|
||||
user_id=RECIPIENT_ID,
|
||||
asset_id=self.asset.id,
|
||||
)
|
||||
|
||||
self.assertEqual(self.asset.id, result.id)
|
||||
self.assertEqual(
|
||||
["active-group-share"],
|
||||
[
|
||||
share.id
|
||||
for share in list_file_shares(
|
||||
self.session,
|
||||
tenant_id=TENANT_ID,
|
||||
asset_id=self.asset.id,
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
@patch(
|
||||
"govoplan_files.backend.storage.files.ensure_share_target_exists",
|
||||
return_value=None,
|
||||
)
|
||||
def test_revoke_is_idempotent_and_future_expiry_is_persisted(
|
||||
self, _target_exists
|
||||
) -> None:
|
||||
expiry = utcnow() + timedelta(days=1)
|
||||
share = share_file(
|
||||
self.session,
|
||||
tenant_id=TENANT_ID,
|
||||
asset=self.asset,
|
||||
target_type="user",
|
||||
target_id=RECIPIENT_ID,
|
||||
permission="read",
|
||||
user_id=OWNER_ID,
|
||||
expires_at=expiry,
|
||||
)
|
||||
self.session.commit()
|
||||
|
||||
revoked, first_changed = revoke_file_share(
|
||||
self.session,
|
||||
tenant_id=TENANT_ID,
|
||||
asset_id=self.asset.id,
|
||||
share_id=share.id,
|
||||
user_id=OWNER_ID,
|
||||
)
|
||||
self.session.commit()
|
||||
repeated, second_changed = revoke_file_share(
|
||||
self.session,
|
||||
tenant_id=TENANT_ID,
|
||||
asset_id=self.asset.id,
|
||||
share_id=share.id,
|
||||
user_id=OWNER_ID,
|
||||
)
|
||||
|
||||
self.assertTrue(first_changed)
|
||||
self.assertFalse(second_changed)
|
||||
self.assertEqual(OWNER_ID, revoked.revoked_by_user_id)
|
||||
self.assertEqual(revoked.revoked_at, repeated.revoked_at)
|
||||
self.assertEqual([], list_file_shares(
|
||||
self.session,
|
||||
tenant_id=TENANT_ID,
|
||||
asset_id=self.asset.id,
|
||||
))
|
||||
self.assertEqual(
|
||||
[share.id],
|
||||
[
|
||||
item.id
|
||||
for item in list_file_shares(
|
||||
self.session,
|
||||
tenant_id=TENANT_ID,
|
||||
asset_id=self.asset.id,
|
||||
include_inactive=True,
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
@patch(
|
||||
"govoplan_files.backend.storage.files.ensure_share_target_exists",
|
||||
return_value=None,
|
||||
)
|
||||
def test_past_expiry_is_rejected(self, _target_exists) -> None:
|
||||
with self.assertRaisesRegex(FileStorageError, "future"):
|
||||
share_file(
|
||||
self.session,
|
||||
tenant_id=TENANT_ID,
|
||||
asset=self.asset,
|
||||
target_type="user",
|
||||
target_id=RECIPIENT_ID,
|
||||
permission="read",
|
||||
user_id=OWNER_ID,
|
||||
expires_at=utcnow() - timedelta(seconds=1),
|
||||
)
|
||||
|
||||
@patch(
|
||||
"govoplan_files.backend.storage.files.user_group_ids",
|
||||
return_value=[],
|
||||
)
|
||||
def test_visibility_query_is_postgresql_json_safe(self, _groups) -> None:
|
||||
query = _asset_visibility_query_for_user(
|
||||
self.session,
|
||||
tenant_id=TENANT_ID,
|
||||
user_id=RECIPIENT_ID,
|
||||
)
|
||||
|
||||
compiled = str(query.statement.compile(dialect=postgresql.dialect()))
|
||||
|
||||
self.assertNotIn("SELECT DISTINCT", compiled.upper())
|
||||
self.assertIn("EXISTS", compiled.upper())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,19 +1,26 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
from types import ModuleType
|
||||
import sys
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, PropertyMock, patch
|
||||
|
||||
from govoplan_files.backend.storage.backends import S3StorageBackend, StorageBackendError
|
||||
|
||||
|
||||
def _backend() -> S3StorageBackend:
|
||||
def _backend(
|
||||
*,
|
||||
endpoint_url: str = "https://objects.example.test",
|
||||
deployment_managed: bool = False,
|
||||
) -> S3StorageBackend:
|
||||
return S3StorageBackend(
|
||||
bucket="files",
|
||||
endpoint_url="https://objects.example.test",
|
||||
endpoint_url=endpoint_url,
|
||||
region_name="test",
|
||||
access_key_id="access",
|
||||
secret_access_key="secret",
|
||||
deployment_managed=deployment_managed,
|
||||
)
|
||||
|
||||
|
||||
@@ -32,6 +39,48 @@ class S3StorageBackendTests(unittest.TestCase):
|
||||
), self.assertRaisesRegex(StorageBackendError, "until that transport supports.*DNS/IP pinning"):
|
||||
_backend().client
|
||||
|
||||
def test_installer_managed_garage_uses_exact_endpoint_and_path_style(self) -> None:
|
||||
boto3 = ModuleType("boto3")
|
||||
boto3.client = MagicMock(return_value=object())
|
||||
botocore = ModuleType("botocore")
|
||||
botocore.__path__ = []
|
||||
botocore_config = ModuleType("botocore.config")
|
||||
|
||||
class Config:
|
||||
def __init__(self, **values):
|
||||
self.values = values
|
||||
|
||||
botocore_config.Config = Config
|
||||
with patch.dict(
|
||||
sys.modules,
|
||||
{
|
||||
"boto3": boto3,
|
||||
"botocore": botocore,
|
||||
"botocore.config": botocore_config,
|
||||
},
|
||||
):
|
||||
_backend(
|
||||
endpoint_url="http://garage:3900",
|
||||
deployment_managed=True,
|
||||
).client
|
||||
|
||||
_args, kwargs = boto3.client.call_args
|
||||
self.assertEqual("http://garage:3900", kwargs["endpoint_url"])
|
||||
self.assertEqual(
|
||||
{"s3": {"addressing_style": "path"}},
|
||||
kwargs["config"].values,
|
||||
)
|
||||
|
||||
def test_installer_managed_garage_rejects_any_other_endpoint(self) -> None:
|
||||
with self.assertRaisesRegex(
|
||||
StorageBackendError,
|
||||
"restricted to http://garage:3900",
|
||||
):
|
||||
_backend(
|
||||
endpoint_url="http://other-s3:3900",
|
||||
deployment_managed=True,
|
||||
).client
|
||||
|
||||
def test_get_bytes_rejects_declared_oversize_object_without_reading(self) -> None:
|
||||
body = MagicMock()
|
||||
client = MagicMock()
|
||||
|
||||
@@ -0,0 +1,646 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from govoplan_access.backend.db.models import Account, Group, User
|
||||
from govoplan_core.core.recovery import (
|
||||
RecoveryCheckpoint,
|
||||
RecoveryOperation,
|
||||
RecoveryStatus,
|
||||
)
|
||||
from govoplan_core.core.change_sequence import ChangeSequenceEntry
|
||||
from govoplan_core.core.runtime_coordination import (
|
||||
DistributedLease,
|
||||
RuntimeIdentity,
|
||||
bind_process_runtime_identity,
|
||||
)
|
||||
from govoplan_core.db.base import Base
|
||||
from govoplan_core.db.session import configure_database, reset_database
|
||||
from govoplan_files.backend.db.models import (
|
||||
FileAsset,
|
||||
FileBlob,
|
||||
FileFormEvidenceGrant,
|
||||
FileIntegrityFinding,
|
||||
FileIntegrityScan,
|
||||
FileShare,
|
||||
FileVersion,
|
||||
)
|
||||
from govoplan_files.backend.storage.backends import (
|
||||
LocalFilesystemStorageBackend,
|
||||
)
|
||||
from govoplan_files.backend.storage.common import FileStorageError
|
||||
from govoplan_files.backend.storage.files import _get_or_create_blob
|
||||
from govoplan_files.backend.storage.integrity import cleanup_orphan_finding
|
||||
from govoplan_files.backend.storage.recovery import begin_blob_write_recovery
|
||||
from govoplan_files.backend.storage.connector_profiles import ConnectorProfile
|
||||
from govoplan_files.backend.storage.connector_writes import write_connector_file
|
||||
from govoplan_files.backend.storage.lifecycle import (
|
||||
execute_asset_purge,
|
||||
garbage_collect_unreferenced_blobs,
|
||||
preview_asset_purge,
|
||||
)
|
||||
|
||||
|
||||
TENANT_ID = "tenant-1"
|
||||
USER_ID = "user-1"
|
||||
|
||||
|
||||
class _MissingS3Object(Exception):
|
||||
response = {
|
||||
"Error": {"Code": "NoSuchKey"},
|
||||
"ResponseMetadata": {"HTTPStatusCode": 404},
|
||||
}
|
||||
|
||||
|
||||
class _WriteS3Client:
|
||||
def __init__(
|
||||
self, *, tamper_metadata: bool = False, fail_post_write_probe: bool = False
|
||||
) -> None:
|
||||
self.object: dict[str, object] | None = None
|
||||
self.tamper_metadata = tamper_metadata
|
||||
self.fail_post_write_probe = fail_post_write_probe
|
||||
self.put_request: dict[str, object] | None = None
|
||||
self.closed = False
|
||||
self.write_completed = False
|
||||
|
||||
def head_object(self, **_kwargs: object) -> dict[str, object]:
|
||||
if self.write_completed and self.fail_post_write_probe:
|
||||
raise RuntimeError("provider unavailable")
|
||||
if self.object is None:
|
||||
raise _MissingS3Object()
|
||||
return dict(self.object)
|
||||
|
||||
def put_object(self, **kwargs: object) -> dict[str, object]:
|
||||
self.put_request = dict(kwargs)
|
||||
metadata = dict(kwargs.get("Metadata") or {})
|
||||
if self.tamper_metadata:
|
||||
metadata["govoplan-sha256"] = "0" * 64
|
||||
self.object = {
|
||||
"ETag": '"etag-written"',
|
||||
"VersionId": "version-written",
|
||||
"Metadata": metadata,
|
||||
}
|
||||
self.write_completed = True
|
||||
return {"ETag": '"etag-written"', "VersionId": "version-written"}
|
||||
|
||||
def close(self) -> None:
|
||||
self.closed = True
|
||||
|
||||
|
||||
class StorageRecoveryTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.temporary_directory = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.temporary_directory.cleanup)
|
||||
root = Path(self.temporary_directory.name)
|
||||
self.backend = LocalFilesystemStorageBackend(root / "objects")
|
||||
database_path = root / "recovery.sqlite3"
|
||||
self.engine = create_engine(f"sqlite:///{database_path}", future=True)
|
||||
Base.metadata.create_all(
|
||||
bind=self.engine,
|
||||
tables=[
|
||||
Account.__table__,
|
||||
User.__table__,
|
||||
Group.__table__,
|
||||
ChangeSequenceEntry.__table__,
|
||||
DistributedLease.__table__,
|
||||
RecoveryOperation.__table__,
|
||||
RecoveryCheckpoint.__table__,
|
||||
FileBlob.__table__,
|
||||
FileAsset.__table__,
|
||||
FileVersion.__table__,
|
||||
FileFormEvidenceGrant.__table__,
|
||||
FileShare.__table__,
|
||||
FileIntegrityScan.__table__,
|
||||
FileIntegrityFinding.__table__,
|
||||
],
|
||||
)
|
||||
configure_database(
|
||||
f"sqlite:///{database_path}",
|
||||
engine=self.engine,
|
||||
dispose_previous=True,
|
||||
)
|
||||
bind_process_runtime_identity(
|
||||
RuntimeIdentity(
|
||||
installation_id="files-recovery-test",
|
||||
node_id="node-1",
|
||||
incarnation="incarnation-1",
|
||||
role="api",
|
||||
software_version="test",
|
||||
composition_hash="a" * 64,
|
||||
)
|
||||
)
|
||||
self.Session = sessionmaker(
|
||||
bind=self.engine,
|
||||
expire_on_commit=False,
|
||||
future=True,
|
||||
)
|
||||
self.enterContext(
|
||||
patch(
|
||||
"govoplan_files.backend.storage.files._storage_backend_name",
|
||||
return_value=self.backend.name,
|
||||
)
|
||||
)
|
||||
self.enterContext(
|
||||
patch(
|
||||
"govoplan_files.backend.storage.files._storage_bucket_name",
|
||||
return_value="",
|
||||
)
|
||||
)
|
||||
self.session = self.Session()
|
||||
self.addCleanup(self._cleanup_runtime)
|
||||
|
||||
def _cleanup_runtime(self) -> None:
|
||||
self.session.close()
|
||||
bind_process_runtime_identity(None)
|
||||
reset_database()
|
||||
self.engine.dispose()
|
||||
|
||||
def test_committed_blob_write_is_independently_verified(self) -> None:
|
||||
observed_running_operation: list[bool] = []
|
||||
put_bytes = self.backend.put_bytes
|
||||
|
||||
class ObservingBackend:
|
||||
name = self.backend.name
|
||||
|
||||
def __getattr__(backend_self, name):
|
||||
return getattr(self.backend, name)
|
||||
|
||||
def put_bytes(backend_self, key, data, *, content_type=None):
|
||||
with self.Session() as evidence_session:
|
||||
operation = evidence_session.query(RecoveryOperation).one_or_none()
|
||||
observed_running_operation.append(
|
||||
operation is not None
|
||||
and operation.status == RecoveryStatus.RUNNING.value
|
||||
and len(operation.request_sha256) == 64
|
||||
)
|
||||
put_bytes(key, data, content_type=content_type)
|
||||
|
||||
observing_backend = ObservingBackend()
|
||||
|
||||
with patch(
|
||||
"govoplan_files.backend.storage.files.get_storage_backend",
|
||||
return_value=observing_backend,
|
||||
):
|
||||
blob = _get_or_create_blob(
|
||||
self.session,
|
||||
tenant_id=TENANT_ID,
|
||||
data=b"durable",
|
||||
filename="private-name.txt",
|
||||
content_type="text/plain",
|
||||
actor_id=USER_ID,
|
||||
)
|
||||
self.session.commit()
|
||||
|
||||
operation = self._only_operation()
|
||||
self.assertEqual(RecoveryStatus.SUCCEEDED.value, operation.status)
|
||||
# SQLite uses an explicit caller-transaction mode because it permits
|
||||
# only one writer. The intent becomes visible with the business commit;
|
||||
# PostgreSQL retains the independent pre-effect commit guarantee.
|
||||
self.assertEqual([False], observed_running_operation)
|
||||
self.assertEqual(
|
||||
"sqlite_caller_transaction",
|
||||
operation.metadata_["durability_mode"],
|
||||
)
|
||||
self.assertTrue(self.backend.exists(blob.storage_key))
|
||||
self.assertNotIn("private-name", blob.storage_key)
|
||||
self.assertEqual(".blob", Path(blob.storage_key).suffix)
|
||||
|
||||
def test_rolled_back_blob_write_is_compensated(self) -> None:
|
||||
with patch(
|
||||
"govoplan_files.backend.storage.files.get_storage_backend",
|
||||
return_value=self.backend,
|
||||
):
|
||||
blob = _get_or_create_blob(
|
||||
self.session,
|
||||
tenant_id=TENANT_ID,
|
||||
data=b"rollback",
|
||||
filename="rollback.txt",
|
||||
content_type="text/plain",
|
||||
actor_id=USER_ID,
|
||||
)
|
||||
storage_key = blob.storage_key
|
||||
self.assertTrue(self.backend.exists(storage_key))
|
||||
self.session.rollback()
|
||||
|
||||
operation = self._only_operation()
|
||||
self.assertEqual(RecoveryStatus.RECOVERED.value, operation.status)
|
||||
self.assertEqual(
|
||||
"sqlite_post_rollback_reconstruction",
|
||||
operation.metadata_["durability_mode"],
|
||||
)
|
||||
self.assertFalse(self.backend.exists(storage_key))
|
||||
with self.Session() as evidence_session:
|
||||
self.assertIsNone(evidence_session.get(FileBlob, blob.id))
|
||||
|
||||
def test_multiple_blob_writes_share_one_sqlite_business_transaction(self) -> None:
|
||||
with patch(
|
||||
"govoplan_files.backend.storage.files.get_storage_backend",
|
||||
return_value=self.backend,
|
||||
):
|
||||
first = _get_or_create_blob(
|
||||
self.session,
|
||||
tenant_id=TENANT_ID,
|
||||
data=b"first archive member",
|
||||
filename="first.txt",
|
||||
content_type="text/plain",
|
||||
actor_id=USER_ID,
|
||||
)
|
||||
second = _get_or_create_blob(
|
||||
self.session,
|
||||
tenant_id=TENANT_ID,
|
||||
data=b"second archive member",
|
||||
filename="second.txt",
|
||||
content_type="text/plain",
|
||||
actor_id=USER_ID,
|
||||
)
|
||||
self.session.commit()
|
||||
|
||||
self.assertTrue(self.backend.exists(first.storage_key))
|
||||
self.assertTrue(self.backend.exists(second.storage_key))
|
||||
with self.Session() as evidence_session:
|
||||
operations = evidence_session.query(RecoveryOperation).all()
|
||||
self.assertEqual(2, len(operations))
|
||||
self.assertEqual(
|
||||
{RecoveryStatus.SUCCEEDED.value},
|
||||
{operation.status for operation in operations},
|
||||
)
|
||||
self.assertEqual(
|
||||
{"sqlite_caller_transaction"},
|
||||
{
|
||||
operation.metadata_["durability_mode"]
|
||||
for operation in operations
|
||||
},
|
||||
)
|
||||
|
||||
def test_post_write_tamper_is_quarantined_and_recovery_required(self) -> None:
|
||||
with patch(
|
||||
"govoplan_files.backend.storage.files.get_storage_backend",
|
||||
return_value=self.backend,
|
||||
):
|
||||
blob = _get_or_create_blob(
|
||||
self.session,
|
||||
tenant_id=TENANT_ID,
|
||||
data=b"expected",
|
||||
filename="evidence.bin",
|
||||
content_type="application/octet-stream",
|
||||
actor_id=USER_ID,
|
||||
)
|
||||
self.backend.put_bytes(blob.storage_key, b"tampered")
|
||||
self.session.commit()
|
||||
|
||||
operation = self._only_operation()
|
||||
self.assertEqual(
|
||||
RecoveryStatus.RECOVERY_REQUIRED.value,
|
||||
operation.status,
|
||||
)
|
||||
with self.Session() as evidence_session:
|
||||
persisted = evidence_session.get(FileBlob, blob.id)
|
||||
self.assertIsNotNone(persisted)
|
||||
self.assertEqual("checksum_mismatch", persisted.integrity_status)
|
||||
self.assertIsNotNone(persisted.quarantined_at)
|
||||
|
||||
def test_missing_optional_encryption_fails_before_object_effect(self) -> None:
|
||||
with patch(
|
||||
"govoplan_files.backend.storage.files.get_storage_backend",
|
||||
return_value=self.backend,
|
||||
), patch(
|
||||
"govoplan_files.backend.storage.content_protection.encryption_content_cipher",
|
||||
return_value=None,
|
||||
), self.assertRaisesRegex(FileStorageError, "Encryption module"):
|
||||
_get_or_create_blob(
|
||||
self.session,
|
||||
tenant_id=TENANT_ID,
|
||||
data=b"protected",
|
||||
filename="protected.bin",
|
||||
content_type="application/octet-stream",
|
||||
actor_id=USER_ID,
|
||||
encryption_vault_id="vault-1",
|
||||
)
|
||||
self.session.rollback()
|
||||
|
||||
operation = self._only_operation()
|
||||
self.assertEqual(RecoveryStatus.REJECTED.value, operation.status)
|
||||
objects = self.backend.list_objects(
|
||||
prefix=f"tenants/{TENANT_ID}/files/",
|
||||
limit=10,
|
||||
)
|
||||
self.assertEqual((), objects.objects)
|
||||
|
||||
def test_orphan_cleanup_forward_completes_after_business_rollback(self) -> None:
|
||||
key = f"tenants/{TENANT_ID}/files/orphan.bin"
|
||||
self.backend.put_bytes(key, b"orphan")
|
||||
scan = FileIntegrityScan(
|
||||
id="scan-1",
|
||||
tenant_id=TENANT_ID,
|
||||
storage_backend=self.backend.name,
|
||||
storage_prefix=f"tenants/{TENANT_ID}/files/",
|
||||
status="completed",
|
||||
)
|
||||
finding = FileIntegrityFinding(
|
||||
id="finding-1",
|
||||
scan_id=scan.id,
|
||||
tenant_id=TENANT_ID,
|
||||
kind="orphan_object",
|
||||
state="open",
|
||||
storage_key=key,
|
||||
observed_size_bytes=6,
|
||||
observed_checksum_sha256=hashlib.sha256(b"orphan").hexdigest(),
|
||||
)
|
||||
self.session.add_all([scan, finding])
|
||||
self.session.commit()
|
||||
|
||||
cleanup_orphan_finding(
|
||||
self.session,
|
||||
finding,
|
||||
user_id=USER_ID,
|
||||
dry_run=False,
|
||||
backend=self.backend,
|
||||
)
|
||||
self.assertFalse(self.backend.exists(key))
|
||||
self.session.rollback()
|
||||
|
||||
operation = self._only_operation()
|
||||
self.assertEqual(RecoveryStatus.SUCCEEDED.value, operation.status)
|
||||
with self.Session() as evidence_session:
|
||||
persisted = evidence_session.get(FileIntegrityFinding, finding.id)
|
||||
self.assertIsNotNone(persisted)
|
||||
self.assertEqual("deleted", persisted.state)
|
||||
|
||||
def test_missing_runtime_identity_blocks_before_object_write(self) -> None:
|
||||
bind_process_runtime_identity(None)
|
||||
with patch(
|
||||
"govoplan_files.backend.storage.files.get_storage_backend",
|
||||
return_value=self.backend,
|
||||
), self.assertRaisesRegex(FileStorageError, "recovery ledger"):
|
||||
_get_or_create_blob(
|
||||
self.session,
|
||||
tenant_id=TENANT_ID,
|
||||
data=b"blocked",
|
||||
filename="blocked.bin",
|
||||
content_type="application/octet-stream",
|
||||
actor_id=USER_ID,
|
||||
)
|
||||
self.session.rollback()
|
||||
objects = self.backend.list_objects(
|
||||
prefix=f"tenants/{TENANT_ID}/files/",
|
||||
limit=10,
|
||||
)
|
||||
self.assertEqual((), objects.objects)
|
||||
|
||||
def test_blob_fence_blocks_a_competing_runtime_before_effect(self) -> None:
|
||||
checksum = hashlib.sha256(b"fenced").hexdigest()
|
||||
begin_blob_write_recovery(
|
||||
self.session,
|
||||
backend=self.backend,
|
||||
tenant_id=TENANT_ID,
|
||||
blob_id="blob-fenced",
|
||||
storage_key=f"tenants/{TENANT_ID}/files/fenced.blob",
|
||||
semantic_checksum_sha256=checksum,
|
||||
semantic_size_bytes=6,
|
||||
protection_discriminator="plaintext",
|
||||
created_new=True,
|
||||
)
|
||||
with self.Session() as competing_session, self.assertRaisesRegex(
|
||||
FileStorageError,
|
||||
"already owned",
|
||||
):
|
||||
begin_blob_write_recovery(
|
||||
competing_session,
|
||||
backend=self.backend,
|
||||
tenant_id=TENANT_ID,
|
||||
blob_id="blob-fenced",
|
||||
storage_key=f"tenants/{TENANT_ID}/files/fenced.blob",
|
||||
semantic_checksum_sha256=checksum,
|
||||
semantic_size_bytes=6,
|
||||
protection_discriminator="plaintext",
|
||||
created_new=True,
|
||||
)
|
||||
self.session.rollback()
|
||||
self.assertEqual(RecoveryStatus.REJECTED.value, self._only_operation().status)
|
||||
|
||||
def test_purge_is_idempotent_and_gc_deletes_only_after_reference_check(self) -> None:
|
||||
key = f"tenants/{TENANT_ID}/files/purge-me.blob"
|
||||
self.backend.put_bytes(key, b"purge-me")
|
||||
blob = FileBlob(
|
||||
id="blob-purge",
|
||||
tenant_id=TENANT_ID,
|
||||
storage_backend=self.backend.name,
|
||||
storage_key=key,
|
||||
checksum_sha256=hashlib.sha256(b"purge-me").hexdigest(),
|
||||
size_bytes=8,
|
||||
ref_count=1,
|
||||
)
|
||||
asset = FileAsset(
|
||||
id="asset-purge",
|
||||
tenant_id=TENANT_ID,
|
||||
owner_type="user",
|
||||
owner_user_id=USER_ID,
|
||||
current_version_id="version-purge",
|
||||
display_path="purge-me.txt",
|
||||
filename="purge-me.txt",
|
||||
deleted_at=datetime.now(UTC),
|
||||
)
|
||||
version = FileVersion(
|
||||
id="version-purge",
|
||||
tenant_id=TENANT_ID,
|
||||
file_asset_id=asset.id,
|
||||
blob_id=blob.id,
|
||||
version_number=1,
|
||||
filename_at_upload=asset.filename,
|
||||
display_path_at_upload=asset.display_path,
|
||||
size_bytes=blob.size_bytes,
|
||||
checksum_sha256=blob.checksum_sha256,
|
||||
)
|
||||
self.session.add_all([blob, asset, version])
|
||||
self.session.commit()
|
||||
preview = preview_asset_purge(
|
||||
self.session, tenant_id=TENANT_ID, file_ids=[asset.id]
|
||||
)
|
||||
|
||||
result = execute_asset_purge(
|
||||
self.session,
|
||||
tenant_id=TENANT_ID,
|
||||
file_ids=[asset.id],
|
||||
preview_sha256=preview.preview_sha256,
|
||||
idempotency_key="purge-request-1",
|
||||
approval_reference="approval-1",
|
||||
)
|
||||
|
||||
self.assertEqual(1, result.purged_files)
|
||||
self.assertEqual(1, result.released_blobs)
|
||||
self.assertIsNone(self.session.get(FileAsset, asset.id))
|
||||
retained_blob = self.session.get(FileBlob, blob.id)
|
||||
self.assertIsNotNone(retained_blob)
|
||||
self.assertEqual(0, retained_blob.ref_count)
|
||||
self.assertTrue(self.backend.exists(key))
|
||||
|
||||
replay = execute_asset_purge(
|
||||
self.session,
|
||||
tenant_id=TENANT_ID,
|
||||
file_ids=[asset.id],
|
||||
preview_sha256=preview.preview_sha256,
|
||||
idempotency_key="purge-request-1",
|
||||
approval_reference="approval-1",
|
||||
)
|
||||
self.assertTrue(replay.replayed)
|
||||
|
||||
with patch(
|
||||
"govoplan_files.backend.storage.lifecycle.get_storage_backend",
|
||||
return_value=self.backend,
|
||||
):
|
||||
gc_result = garbage_collect_unreferenced_blobs(
|
||||
self.session,
|
||||
tenant_id=TENANT_ID,
|
||||
limit=10,
|
||||
approval_reference="approval-gc-1",
|
||||
)
|
||||
self.assertEqual(1, gc_result.deleted_blobs)
|
||||
self.assertFalse(self.backend.exists(key))
|
||||
self.assertIsNone(self.session.get(FileBlob, blob.id))
|
||||
|
||||
with self.Session() as evidence_session:
|
||||
operations = evidence_session.query(RecoveryOperation).all()
|
||||
self.assertEqual(2, len(operations))
|
||||
self.assertEqual(
|
||||
{RecoveryStatus.SUCCEEDED.value},
|
||||
{operation.status for operation in operations},
|
||||
)
|
||||
|
||||
def test_s3_write_is_conditional_verified_and_idempotent(self) -> None:
|
||||
client = _WriteS3Client()
|
||||
profile = ConnectorProfile(
|
||||
id="s3-write",
|
||||
label="S3 write",
|
||||
provider="s3",
|
||||
base_path="root",
|
||||
capabilities=("browse", "write"),
|
||||
metadata={"bucket": "files"},
|
||||
)
|
||||
with patch(
|
||||
"govoplan_files.backend.storage.connector_writes._s3_client",
|
||||
return_value=client,
|
||||
):
|
||||
result = write_connector_file(
|
||||
profile,
|
||||
tenant_id=TENANT_ID,
|
||||
library_id=None,
|
||||
remote_path="out/report.txt",
|
||||
data=b"report",
|
||||
content_type="text/plain",
|
||||
expected_revision=None,
|
||||
idempotency_key="connector-write-1",
|
||||
)
|
||||
replay = write_connector_file(
|
||||
profile,
|
||||
tenant_id=TENANT_ID,
|
||||
library_id=None,
|
||||
remote_path="out/report.txt",
|
||||
data=b"report",
|
||||
content_type="text/plain",
|
||||
expected_revision=None,
|
||||
idempotency_key="connector-write-1",
|
||||
)
|
||||
|
||||
self.assertEqual(RecoveryStatus.SUCCEEDED.value, result.status)
|
||||
self.assertTrue(replay.replayed)
|
||||
self.assertEqual("*", client.put_request["IfNoneMatch"])
|
||||
self.assertEqual("root/out/report.txt", client.put_request["Key"])
|
||||
self.assertNotIn(b"report", repr(self._only_operation().metadata_).encode())
|
||||
self.assertTrue(client.closed)
|
||||
|
||||
def test_s3_write_tamper_is_visible_and_blocks_another_writer(self) -> None:
|
||||
client = _WriteS3Client(tamper_metadata=True)
|
||||
profile = ConnectorProfile(
|
||||
id="s3-write",
|
||||
label="S3 write",
|
||||
provider="s3",
|
||||
capabilities=("write",),
|
||||
metadata={"bucket": "files"},
|
||||
)
|
||||
with patch(
|
||||
"govoplan_files.backend.storage.connector_writes._s3_client",
|
||||
return_value=client,
|
||||
):
|
||||
result = write_connector_file(
|
||||
profile,
|
||||
tenant_id=TENANT_ID,
|
||||
library_id=None,
|
||||
remote_path="report.txt",
|
||||
data=b"report",
|
||||
content_type="text/plain",
|
||||
expected_revision=None,
|
||||
idempotency_key="connector-write-tampered",
|
||||
)
|
||||
with self.assertRaisesRegex(FileStorageError, "unresolved recovery"):
|
||||
write_connector_file(
|
||||
profile,
|
||||
tenant_id=TENANT_ID,
|
||||
library_id=None,
|
||||
remote_path="report.txt",
|
||||
data=b"replacement",
|
||||
content_type="text/plain",
|
||||
expected_revision="version-written",
|
||||
idempotency_key="connector-write-after-tamper",
|
||||
)
|
||||
|
||||
self.assertEqual(RecoveryStatus.RECOVERY_REQUIRED.value, result.status)
|
||||
self.assertEqual(
|
||||
RecoveryStatus.RECOVERY_REQUIRED.value, self._only_operation().status
|
||||
)
|
||||
|
||||
def test_s3_write_probe_failure_is_outcome_unknown_not_confirmed_absent(self) -> None:
|
||||
client = _WriteS3Client(fail_post_write_probe=True)
|
||||
profile = ConnectorProfile(
|
||||
id="s3-write",
|
||||
label="S3 write",
|
||||
provider="s3",
|
||||
capabilities=("write",),
|
||||
metadata={"bucket": "files"},
|
||||
)
|
||||
with patch(
|
||||
"govoplan_files.backend.storage.connector_writes._s3_client",
|
||||
return_value=client,
|
||||
):
|
||||
result = write_connector_file(
|
||||
profile,
|
||||
tenant_id=TENANT_ID,
|
||||
library_id=None,
|
||||
remote_path="report.txt",
|
||||
data=b"report",
|
||||
content_type="text/plain",
|
||||
expected_revision=None,
|
||||
idempotency_key="connector-write-probe-failed",
|
||||
)
|
||||
|
||||
self.assertEqual(RecoveryStatus.OUTCOME_UNKNOWN.value, result.status)
|
||||
operation = self._only_operation()
|
||||
self.assertEqual(RecoveryStatus.OUTCOME_UNKNOWN.value, operation.status)
|
||||
with self.Session() as session:
|
||||
checkpoint = (
|
||||
session.query(RecoveryCheckpoint)
|
||||
.filter(RecoveryCheckpoint.operation_id == operation.id)
|
||||
.order_by(RecoveryCheckpoint.sequence.desc())
|
||||
.first()
|
||||
)
|
||||
self.assertIsNotNone(checkpoint)
|
||||
self.assertFalse(checkpoint.evidence["observed"]["probe_verified"])
|
||||
self.assertFalse(checkpoint.evidence["observed"]["present"])
|
||||
|
||||
def _only_operation(self) -> RecoveryOperation:
|
||||
with self.Session() as session:
|
||||
operations = session.query(RecoveryOperation).all()
|
||||
self.assertEqual(1, len(operations))
|
||||
session.expunge(operations[0])
|
||||
return operations[0]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,175 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from datetime import UTC, datetime
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from govoplan_core.auth import ApiPrincipal
|
||||
from govoplan_core.core.access import PrincipalRef
|
||||
from govoplan_core.core.files import (
|
||||
ManagedTabularFileAccessError,
|
||||
ManagedTabularFileValidationError,
|
||||
)
|
||||
from govoplan_files.backend.capabilities import FilesManagedTabularFileService
|
||||
from govoplan_files.backend.storage.common import FileStorageError
|
||||
|
||||
|
||||
def principal(
|
||||
tenant_id: str = "tenant-1",
|
||||
*,
|
||||
scopes: tuple[str, ...] = (
|
||||
"files:file:read",
|
||||
"files:file:download",
|
||||
"files:file:admin",
|
||||
),
|
||||
) -> ApiPrincipal:
|
||||
return ApiPrincipal(
|
||||
principal=PrincipalRef(
|
||||
account_id="account-1",
|
||||
membership_id="membership-1",
|
||||
tenant_id=tenant_id,
|
||||
scopes=frozenset(scopes),
|
||||
),
|
||||
account=object(),
|
||||
user=SimpleNamespace(id="user-1"),
|
||||
)
|
||||
|
||||
|
||||
class FilesManagedTabularContentTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.engine = create_engine("sqlite+pysqlite:///:memory:")
|
||||
self.session = sessionmaker(bind=self.engine)()
|
||||
self.blob = SimpleNamespace(
|
||||
id="blob-1",
|
||||
tenant_id="tenant-1",
|
||||
storage_backend="local",
|
||||
storage_key="tenants/tenant-1/files/blob-1",
|
||||
checksum_sha256="a" * 64,
|
||||
size_bytes=8,
|
||||
content_type="text/csv",
|
||||
integrity_status="verified",
|
||||
)
|
||||
self.asset = SimpleNamespace(
|
||||
id="asset-1",
|
||||
tenant_id="tenant-1",
|
||||
owner_type="user",
|
||||
owner_user_id="user-1",
|
||||
current_version_id="version-1",
|
||||
display_path="Imports/source.csv",
|
||||
filename="source.csv",
|
||||
description=None,
|
||||
updated_at=datetime(2026, 8, 21, tzinfo=UTC),
|
||||
)
|
||||
self.version = SimpleNamespace(
|
||||
id="version-1",
|
||||
tenant_id="tenant-1",
|
||||
file_asset_id="asset-1",
|
||||
blob_id="blob-1",
|
||||
version_number=1,
|
||||
filename_at_upload="source.csv",
|
||||
display_path_at_upload="Imports/source.csv",
|
||||
content_type="text/csv",
|
||||
size_bytes=8,
|
||||
checksum_sha256="a" * 64,
|
||||
created_by_user_id="user-1",
|
||||
created_at=datetime(2026, 8, 21, tzinfo=UTC),
|
||||
)
|
||||
self.provider = FilesManagedTabularFileService()
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self.session.close()
|
||||
self.engine.dispose()
|
||||
|
||||
def test_lists_and_reads_only_exact_authorized_tabular_versions(self) -> None:
|
||||
with (
|
||||
patch(
|
||||
"govoplan_files.backend.capabilities.list_recent_assets_for_user",
|
||||
return_value=[self.asset],
|
||||
),
|
||||
patch(
|
||||
"govoplan_files.backend.capabilities.current_version_and_blob",
|
||||
return_value=(self.version, self.blob),
|
||||
),
|
||||
patch(
|
||||
"govoplan_files.backend.capabilities.get_asset_for_user",
|
||||
return_value=self.asset,
|
||||
),
|
||||
patch.object(
|
||||
self.session,
|
||||
"get",
|
||||
side_effect=lambda model, object_id: {
|
||||
"version-1": self.version,
|
||||
"blob-1": self.blob,
|
||||
}.get(object_id),
|
||||
),
|
||||
patch(
|
||||
"govoplan_files.backend.capabilities.read_asset_version_bytes",
|
||||
return_value=(b"id\n1\n", self.version, self.blob),
|
||||
) as read_bytes,
|
||||
patch("govoplan_files.backend.capabilities.audit_from_principal"),
|
||||
):
|
||||
listed = self.provider.list_tabular_files(self.session, principal())
|
||||
result = self.provider.read_tabular_file(
|
||||
self.session,
|
||||
principal(),
|
||||
file_asset_id="asset-1",
|
||||
file_version_id="version-1",
|
||||
max_bytes=100,
|
||||
)
|
||||
|
||||
self.assertEqual(("version-1",), tuple(item.file_version_id for item in listed))
|
||||
self.assertEqual(b"id\n1\n", result.payload)
|
||||
self.assertTrue(result.file.current_version)
|
||||
read_bytes.assert_called_once()
|
||||
|
||||
def test_tenant_scope_permissions_and_pre_read_size_limit_fail_closed(self) -> None:
|
||||
with patch(
|
||||
"govoplan_files.backend.capabilities.get_asset_for_user",
|
||||
side_effect=FileStorageError("File not found"),
|
||||
):
|
||||
self.assertIsNone(
|
||||
self.provider.get_tabular_file(
|
||||
self.session,
|
||||
principal("tenant-2"),
|
||||
file_asset_id="asset-1",
|
||||
)
|
||||
)
|
||||
with self.assertRaises(ManagedTabularFileAccessError):
|
||||
self.provider.list_tabular_files(
|
||||
self.session,
|
||||
principal(scopes=()),
|
||||
)
|
||||
with (
|
||||
patch(
|
||||
"govoplan_files.backend.capabilities.get_asset_for_user",
|
||||
return_value=self.asset,
|
||||
),
|
||||
patch.object(
|
||||
self.session,
|
||||
"get",
|
||||
side_effect=lambda model, object_id: {
|
||||
"version-1": self.version,
|
||||
"blob-1": self.blob,
|
||||
}.get(object_id),
|
||||
),
|
||||
patch(
|
||||
"govoplan_files.backend.capabilities.read_asset_version_bytes"
|
||||
) as read_bytes,
|
||||
self.assertRaises(ManagedTabularFileValidationError),
|
||||
):
|
||||
self.provider.read_tabular_file(
|
||||
self.session,
|
||||
principal(),
|
||||
file_asset_id="asset-1",
|
||||
file_version_id="version-1",
|
||||
max_bytes=4,
|
||||
)
|
||||
read_bytes.assert_not_called()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,127 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
import unittest
|
||||
|
||||
from sqlalchemy import create_engine, event
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from govoplan_access.backend.db.models import Account, Group, User
|
||||
from govoplan_core.core.change_sequence import ChangeSequenceEntry
|
||||
from govoplan_core.db.base import Base
|
||||
from govoplan_files.backend.db.models import (
|
||||
FileAsset,
|
||||
FileConnectorCredential,
|
||||
FileConnectorPolicy,
|
||||
FileConnectorProfile,
|
||||
FileConnectorSpace,
|
||||
FileFormEvidenceGrant,
|
||||
)
|
||||
from govoplan_files.backend.manifest import _tenant_summary_batch
|
||||
|
||||
|
||||
class FilesTenantSummaryBatchTests(unittest.TestCase):
|
||||
def test_batch_summary_uses_one_grouped_query_per_owned_table(self) -> None:
|
||||
engine = create_engine("sqlite+pysqlite:///:memory:")
|
||||
Base.metadata.create_all(
|
||||
engine,
|
||||
tables=[
|
||||
Account.__table__,
|
||||
User.__table__,
|
||||
Group.__table__,
|
||||
FileAsset.__table__,
|
||||
FileConnectorCredential.__table__,
|
||||
FileConnectorPolicy.__table__,
|
||||
FileConnectorProfile.__table__,
|
||||
FileConnectorSpace.__table__,
|
||||
FileFormEvidenceGrant.__table__,
|
||||
ChangeSequenceEntry.__table__,
|
||||
],
|
||||
)
|
||||
try:
|
||||
with Session(engine) as session:
|
||||
session.add_all(
|
||||
[
|
||||
FileAsset(
|
||||
id="file-1",
|
||||
tenant_id="tenant-1",
|
||||
owner_type="tenant",
|
||||
display_path="/one.txt",
|
||||
filename="one.txt",
|
||||
),
|
||||
FileConnectorCredential(
|
||||
id="credential-1",
|
||||
tenant_id="tenant-1",
|
||||
label="Credential",
|
||||
),
|
||||
FileConnectorPolicy(
|
||||
id="policy-1",
|
||||
tenant_id="tenant-1",
|
||||
scope_type="tenant",
|
||||
),
|
||||
FileConnectorProfile(
|
||||
id="profile-1",
|
||||
tenant_id="tenant-1",
|
||||
label="Profile",
|
||||
provider="webdav",
|
||||
),
|
||||
FileConnectorSpace(
|
||||
id="space-1",
|
||||
tenant_id="tenant-1",
|
||||
owner_type="tenant",
|
||||
label="Space",
|
||||
connector_profile_id="profile-1",
|
||||
provider="webdav",
|
||||
),
|
||||
FileFormEvidenceGrant(
|
||||
id="grant-1",
|
||||
tenant_id="tenant-1",
|
||||
form_instance_id="form-instance-1",
|
||||
form_definition_id="form-1",
|
||||
form_definition_revision="1",
|
||||
token_sha256="a" * 64,
|
||||
idempotency_key="grant-1",
|
||||
request_sha256="b" * 64,
|
||||
custodian_user_id="user-1",
|
||||
evidence_kind="document",
|
||||
purpose="submission attachment",
|
||||
expires_at=datetime.now(UTC),
|
||||
max_size_bytes=1024,
|
||||
),
|
||||
]
|
||||
)
|
||||
session.commit()
|
||||
query_count = 0
|
||||
|
||||
def count_query(*_args: object) -> None:
|
||||
nonlocal query_count
|
||||
query_count += 1
|
||||
|
||||
event.listen(engine, "before_cursor_execute", count_query)
|
||||
try:
|
||||
counts = _tenant_summary_batch(
|
||||
session,
|
||||
["tenant-1", "tenant-empty"],
|
||||
)
|
||||
finally:
|
||||
event.remove(engine, "before_cursor_execute", count_query)
|
||||
|
||||
self.assertEqual(6, query_count)
|
||||
self.assertEqual(
|
||||
{
|
||||
"files": 1,
|
||||
"connector_credentials": 1,
|
||||
"connector_policies": 1,
|
||||
"connector_profiles": 1,
|
||||
"connector_spaces": 1,
|
||||
"form_evidence_upload_grants": 1,
|
||||
},
|
||||
counts["tenant-1"],
|
||||
)
|
||||
self.assertEqual(0, counts["tenant-empty"]["files"])
|
||||
finally:
|
||||
engine.dispose()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user