565 lines
23 KiB
Python
565 lines
23 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import hmac
|
|
import json
|
|
from contextlib import ExitStack, contextmanager
|
|
from datetime import UTC, datetime, timedelta
|
|
from pathlib import Path
|
|
from typing import Literal
|
|
from fastapi import APIRouter, Depends, File as FastAPIFile, Form, HTTPException, UploadFile
|
|
from sqlalchemy.orm import Session
|
|
|
|
from govoplan_core.auth import ApiPrincipal, require_scope
|
|
from govoplan_core.security.secrets import (
|
|
TransientPayloadError,
|
|
open_transient_payload,
|
|
seal_transient_payload,
|
|
)
|
|
from govoplan_files.backend.schemas import (
|
|
ArchiveEntryResponse,
|
|
ArchivePreviewResponse,
|
|
ConflictResolutionRequest,
|
|
FileUploadResponse,
|
|
_conflict_resolutions,
|
|
)
|
|
from govoplan_files.backend.db.models import FileAsset
|
|
from govoplan_core.db.session import get_session
|
|
from govoplan_files.backend.runtime import settings
|
|
from govoplan_files.backend.archive_work import (
|
|
ArchiveProgress, ArchiveWorkExpired, discard_staged_upload, read_progress,
|
|
stage_upload, use_staged_upload,
|
|
)
|
|
from govoplan_files.backend.storage.paths import (
|
|
UnsafeFilePathError,
|
|
normalize_folder,
|
|
)
|
|
from govoplan_files.backend.storage.archives import (
|
|
archive_format_for_filename,
|
|
extract_archive_upload,
|
|
extract_zip_upload,
|
|
inspect_archive,
|
|
)
|
|
from govoplan_files.backend.storage.common import FileStorageError
|
|
from govoplan_files.backend.storage.connector_policy import (
|
|
ConnectorPolicyDenied,
|
|
)
|
|
from govoplan_files.backend.storage.files import (
|
|
create_file_asset,
|
|
)
|
|
|
|
|
|
from govoplan_files.backend.route_support import (
|
|
_asset_list_response,
|
|
_audit_connector_imports,
|
|
_cleanup_temp_file,
|
|
_connector_policy_error,
|
|
_enforce_connector_policy,
|
|
_http_error,
|
|
_is_admin,
|
|
_read_limited_upload,
|
|
_source_metadata_from_form,
|
|
_spool_limited_upload_to_temp,
|
|
)
|
|
|
|
router = APIRouter(prefix="/files", tags=["files"])
|
|
_ARCHIVE_PREVIEW_PURPOSE = "files.archive-preview.v1"
|
|
|
|
|
|
@contextmanager
|
|
def _archive_source(file, staged_upload_id, preview_token, principal):
|
|
if isinstance(staged_upload_id, str):
|
|
if file is not None and hasattr(file, "file"):
|
|
raise FileStorageError("Choose either the staged archive or a new upload")
|
|
if not isinstance(preview_token, str) or not preview_token:
|
|
raise FileStorageError("A valid preview token is required for a staged archive")
|
|
payload = open_transient_payload(preview_token, ttl_seconds=settings.file_archive_preview_ttl_seconds)
|
|
expected = {"purpose": _ARCHIVE_PREVIEW_PURPOSE, "tenant_id": principal.tenant_id,
|
|
"user_id": principal.user.id, "staged_upload_id": staged_upload_id}
|
|
if any(payload.get(key) != value for key, value in expected.items()) or not isinstance(payload.get("filename"), str):
|
|
raise FileStorageError("Archive preview does not match this staged upload")
|
|
with use_staged_upload(settings, staged_upload_id, tenant_id=principal.tenant_id, user_id=principal.user.id) as path:
|
|
token_digest = payload.get("archive_sha256")
|
|
if not isinstance(token_digest, str) or not hmac.compare_digest(_archive_sha256(path), token_digest):
|
|
raise FileStorageError("Archive contents changed after preview; preview it again")
|
|
yield path, payload["filename"]
|
|
return
|
|
if file is None or not hasattr(file, "file"):
|
|
raise FileStorageError("Select an archive to upload")
|
|
filename = file.filename or "archive"
|
|
path = _spool_limited_upload_to_temp(file, max_bytes=settings.file_upload_zip_max_bytes, suffix=_archive_suffix(filename))
|
|
try:
|
|
yield path, filename
|
|
finally:
|
|
try:
|
|
_cleanup_temp_file(path)
|
|
except OSError:
|
|
pass
|
|
|
|
|
|
@router.get("/archive-progress/{operation_id}")
|
|
async def archive_operation_progress(operation_id: str, principal: ApiPrincipal = Depends(require_scope("files:file:upload"))):
|
|
# Small local receipt read stays available even while sync upload workers
|
|
# are busy. The request owns the transaction; this is not a job retry API.
|
|
try:
|
|
return read_progress(settings, operation_id, tenant_id=principal.tenant_id, user_id=principal.user.id)
|
|
except ArchiveWorkExpired as exc:
|
|
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
|
except FileStorageError as exc:
|
|
raise _http_error(exc) from exc
|
|
|
|
|
|
@router.delete("/archive-staging/{stage_id}", status_code=204)
|
|
def release_staged_archive(stage_id: str, principal: ApiPrincipal = Depends(require_scope("files:file:upload"))):
|
|
try:
|
|
discard_staged_upload(settings, stage_id, tenant_id=principal.tenant_id, user_id=principal.user.id)
|
|
except FileStorageError as exc:
|
|
raise _http_error(exc) from exc
|
|
|
|
|
|
def _archive_suffix(filename: str) -> str:
|
|
lowered = filename.casefold()
|
|
for suffix in (".tar.bz2", ".tar.gz", ".tar.xz", ".tbz2", ".tgz", ".txz", ".tar", ".zip"):
|
|
if lowered.endswith(suffix):
|
|
return suffix
|
|
return ".archive"
|
|
|
|
|
|
def _archive_sha256(path: str) -> str:
|
|
digest = hashlib.sha256()
|
|
with open(path, "rb") as source:
|
|
while chunk := source.read(1024 * 1024):
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def _archive_selected_paths(value: str) -> list[str]:
|
|
parsed = json.loads(value)
|
|
if not isinstance(parsed, list) or not parsed:
|
|
raise FileStorageError("Select at least one archive file or folder")
|
|
if len(parsed) > settings.file_archive_max_entries:
|
|
raise FileStorageError(
|
|
"Archive selection exceeds the configured entry limit"
|
|
)
|
|
selected: list[str] = []
|
|
for item in parsed:
|
|
if not isinstance(item, str) or not item.strip():
|
|
raise FileStorageError("Archive selection contains an invalid path")
|
|
if len(item) > 4096:
|
|
raise FileStorageError("Archive selection path is too long")
|
|
selected.append(item)
|
|
return selected
|
|
|
|
|
|
def _validate_archive_preview_token(
|
|
token: str,
|
|
*,
|
|
tenant_id: str,
|
|
user_id: str,
|
|
owner_type: str,
|
|
owner_id: str,
|
|
path: str,
|
|
campaign_id: str | None,
|
|
archive_format: str,
|
|
archive_sha256: str,
|
|
) -> None:
|
|
payload = open_transient_payload(
|
|
token,
|
|
ttl_seconds=settings.file_archive_preview_ttl_seconds,
|
|
)
|
|
expected = {
|
|
"purpose": _ARCHIVE_PREVIEW_PURPOSE,
|
|
"tenant_id": tenant_id,
|
|
"user_id": user_id,
|
|
"owner_type": owner_type,
|
|
"owner_id": owner_id,
|
|
"path": path,
|
|
"campaign_id": campaign_id or "",
|
|
"archive_format": archive_format,
|
|
}
|
|
if any(payload.get(key) != value for key, value in expected.items()):
|
|
raise FileStorageError(
|
|
"Archive preview does not match this upload destination"
|
|
)
|
|
token_digest = payload.get("archive_sha256")
|
|
if not isinstance(token_digest, str) or not hmac.compare_digest(
|
|
token_digest, archive_sha256
|
|
):
|
|
raise FileStorageError(
|
|
"Archive contents changed after preview; preview it again"
|
|
)
|
|
|
|
|
|
@router.post("/archive-preview", response_model=ArchivePreviewResponse)
|
|
def preview_archive_upload(
|
|
file: UploadFile | None = FastAPIFile(default=None),
|
|
owner_type: Literal["user", "group"] = Form(default="user"),
|
|
owner_id: str | None = Form(default=None),
|
|
path: str = Form(default=""),
|
|
campaign_id: str | None = Form(default=None),
|
|
password: str | None = Form(default=None),
|
|
retain_upload: bool = Form(default=False),
|
|
staged_upload_id: str | None = Form(default=None),
|
|
preview_token: str | None = Form(default=None),
|
|
principal: ApiPrincipal = Depends(require_scope("files:file:upload")),
|
|
):
|
|
target_owner = owner_id or principal.user.id
|
|
sources = ExitStack()
|
|
try:
|
|
archive_path, filename = sources.enter_context(_archive_source(file, staged_upload_id, preview_token, principal))
|
|
archive_format = archive_format_for_filename(filename)
|
|
inspection = inspect_archive(
|
|
archive_path,
|
|
filename=filename,
|
|
password=password,
|
|
max_entries=settings.file_archive_max_entries,
|
|
max_expanded_bytes=settings.file_archive_max_expanded_bytes,
|
|
max_expansion_ratio=settings.file_archive_max_expansion_ratio,
|
|
)
|
|
digest = _archive_sha256(archive_path)
|
|
normalized_path = normalize_folder(path)
|
|
stage_id = staged_upload_id if isinstance(staged_upload_id, str) else None
|
|
if retain_upload is True and stage_id is None:
|
|
stage_id = stage_upload(settings, archive_path, tenant_id=principal.tenant_id, user_id=principal.user.id)
|
|
preview_token = seal_transient_payload(
|
|
{
|
|
"purpose": _ARCHIVE_PREVIEW_PURPOSE,
|
|
"tenant_id": principal.tenant_id,
|
|
"user_id": principal.user.id,
|
|
"owner_type": owner_type,
|
|
"owner_id": target_owner,
|
|
"path": normalized_path,
|
|
"campaign_id": campaign_id or "",
|
|
"archive_format": archive_format,
|
|
"archive_sha256": digest,
|
|
**({"staged_upload_id": stage_id, "filename": filename} if stage_id else {}),
|
|
}
|
|
)
|
|
expires_at = datetime.now(UTC) + timedelta(
|
|
seconds=settings.file_archive_preview_ttl_seconds
|
|
)
|
|
if isinstance(staged_upload_id, str):
|
|
# Repreviewing (for example with a password) does not renew the
|
|
# underlying private stage's lifetime.
|
|
expires_at = min(expires_at, datetime.fromtimestamp(
|
|
Path(archive_path).stat().st_mtime, UTC
|
|
) + timedelta(seconds=settings.file_archive_preview_ttl_seconds))
|
|
return ArchivePreviewResponse(
|
|
preview_token=preview_token,
|
|
staged_upload_id=stage_id,
|
|
archive_format=inspection.archive_format,
|
|
entries=[
|
|
ArchiveEntryResponse(
|
|
path=entry.path,
|
|
kind=entry.kind,
|
|
size_bytes=entry.size_bytes,
|
|
compressed_size_bytes=entry.compressed_size_bytes,
|
|
encrypted=entry.encrypted,
|
|
)
|
|
for entry in inspection.entries
|
|
],
|
|
file_count=inspection.file_count,
|
|
directory_count=inspection.directory_count,
|
|
expanded_size_bytes=inspection.expanded_size_bytes,
|
|
compressed_size_bytes=inspection.compressed_size_bytes,
|
|
requires_password=inspection.requires_password,
|
|
password_verified=inspection.password_verified,
|
|
expires_at=expires_at.isoformat(),
|
|
)
|
|
except ArchiveWorkExpired as exc:
|
|
raise HTTPException(status_code=410, detail=str(exc)) from exc
|
|
except (FileStorageError, TransientPayloadError, UnsafeFilePathError, ValueError) as exc:
|
|
raise _http_error(exc) from exc
|
|
finally:
|
|
sources.close()
|
|
|
|
|
|
@router.post("/archive-confirm", response_model=FileUploadResponse)
|
|
def confirm_archive_upload(
|
|
file: UploadFile | None = FastAPIFile(default=None),
|
|
preview_token: str = Form(...),
|
|
selected_paths_json: str = Form(...),
|
|
owner_type: Literal["user", "group"] = Form(default="user"),
|
|
owner_id: str | None = Form(default=None),
|
|
path: str = Form(default=""),
|
|
campaign_id: str | None = Form(default=None),
|
|
password: str | None = Form(default=None),
|
|
conflict_strategy: Literal["reject", "overwrite", "rename"] = Form(
|
|
default="reject"
|
|
),
|
|
conflict_resolutions_json: str | None = Form(default=None),
|
|
source_provenance_json: str | None = Form(default=None),
|
|
source_revision: str | None = Form(default=None),
|
|
connector_policy_json: str | None = Form(default=None),
|
|
encryption_vault_id: str | None = Form(default=None),
|
|
staged_upload_id: str | None = Form(default=None),
|
|
operation_id: str | None = Form(default=None),
|
|
session: Session = Depends(get_session),
|
|
principal: ApiPrincipal = Depends(require_scope("files:file:upload")),
|
|
):
|
|
target_owner = owner_id or principal.user.id
|
|
sources = ExitStack()
|
|
progress = None
|
|
committed = False
|
|
try:
|
|
archive_path, filename = sources.enter_context(_archive_source(file, staged_upload_id, preview_token, principal))
|
|
raw_resolutions = (
|
|
json.loads(conflict_resolutions_json)
|
|
if conflict_resolutions_json
|
|
else []
|
|
)
|
|
upload_resolutions = _conflict_resolutions(
|
|
[ConflictResolutionRequest(**item) for item in raw_resolutions]
|
|
)
|
|
selected_paths = _archive_selected_paths(selected_paths_json)
|
|
_enforce_connector_policy(
|
|
source_provenance_json,
|
|
connector_policy_json,
|
|
operation="import",
|
|
)
|
|
metadata = _source_metadata_from_form(
|
|
source_provenance_json, source_revision
|
|
)
|
|
archive_format = archive_format_for_filename(filename)
|
|
normalized_path = normalize_folder(path)
|
|
digest = _archive_sha256(archive_path)
|
|
_validate_archive_preview_token(
|
|
preview_token,
|
|
tenant_id=principal.tenant_id,
|
|
user_id=principal.user.id,
|
|
owner_type=owner_type,
|
|
owner_id=target_owner,
|
|
path=normalized_path,
|
|
campaign_id=campaign_id,
|
|
archive_format=archive_format,
|
|
archive_sha256=digest,
|
|
)
|
|
progress = ArchiveProgress(settings, operation_id, tenant_id=principal.tenant_id, user_id=principal.user.id)
|
|
extracted = extract_archive_upload(
|
|
session,
|
|
tenant_id=principal.tenant_id,
|
|
owner_type=owner_type,
|
|
owner_id=target_owner,
|
|
user_id=principal.user.id,
|
|
archive_data=archive_path,
|
|
filename=filename,
|
|
folder=normalized_path,
|
|
campaign_id=campaign_id,
|
|
selected_paths=selected_paths,
|
|
password=password,
|
|
conflict_strategy=conflict_strategy,
|
|
conflict_resolutions=upload_resolutions,
|
|
metadata=metadata,
|
|
is_admin=_is_admin(principal),
|
|
encryption_vault_id=encryption_vault_id,
|
|
max_entries=settings.file_archive_max_entries,
|
|
max_file_bytes=settings.file_upload_max_bytes,
|
|
max_expanded_bytes=settings.file_archive_max_expanded_bytes,
|
|
max_expansion_ratio=settings.file_archive_max_expansion_ratio,
|
|
progress=progress,
|
|
)
|
|
uploaded_assets = [item.asset for item in extracted]
|
|
_audit_connector_imports(session, principal, uploaded_assets)
|
|
progress("finalizing", len(extracted), len(extracted), progress.value["completed_bytes"], progress.value["total_bytes"])
|
|
session.flush()
|
|
response = FileUploadResponse(files=_asset_list_response(session, uploaded_assets, include_shares=True))
|
|
session.commit()
|
|
committed = True
|
|
progress.finish(True)
|
|
return response
|
|
except ArchiveWorkExpired as exc:
|
|
session.rollback()
|
|
raise HTTPException(status_code=410, detail=str(exc)) from exc
|
|
except ConnectorPolicyDenied as exc:
|
|
session.rollback()
|
|
raise _connector_policy_error(exc) from exc
|
|
except (
|
|
FileStorageError,
|
|
TransientPayloadError,
|
|
UnsafeFilePathError,
|
|
ValueError,
|
|
json.JSONDecodeError,
|
|
) as exc:
|
|
session.rollback()
|
|
raise _http_error(exc) from exc
|
|
except Exception:
|
|
session.rollback()
|
|
raise
|
|
finally:
|
|
if progress and not committed:
|
|
progress.finish(False)
|
|
try:
|
|
sources.close()
|
|
except OSError:
|
|
pass
|
|
if committed and isinstance(staged_upload_id, str):
|
|
try:
|
|
discard_staged_upload(settings, staged_upload_id, tenant_id=principal.tenant_id, user_id=principal.user.id)
|
|
except (OSError, FileStorageError):
|
|
pass
|
|
|
|
|
|
@router.post("/upload", response_model=FileUploadResponse)
|
|
def upload_files(
|
|
files: list[UploadFile] = FastAPIFile(...),
|
|
owner_type: Literal["user", "group"] = Form(default="user"),
|
|
owner_id: str | None = Form(default=None),
|
|
path: str = Form(default=""),
|
|
campaign_id: str | None = Form(default=None),
|
|
unpack_zip: bool = Form(default=False),
|
|
conflict_strategy: Literal["reject", "overwrite", "rename"] = Form(
|
|
default="reject"
|
|
),
|
|
conflict_resolutions_json: str | None = Form(default=None),
|
|
source_provenance_json: str | None = Form(default=None),
|
|
source_revision: str | None = Form(default=None),
|
|
connector_policy_json: str | None = Form(default=None),
|
|
encryption_vault_id: str | None = Form(default=None),
|
|
session: Session = Depends(get_session),
|
|
principal: ApiPrincipal = Depends(require_scope("files:file:upload")),
|
|
):
|
|
target_owner = owner_id or principal.user.id
|
|
uploaded_assets: list[FileAsset] = []
|
|
try:
|
|
raw_resolutions = (
|
|
json.loads(conflict_resolutions_json) if conflict_resolutions_json else []
|
|
)
|
|
upload_resolutions = _conflict_resolutions(
|
|
[ConflictResolutionRequest(**item) for item in raw_resolutions]
|
|
)
|
|
_enforce_connector_policy(
|
|
source_provenance_json, connector_policy_json, operation="import"
|
|
)
|
|
metadata = _source_metadata_from_form(source_provenance_json, source_revision)
|
|
for upload in files:
|
|
filename = upload.filename or "file"
|
|
content_type = upload.content_type or None
|
|
upload_limit = (
|
|
settings.file_upload_zip_max_bytes
|
|
if unpack_zip and filename.lower().endswith(".zip")
|
|
else settings.file_upload_max_bytes
|
|
)
|
|
if unpack_zip and filename.lower().endswith(".zip"):
|
|
zip_path = _spool_limited_upload_to_temp(
|
|
upload, max_bytes=upload_limit, suffix=".zip"
|
|
)
|
|
try:
|
|
extracted = extract_zip_upload(
|
|
session,
|
|
tenant_id=principal.tenant_id,
|
|
owner_type=owner_type,
|
|
owner_id=target_owner,
|
|
user_id=principal.user.id,
|
|
zip_data=zip_path,
|
|
folder=path,
|
|
campaign_id=campaign_id,
|
|
conflict_strategy=conflict_strategy,
|
|
conflict_resolutions=upload_resolutions,
|
|
metadata=metadata,
|
|
is_admin=_is_admin(principal),
|
|
encryption_vault_id=encryption_vault_id,
|
|
max_file_bytes=settings.file_upload_max_bytes,
|
|
max_total_bytes=settings.file_upload_zip_max_bytes,
|
|
)
|
|
finally:
|
|
_cleanup_temp_file(zip_path)
|
|
uploaded_assets.extend(item.asset for item in extracted)
|
|
continue
|
|
data = _read_limited_upload(upload, max_bytes=upload_limit)
|
|
stored = create_file_asset(
|
|
session,
|
|
tenant_id=principal.tenant_id,
|
|
owner_type=owner_type,
|
|
owner_id=target_owner,
|
|
user_id=principal.user.id,
|
|
filename=filename,
|
|
data=data,
|
|
folder=path,
|
|
content_type=content_type,
|
|
campaign_id=campaign_id,
|
|
conflict_strategy=conflict_strategy,
|
|
conflict_resolutions=upload_resolutions,
|
|
metadata=metadata,
|
|
is_admin=_is_admin(principal),
|
|
encryption_vault_id=encryption_vault_id,
|
|
)
|
|
uploaded_assets.append(stored.asset)
|
|
_audit_connector_imports(session, principal, uploaded_assets)
|
|
session.flush()
|
|
response = FileUploadResponse(files=_asset_list_response(session, uploaded_assets, include_shares=True))
|
|
session.commit()
|
|
except ConnectorPolicyDenied as exc:
|
|
session.rollback()
|
|
raise _connector_policy_error(exc) from exc
|
|
except (FileStorageError, UnsafeFilePathError, ValueError) as exc:
|
|
session.rollback()
|
|
raise _http_error(exc) from exc
|
|
return response
|
|
|
|
|
|
@router.post("/upload-zip", response_model=FileUploadResponse)
|
|
def upload_zip(
|
|
file: UploadFile = FastAPIFile(...),
|
|
owner_type: Literal["user", "group"] = Form(default="user"),
|
|
owner_id: str | None = Form(default=None),
|
|
path: str = Form(default=""),
|
|
campaign_id: str | None = Form(default=None),
|
|
conflict_strategy: Literal["reject", "overwrite", "rename"] = Form(
|
|
default="reject"
|
|
),
|
|
conflict_resolutions_json: str | None = Form(default=None),
|
|
source_provenance_json: str | None = Form(default=None),
|
|
source_revision: str | None = Form(default=None),
|
|
connector_policy_json: str | None = Form(default=None),
|
|
encryption_vault_id: str | None = Form(default=None),
|
|
session: Session = Depends(get_session),
|
|
principal: ApiPrincipal = Depends(require_scope("files:file:upload")),
|
|
):
|
|
target_owner = owner_id or principal.user.id
|
|
zip_path: str | None = None
|
|
try:
|
|
raw_resolutions = (
|
|
json.loads(conflict_resolutions_json) if conflict_resolutions_json else []
|
|
)
|
|
upload_resolutions = _conflict_resolutions(
|
|
[ConflictResolutionRequest(**item) for item in raw_resolutions]
|
|
)
|
|
_enforce_connector_policy(
|
|
source_provenance_json, connector_policy_json, operation="import"
|
|
)
|
|
metadata = _source_metadata_from_form(source_provenance_json, source_revision)
|
|
zip_path = _spool_limited_upload_to_temp(
|
|
file, max_bytes=settings.file_upload_zip_max_bytes, suffix=".zip"
|
|
)
|
|
extracted = extract_zip_upload(
|
|
session,
|
|
tenant_id=principal.tenant_id,
|
|
owner_type=owner_type,
|
|
owner_id=target_owner,
|
|
user_id=principal.user.id,
|
|
zip_data=zip_path,
|
|
folder=path,
|
|
campaign_id=campaign_id,
|
|
conflict_strategy=conflict_strategy,
|
|
conflict_resolutions=upload_resolutions,
|
|
metadata=metadata,
|
|
is_admin=_is_admin(principal),
|
|
encryption_vault_id=encryption_vault_id,
|
|
max_file_bytes=settings.file_upload_max_bytes,
|
|
max_total_bytes=settings.file_upload_zip_max_bytes,
|
|
)
|
|
_audit_connector_imports(session, principal, [item.asset for item in extracted])
|
|
session.flush()
|
|
response = FileUploadResponse(files=_asset_list_response(session, [item.asset for item in extracted], include_shares=True))
|
|
session.commit()
|
|
except ConnectorPolicyDenied as exc:
|
|
session.rollback()
|
|
raise _connector_policy_error(exc) from exc
|
|
except (FileStorageError, UnsafeFilePathError, ValueError) as exc:
|
|
session.rollback()
|
|
raise _http_error(exc) from exc
|
|
finally:
|
|
if zip_path:
|
|
_cleanup_temp_file(zip_path)
|
|
return response
|