feat: support optional encrypted file content

This commit is contained in:
2026-08-02 03:40:54 +02:00
parent 233ce40983
commit 359c4e9570
11 changed files with 478 additions and 30 deletions
+20 -10
View File
@@ -29,12 +29,18 @@ a document collaboration engine, or a records-management system.
A managed file is a tenant-scoped logical asset with exactly one user or group
owner, a normalized path, and a current version. The current version points to a
blob record containing the storage location, SHA-256 checksum, byte size, and
content type. Current service paths append a version when connector sync finds
content type. A protected blob also records its Encryption envelope, protection
discriminator, stored-ciphertext checksum, and stored-ciphertext size. Current
service paths append a version when connector sync finds
changed content; they do not mutate the previous version record.
Content with the same tenant, SHA-256 checksum, and size can reuse one blob.
This is storage deduplication and integrity evidence, not proof of authorship or
source authenticity.
Unprotected content with the same tenant, plaintext SHA-256 checksum, size, and
protection discriminator can reuse one blob. Protected content is deduplicated
only inside the same vault/profile discriminator; ciphertext is never silently
reused across protection boundaries. Plaintext checksums remain semantic
version evidence, while download and integrity scans verify stored ciphertext
before asking Encryption to open it. Neither digest proves authorship or source
authenticity.
The main domain objects are:
@@ -42,7 +48,7 @@ The main domain objects are:
| --- | --- | --- |
| File asset | The user-facing file identity, owner, logical path, description, metadata, and current version | Created, organized, shared, and soft-deleted |
| File version | A numbered snapshot of one asset and its blob | Appended by connector sync when bytes change; retained |
| File blob | Stored bytes plus checksum, size, backend, key, reference count, and optional retention timestamp | Reused within a tenant; no automated garbage collection |
| File blob | Stored plaintext semantics plus stored-byte integrity, backend key, optional Encryption envelope, reference count, and retention timestamp | Reused only within a tenant and matching protection boundary; no automated garbage collection |
| Folder | An explicit logical path in a user or group space | Created, moved/renamed through organize operations, and soft-deleted |
| Share | A grant from an asset to a user, group, tenant, or campaign with `read`, `write`, or `manage` permission | Created or updated; no public revocation endpoint yet |
| Connector profile | A governed external endpoint, scope, optional credential link, policy, and descriptive capabilities | Created, updated, disabled, or credential-scrubbed on deletion |
@@ -634,13 +640,17 @@ Files baseline indiscriminately.
secrets and deployment references.
- API metadata is recursively checked for secret-like values.
- Downloads use sanitized attachment filenames.
- SHA-256 and byte size are recorded for every blob/version.
- Plaintext semantic and stored-byte SHA-256/size evidence are recorded for
every protected blob; they are identical for unprotected blobs.
- Upload and archive-confirm APIs can select an Encryption vault. Protected
writes and reads fail closed if the optional Encryption capability is absent.
The module does **not** currently provide malware scanning, content disarm and
reconstruction, a file-type allowlist, per-user quota, at-rest encryption for
local blob bytes, or a dedicated preview sandbox. Deployments that require
these controls must supply them outside Files until explicit module contracts
exist.
reconstruction, a file-type allowlist, per-user quota, automatic encryption
policy assignment, client E2EE, or a dedicated preview sandbox. The optional
server-envelope profile protects selected managed blob bytes at rest but remains
server-decryptable. Deployments that require the other controls must supply
them outside Files until explicit module contracts exist.
### Provenance
+13 -1
View File
@@ -16,7 +16,15 @@ def new_uuid() -> str:
class FileBlob(Base, TimestampMixin):
__tablename__ = "file_blobs"
__table_args__ = (UniqueConstraint("tenant_id", "checksum_sha256", "size_bytes", name="uq_file_blobs_tenant_checksum_size"),)
__table_args__ = (
UniqueConstraint(
"tenant_id",
"checksum_sha256",
"size_bytes",
"protection_discriminator",
name="uq_file_blobs_tenant_checksum_size_protection",
),
)
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
@@ -25,6 +33,10 @@ class FileBlob(Base, TimestampMixin):
storage_key: Mapped[str] = mapped_column(String(1000), nullable=False)
checksum_sha256: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
size_bytes: Mapped[int] = mapped_column(Integer, nullable=False)
protection_discriminator: Mapped[str] = mapped_column(String(320), default="plaintext", nullable=False, index=True)
encryption_envelope_id: Mapped[str | None] = mapped_column(String(255), nullable=True, index=True)
storage_checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
storage_size_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True)
content_type: Mapped[str | None] = mapped_column(String(255))
ref_count: Mapped[int] = mapped_column(Integer, default=1, nullable=False)
retained_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+12 -5
View File
@@ -6,6 +6,7 @@ from pathlib import Path
from sqlalchemy import inspect
from govoplan_core.core.access import CAPABILITY_AUTH_PERMISSION_EVALUATOR, CAPABILITY_AUTH_PRINCIPAL_RESOLVER
from govoplan_core.core.encryption import CAPABILITY_ENCRYPTION_CONTENT_CIPHER
from govoplan_core.core.files import CAPABILITY_FILES_ACCESS
from govoplan_core.core.module_guards import drop_table_retirement_provider, persistent_table_uninstall_guard
from govoplan_core.core.modules import (
@@ -275,7 +276,7 @@ manifest = ModuleManifest(
name="Files",
version="0.1.9",
required_capabilities=(CAPABILITY_AUTH_PRINCIPAL_RESOLVER, CAPABILITY_AUTH_PERMISSION_EVALUATOR),
optional_dependencies=("campaigns",),
optional_dependencies=("campaigns", "encryption"),
provides_interfaces=(
ModuleInterfaceProvider(name="files.access", version="0.1.6"),
ModuleInterfaceProvider(name="files.campaign_attachments", version="0.1.6"),
@@ -287,6 +288,12 @@ manifest = ModuleManifest(
version_max_exclusive="0.2.0",
optional=True,
),
ModuleInterfaceRequirement(
name=CAPABILITY_ENCRYPTION_CONTENT_CIPHER,
version_min="1.0.0",
version_max_exclusive="2.0.0",
optional=True,
),
),
permissions=PERMISSIONS,
route_factory=_files_router,
@@ -570,9 +577,9 @@ manifest = ModuleManifest(
DocumentationTopic(
id="files.reference.integrity-recovery-and-fail-closed-transports",
title="Operate Files integrity, recovery, and connector transport safety",
summary="Back up database evidence, blob bytes, and the encryption key as one recovery unit, and keep unsupported SDK transports fail-closed.",
summary="Back up database evidence, blob ciphertext, and Encryption custody as one recovery unit, and keep unsupported SDK transports fail-closed.",
body=(
"Local durable storage is the operational baseline. Recover Files from a coordinated database/blob snapshot with the original master key, then run the bounded resumable integrity scan and verify representative access paths. Missing or mismatched blobs are quarantined; orphan objects are reported before dry-run-first, explicitly authorized cleanup. "
"Local durable storage is the operational baseline. Recover Files from a coordinated database/blob snapshot with the matching Encryption tables and original deployment master key, then run the bounded resumable integrity scan and verify representative protected and unprotected access paths. Protected scans verify stored ciphertext before decryption and then verify plaintext semantic evidence. Missing or mismatched blobs are quarantined; orphan objects are reported before dry-run-first, explicitly authorized cleanup. "
"Arbitrary external S3 managed storage/connectors and SMB connectors fail closed until botocore redirects/endpoint discovery and SMB initial connections/DFS referrals support connection-time DNS/IP pinning. Installer-owned Garage storage is supported only at the exact deployment service endpoint with its explicit trust marker. Destructive module retirement drops database tables but does not remove backend blob objects."
),
layer="configured",
@@ -596,7 +603,7 @@ manifest = ModuleManifest(
DocumentationLink(label="Create an integrity scan", href="/api/v1/files/integrity/scans", kind="api"),
DocumentationLink(label="Files handbook", href="govoplan-files/docs/FILES_HANDBOOK.md", kind="repository"),
),
related_modules=("audit", "ops"),
related_modules=("audit", "encryption", "ops"),
unlocks=("Recoverable managed-file evidence without weakening outbound peer validation.",),
configuration_keys=(
"FILE_STORAGE_BACKEND",
@@ -619,7 +626,7 @@ manifest = ModuleManifest(
"route": "/admin?section=system-file-connectors",
"screen": "System file connections and deployment operations",
"section": "Storage integrity, backup/recovery, and fail-closed transports",
"recovery_unit": ["Files database rows", "managed blob namespace", "MASTER_KEY_B64", "deployment-owned connector configuration"],
"recovery_unit": ["Files database rows", "Encryption envelope and wrapped-key rows", "managed blob namespace", "MASTER_KEY_B64", "deployment-owned connector configuration"],
"verification": "After restore, complete a checksum-enabled integrity scan, resolve every missing/corrupt finding, approve or retain every reported orphan, verify authorized and denied access, and test one permitted pinned HTTP connector.",
"related_topic_ids": [
"files.governed-connectors-and-provenance",
@@ -0,0 +1,79 @@
"""file content-protection metadata
Revision ID: d0e1f2a3b4c6
Revises: c9d0e1f2a3b5
Create Date: 2026-08-02 00:00:00.000000
"""
from __future__ import annotations
from alembic import op
import sqlalchemy as sa
revision = "d0e1f2a3b4c6"
down_revision = "c9d0e1f2a3b5"
branch_labels = None
depends_on = None
def upgrade() -> None:
with op.batch_alter_table("file_blobs") as batch_op:
batch_op.add_column(
sa.Column(
"protection_discriminator",
sa.String(length=320),
nullable=False,
server_default="plaintext",
)
)
batch_op.add_column(
sa.Column("encryption_envelope_id", sa.String(length=255), nullable=True)
)
batch_op.add_column(
sa.Column("storage_checksum_sha256", sa.String(length=64), nullable=True)
)
batch_op.add_column(
sa.Column("storage_size_bytes", sa.Integer(), nullable=True)
)
batch_op.drop_constraint(
"uq_file_blobs_tenant_checksum_size",
type_="unique",
)
batch_op.create_unique_constraint(
"uq_file_blobs_tenant_checksum_size_protection",
[
"tenant_id",
"checksum_sha256",
"size_bytes",
"protection_discriminator",
],
)
batch_op.create_index(
op.f("ix_file_blobs_protection_discriminator"),
["protection_discriminator"],
unique=False,
)
batch_op.create_index(
op.f("ix_file_blobs_encryption_envelope_id"),
["encryption_envelope_id"],
unique=False,
)
def downgrade() -> None:
with op.batch_alter_table("file_blobs") as batch_op:
batch_op.drop_index(op.f("ix_file_blobs_encryption_envelope_id"))
batch_op.drop_index(op.f("ix_file_blobs_protection_discriminator"))
batch_op.drop_constraint(
"uq_file_blobs_tenant_checksum_size_protection",
type_="unique",
)
batch_op.create_unique_constraint(
"uq_file_blobs_tenant_checksum_size",
["tenant_id", "checksum_sha256", "size_bytes"],
)
batch_op.drop_column("storage_size_bytes")
batch_op.drop_column("storage_checksum_sha256")
batch_op.drop_column("encryption_envelope_id")
batch_op.drop_column("protection_discriminator")
@@ -224,6 +224,7 @@ def confirm_archive_upload(
source_provenance_json: str | None = Form(default=None),
source_revision: str | None = Form(default=None),
connector_policy_json: str | None = Form(default=None),
encryption_vault_id: str | None = Form(default=None),
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(require_scope("files:file:upload")),
):
@@ -283,6 +284,7 @@ def confirm_archive_upload(
conflict_resolutions=upload_resolutions,
metadata=metadata,
is_admin=_is_admin(principal),
encryption_vault_id=encryption_vault_id,
max_entries=settings.file_archive_max_entries,
max_file_bytes=settings.file_upload_max_bytes,
max_expanded_bytes=settings.file_archive_max_expanded_bytes,
@@ -329,6 +331,7 @@ def upload_files(
source_provenance_json: str | None = Form(default=None),
source_revision: str | None = Form(default=None),
connector_policy_json: str | None = Form(default=None),
encryption_vault_id: str | None = Form(default=None),
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(require_scope("files:file:upload")),
):
@@ -371,6 +374,7 @@ def upload_files(
conflict_resolutions=upload_resolutions,
metadata=metadata,
is_admin=_is_admin(principal),
encryption_vault_id=encryption_vault_id,
max_file_bytes=settings.file_upload_max_bytes,
max_total_bytes=settings.file_upload_zip_max_bytes,
)
@@ -394,6 +398,7 @@ def upload_files(
conflict_resolutions=upload_resolutions,
metadata=metadata,
is_admin=_is_admin(principal),
encryption_vault_id=encryption_vault_id,
)
uploaded_assets.append(stored.asset)
_audit_connector_imports(session, principal, uploaded_assets)
@@ -426,6 +431,7 @@ def upload_zip(
source_provenance_json: str | None = Form(default=None),
source_revision: str | None = Form(default=None),
connector_policy_json: str | None = Form(default=None),
encryption_vault_id: str | None = Form(default=None),
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(require_scope("files:file:upload")),
):
@@ -458,6 +464,7 @@ def upload_zip(
conflict_resolutions=upload_resolutions,
metadata=metadata,
is_admin=_is_admin(principal),
encryption_vault_id=encryption_vault_id,
max_file_bytes=settings.file_upload_max_bytes,
max_total_bytes=settings.file_upload_zip_max_bytes,
)
@@ -185,6 +185,7 @@ def extract_archive_upload(
conflict_resolutions: Iterable[FileConflictResolution] | None = None,
metadata: dict[str, Any] | None = None,
is_admin: bool = False,
encryption_vault_id: str | None = None,
max_entries: int = ARCHIVE_UPLOAD_MAX_ENTRIES,
max_file_bytes: int = 50 * 1024 * 1024,
max_expanded_bytes: int = 2 * 1024 * 1024 * 1024,
@@ -235,6 +236,7 @@ def extract_archive_upload(
conflict_resolutions=conflict_resolutions,
metadata=metadata,
is_admin=is_admin,
encryption_vault_id=encryption_vault_id,
)
@@ -252,6 +254,7 @@ def extract_zip_upload(
conflict_resolutions: Iterable[FileConflictResolution] | None = None,
metadata: dict[str, Any] | None = None,
is_admin: bool = False,
encryption_vault_id: str | None = None,
max_files: int = ZIP_UPLOAD_MAX_FILES,
max_file_bytes: int = 50 * 1024 * 1024,
max_total_bytes: int = 250 * 1024 * 1024,
@@ -272,6 +275,7 @@ def extract_zip_upload(
conflict_resolutions=conflict_resolutions,
metadata=metadata,
is_admin=is_admin,
encryption_vault_id=encryption_vault_id,
max_entries=max_files,
max_file_bytes=max_file_bytes,
max_expanded_bytes=max_total_bytes,
@@ -584,6 +588,7 @@ def _store_archive_members(
conflict_resolutions: Iterable[FileConflictResolution] | None,
metadata: dict[str, Any] | None,
is_admin: bool,
encryption_vault_id: str | None,
) -> list[UploadedStoredFile]:
uploaded: list[UploadedStoredFile] = []
base_folder = normalize_folder(folder)
@@ -608,6 +613,7 @@ def _store_archive_members(
conflict_strategy=conflict_strategy,
conflict_resolutions=conflict_resolutions,
is_admin=is_admin,
encryption_vault_id=encryption_vault_id,
)
)
return uploaded
@@ -0,0 +1,93 @@
from __future__ import annotations
from govoplan_core.core.encryption import (
ContentProtectionRequest,
ContentUnprotectionRequest,
ProtectedContent,
encryption_content_cipher,
)
from govoplan_files.backend.runtime import get_registry
from govoplan_files.backend.storage.common import FileStorageError
FILES_PROTECTION_PROFILE = "files-server-envelope-v1"
def protect_blob_content(
session: object,
*,
tenant_id: str,
blob_id: str,
vault_id: str,
ciphertext_ref: str,
plaintext: bytes,
actor_id: str,
content_type: str | None,
) -> ProtectedContent:
capability = encryption_content_cipher(get_registry())
if capability is None:
raise FileStorageError(
"File encryption was requested, but the Encryption module is unavailable."
)
try:
return capability.protect_content(
session,
request=ContentProtectionRequest(
tenant_id=tenant_id,
owner_module="files",
resource_type="file_blob",
resource_id=blob_id,
profile_id=FILES_PROTECTION_PROFILE,
vault_id=vault_id,
ciphertext_ref=ciphertext_ref,
plaintext=plaintext,
policy_decision_ref="files:explicit-vault-selection:v1",
idempotency_key=f"file-blob:{blob_id}:content:v1",
actor_id=actor_id,
metadata={
"content_type": content_type or "application/octet-stream",
},
),
)
except Exception as exc:
raise FileStorageError(
"Managed file content could not be protected by the configured vault."
) from exc
def unprotect_blob_content(
session: object,
*,
tenant_id: str,
blob_id: str,
envelope_id: str,
ciphertext: bytes,
) -> bytes:
capability = encryption_content_cipher(get_registry())
if capability is None:
raise FileStorageError(
"This file is encrypted and cannot be read while Encryption is unavailable."
)
try:
return capability.unprotect_content(
session,
request=ContentUnprotectionRequest(
tenant_id=tenant_id,
owner_module="files",
resource_type="file_blob",
resource_id=blob_id,
envelope_id=envelope_id,
ciphertext=ciphertext,
),
)
except Exception as exc:
raise FileStorageError(
"Managed file content could not be opened with its protection envelope."
) from exc
__all__ = [
"FILES_PROTECTION_PROFILE",
"protect_blob_content",
"unprotect_blob_content",
]
+68 -6
View File
@@ -72,12 +72,16 @@ def _get_or_create_blob(
data: bytes,
filename: str,
content_type: str | None,
actor_id: str,
encryption_vault_id: str | None = None,
) -> FileBlob:
checksum = hashlib.sha256(data).hexdigest()
size = len(data)
vault_id = str(encryption_vault_id or "").strip() or None
protection_discriminator = f"vault:{vault_id}" if vault_id else "plaintext"
blob = (
session.query(FileBlob)
.filter(FileBlob.tenant_id == tenant_id, FileBlob.checksum_sha256 == checksum, FileBlob.size_bytes == size)
.filter(FileBlob.tenant_id == tenant_id, FileBlob.checksum_sha256 == checksum, FileBlob.size_bytes == size, FileBlob.protection_discriminator == protection_discriminator)
.one_or_none()
)
if blob:
@@ -93,8 +97,28 @@ def _get_or_create_blob(
except StorageBackendError as exc:
raise FileStorageError(str(exc)) from exc
if repair_required:
stored_data = data
if vault_id:
from govoplan_files.backend.storage.content_protection import protect_blob_content
protected = protect_blob_content(
session,
tenant_id=tenant_id,
blob_id=blob.id,
vault_id=vault_id,
ciphertext_ref=blob.storage_key,
plaintext=data,
actor_id=actor_id,
content_type=content_type,
)
if blob.encryption_envelope_id not in {None, protected.envelope.envelope_id}:
raise FileStorageError("The existing encrypted blob has another protection envelope.")
stored_data = protected.ciphertext
blob.encryption_envelope_id = protected.envelope.envelope_id
blob.storage_checksum_sha256 = hashlib.sha256(stored_data).hexdigest()
blob.storage_size_bytes = len(stored_data)
try:
backend.put_bytes(blob.storage_key, data, content_type=content_type)
backend.put_bytes(blob.storage_key, stored_data, content_type="application/octet-stream" if vault_id else content_type)
except StorageBackendError as exc:
raise FileStorageError(str(exc)) from exc
blob.integrity_status = "verified"
@@ -105,19 +129,42 @@ def _get_or_create_blob(
session.add(blob)
return blob
blob_id = str(uuid4())
storage_key = _storage_key(tenant_id=tenant_id, checksum=checksum, filename=filename)
stored_data = data
envelope_id = None
if vault_id:
from govoplan_files.backend.storage.content_protection import protect_blob_content
protected = protect_blob_content(
session,
tenant_id=tenant_id,
blob_id=blob_id,
vault_id=vault_id,
ciphertext_ref=storage_key,
plaintext=data,
actor_id=actor_id,
content_type=content_type,
)
stored_data = protected.ciphertext
envelope_id = protected.envelope.envelope_id
backend = get_storage_backend()
try:
backend.put_bytes(storage_key, data, content_type=content_type)
backend.put_bytes(storage_key, stored_data, content_type="application/octet-stream" if vault_id else content_type)
except StorageBackendError as exc:
raise FileStorageError(str(exc)) from exc
blob = FileBlob(
id=blob_id,
tenant_id=tenant_id,
storage_backend=_storage_backend_name(),
storage_bucket=_storage_bucket_name(),
storage_key=storage_key,
checksum_sha256=checksum,
size_bytes=size,
protection_discriminator=protection_discriminator,
encryption_envelope_id=envelope_id,
storage_checksum_sha256=hashlib.sha256(stored_data).hexdigest() if vault_id else None,
storage_size_bytes=len(stored_data) if vault_id else None,
content_type=content_type,
ref_count=1,
integrity_status="verified",
@@ -146,6 +193,7 @@ def create_file_asset(
conflict_strategy: str = "reject",
conflict_resolutions: Iterable[FileConflictResolution] | None = None,
is_admin: bool = False,
encryption_vault_id: str | None = None,
) -> UploadedStoredFile:
owner_type = owner_type.lower().strip()
ensure_owner_access(session, tenant_id=tenant_id, owner_type=owner_type, owner_id=owner_id, user_id=user_id, is_admin=is_admin)
@@ -173,7 +221,7 @@ def create_file_asset(
elif action == "rename":
logical_path = _next_available_logical_path(session, tenant_id=tenant_id, owner_type=owner_type, owner_id=owner_id, desired_path=logical_path)
blob = _get_or_create_blob(session, tenant_id=tenant_id, data=data, filename=safe_filename, content_type=content_type)
blob = _get_or_create_blob(session, tenant_id=tenant_id, data=data, filename=safe_filename, content_type=content_type, actor_id=user_id, encryption_vault_id=encryption_vault_id)
asset = FileAsset(
tenant_id=tenant_id,
owner_type=owner_type,
@@ -305,6 +353,7 @@ def update_file_asset_content(
data: bytes,
content_type: str | None,
metadata: dict[str, Any],
encryption_vault_id: str | None = None,
) -> tuple[UploadedStoredFile, str]:
if asset.tenant_id != tenant_id or asset.deleted_at is not None:
raise FileStorageError("File not found")
@@ -315,10 +364,23 @@ def update_file_asset_content(
checksum = hashlib.sha256(data).hexdigest()
asset.metadata_ = metadata
session.add(asset)
if current_blob.checksum_sha256 == checksum and current_blob.size_bytes == len(data):
inherited_vault_id = encryption_vault_id
if inherited_vault_id is None and current_blob.encryption_envelope_id:
prefix = "vault:"
if current_blob.protection_discriminator.startswith(prefix):
inherited_vault_id = current_blob.protection_discriminator[len(prefix) :]
inherited_vault_id = str(inherited_vault_id or "").strip() or None
target_protection = (
f"vault:{inherited_vault_id}" if inherited_vault_id else "plaintext"
)
if (
current_blob.checksum_sha256 == checksum
and current_blob.size_bytes == len(data)
and current_blob.protection_discriminator == target_protection
):
return UploadedStoredFile(asset=asset, version=current_version, blob=current_blob), "unchanged"
blob = _get_or_create_blob(session, tenant_id=tenant_id, data=data, filename=safe_filename, content_type=content_type)
blob = _get_or_create_blob(session, tenant_id=tenant_id, data=data, filename=safe_filename, content_type=content_type, actor_id=user_id, encryption_vault_id=inherited_vault_id)
version = FileVersion(
tenant_id=tenant_id,
file_asset_id=asset.id,
@@ -3,7 +3,7 @@ from __future__ import annotations
import hashlib
from dataclasses import dataclass
from sqlalchemy.orm import Session
from sqlalchemy.orm import Session, object_session
from govoplan_files.backend.db.models import (
FileBlob,
@@ -115,11 +115,13 @@ def inspect_blob(
backend: StorageBackend,
verify_checksum: bool = True,
) -> BlobInspection:
expected_size = blob.storage_size_bytes if blob.storage_size_bytes is not None else blob.size_bytes
expected_checksum = blob.storage_checksum_sha256 if blob.storage_checksum_sha256 is not None else blob.checksum_sha256
try:
info = backend.stat(blob.storage_key)
except StorageObjectMissing:
return BlobInspection(valid=False, kind="missing")
if info.size_bytes != blob.size_bytes:
if info.size_bytes != expected_size:
return BlobInspection(
valid=False,
kind="size_mismatch",
@@ -137,14 +139,14 @@ def inspect_blob(
observed_size += len(chunk)
digest.update(chunk)
observed_checksum = digest.hexdigest()
if observed_size != blob.size_bytes:
if observed_size != expected_size:
return BlobInspection(
valid=False,
kind="size_mismatch",
observed_size_bytes=observed_size,
observed_checksum_sha256=observed_checksum,
)
if observed_checksum != blob.checksum_sha256:
if observed_checksum != expected_checksum:
return BlobInspection(
valid=False,
kind="checksum_mismatch",
@@ -196,14 +198,33 @@ def read_verified_blob_bytes(
data = backend.get_bytes(blob.storage_key)
except StorageBackendError as exc:
raise FileStorageError("Managed file content is not available") from exc
if len(data) != blob.size_bytes:
expected_storage_size = blob.storage_size_bytes if blob.storage_size_bytes is not None else blob.size_bytes
expected_storage_checksum = blob.storage_checksum_sha256 if blob.storage_checksum_sha256 is not None else blob.checksum_sha256
if len(data) != expected_storage_size:
raise FileStorageError(
"Managed file content failed its recorded size verification"
)
if hashlib.sha256(data).hexdigest() != blob.checksum_sha256:
if hashlib.sha256(data).hexdigest() != expected_storage_checksum:
raise FileStorageError(
"Managed file content failed its recorded checksum verification"
)
if blob.encryption_envelope_id:
session = object_session(blob)
if session is None:
raise FileStorageError("Encrypted managed content requires an attached database session")
from govoplan_files.backend.storage.content_protection import unprotect_blob_content
data = unprotect_blob_content(
session,
tenant_id=blob.tenant_id,
blob_id=blob.id,
envelope_id=blob.encryption_envelope_id,
ciphertext=data,
)
if len(data) != blob.size_bytes:
raise FileStorageError("Decrypted managed file content failed its recorded size verification")
if hashlib.sha256(data).hexdigest() != blob.checksum_sha256:
raise FileStorageError("Decrypted managed file content failed its recorded checksum verification")
return data
@@ -432,8 +453,8 @@ def _record_blob_finding(
kind=inspection.kind,
blob_id=blob.id,
storage_key=blob.storage_key,
expected_size_bytes=blob.size_bytes,
expected_checksum_sha256=blob.checksum_sha256,
expected_size_bytes=blob.storage_size_bytes if blob.storage_size_bytes is not None else blob.size_bytes,
expected_checksum_sha256=blob.storage_checksum_sha256 if blob.storage_checksum_sha256 is not None else blob.checksum_sha256,
)
_update_finding_from_inspection(finding, inspection)
session.add(finding)
+149
View File
@@ -0,0 +1,149 @@
from __future__ import annotations
from datetime import UTC, datetime
import hashlib
import unittest
from sqlalchemy import create_engine
from sqlalchemy.orm import Session
from govoplan_core.core.encryption import (
CAPABILITY_ENCRYPTION_CONTENT_CIPHER,
ContentProtectionEnvelope,
ProtectedContent,
)
from govoplan_files.backend.db.models import FileBlob
from govoplan_files.backend.runtime import configure_runtime
from govoplan_files.backend.storage.common import FileStorageError
from govoplan_files.backend.storage.content_protection import protect_blob_content
from govoplan_files.backend.storage.integrity import read_verified_blob_bytes
class Registry:
def __init__(self, capability=None) -> None:
self.capability_value = capability
def has_capability(self, name: str) -> bool:
return (
name == CAPABILITY_ENCRYPTION_CONTENT_CIPHER
and self.capability_value is not None
)
def capability(self, _name: str):
return self.capability_value
class Cipher:
def protect_content(self, _session, *, request):
ciphertext = b"protected:" + request.plaintext
envelope = ContentProtectionEnvelope(
envelope_id="envelope-file-1",
tenant_id=request.tenant_id,
owner_module=request.owner_module,
resource_type=request.resource_type,
resource_id=request.resource_id,
profile_kind="server_envelope",
profile_id=request.profile_id,
provider_id="test",
vault_id=request.vault_id,
key_version=1,
algorithm_suite="AES-256-GCM",
ciphertext_ref=request.ciphertext_ref,
ciphertext_digest=f"sha256:{hashlib.sha256(ciphertext).hexdigest()}",
authenticated_context_digest="sha256:context",
state="active",
created_at=datetime.now(tz=UTC),
wrapped_key_refs=("wrapped:1",),
)
return ProtectedContent(envelope=envelope, ciphertext=ciphertext)
def unprotect_content(self, _session, *, request):
if not request.ciphertext.startswith(b"protected:"):
raise ValueError("invalid fixture ciphertext")
return request.ciphertext.removeprefix(b"protected:")
def execute_rewrap(self, *_args, **_kwargs):
raise NotImplementedError
def prepare_reencryption(self, *_args, **_kwargs):
raise NotImplementedError
class Backend:
def __init__(self, data: bytes) -> None:
self.data = data
def get_bytes(self, _key: str) -> bytes:
return self.data
class FileContentProtectionTests(unittest.TestCase):
def setUp(self) -> None:
self.engine = create_engine("sqlite+pysqlite:///:memory:")
FileBlob.__table__.create(self.engine)
self.session = Session(self.engine)
def tearDown(self) -> None:
self.session.close()
self.engine.dispose()
def test_encrypted_blob_is_opened_after_stored_object_integrity_check(self) -> None:
configure_runtime(registry=Registry(Cipher()), settings=object())
plaintext = b"monthly records"
protected = protect_blob_content(
self.session,
tenant_id="tenant-1",
blob_id="blob-1",
vault_id="vault-1",
ciphertext_ref="tenants/tenant-1/files/blob-1",
plaintext=plaintext,
actor_id="account-1",
content_type="text/plain",
)
blob = FileBlob(
id="blob-1",
tenant_id="tenant-1",
storage_backend="test",
storage_key="tenants/tenant-1/files/blob-1",
checksum_sha256=hashlib.sha256(plaintext).hexdigest(),
size_bytes=len(plaintext),
protection_discriminator="vault:vault-1",
encryption_envelope_id=protected.envelope.envelope_id,
storage_checksum_sha256=hashlib.sha256(protected.ciphertext).hexdigest(),
storage_size_bytes=len(protected.ciphertext),
content_type="text/plain",
ref_count=1,
integrity_status="verified",
)
self.session.add(blob)
self.session.flush()
self.assertEqual(
plaintext,
read_verified_blob_bytes(blob, backend=Backend(protected.ciphertext)),
)
def test_encrypted_blob_fails_closed_without_encryption_capability(self) -> None:
configure_runtime(registry=Registry(), settings=object())
ciphertext = b"protected:monthly records"
blob = FileBlob(
id="blob-1",
tenant_id="tenant-1",
storage_backend="test",
storage_key="tenants/tenant-1/files/blob-1",
checksum_sha256=hashlib.sha256(b"monthly records").hexdigest(),
size_bytes=len(b"monthly records"),
protection_discriminator="vault:vault-1",
encryption_envelope_id="envelope-file-1",
storage_checksum_sha256=hashlib.sha256(ciphertext).hexdigest(),
storage_size_bytes=len(ciphertext),
ref_count=1,
integrity_status="verified",
)
self.session.add(blob)
self.session.flush()
with self.assertRaisesRegex(FileStorageError, "Encryption is unavailable"):
read_verified_blob_bytes(blob, backend=Backend(ciphertext))
if __name__ == "__main__":
unittest.main()
+2
View File
@@ -10,6 +10,7 @@ STATIC_TOPIC_IDS = {
"files.workflow.delete-managed-files",
"files.governed-connectors-and-provenance",
"files.reference.integrity-recovery-and-fail-closed-transports",
"files.reference.shared-storage-profile",
"files.reference.snapshot-provenance-and-capabilities",
"files.assurance.process-and-release-readiness",
}
@@ -157,6 +158,7 @@ class FilesManifestDocumentationTests(unittest.TestCase):
self.assertIn("bounded resumable integrity scan", topic.body)
self.assertIn("quarantined", topic.body)
self.assertIn("MASTER_KEY_B64", topic.metadata["recovery_unit"])
self.assertIn("Encryption envelope and wrapped-key rows", topic.metadata["recovery_unit"])
self.assertTrue(topic.metadata["verification"])
self.assertIn(
"/api/v1/files/integrity/scans",