feat: support optional encrypted file content
This commit is contained in:
+20
-10
@@ -29,12 +29,18 @@ a document collaboration engine, or a records-management system.
|
||||
A managed file is a tenant-scoped logical asset with exactly one user or group
|
||||
owner, a normalized path, and a current version. The current version points to a
|
||||
blob record containing the storage location, SHA-256 checksum, byte size, and
|
||||
content type. Current service paths append a version when connector sync finds
|
||||
content type. A protected blob also records its Encryption envelope, protection
|
||||
discriminator, stored-ciphertext checksum, and stored-ciphertext size. Current
|
||||
service paths append a version when connector sync finds
|
||||
changed content; they do not mutate the previous version record.
|
||||
|
||||
Content with the same tenant, SHA-256 checksum, and size can reuse one blob.
|
||||
This is storage deduplication and integrity evidence, not proof of authorship or
|
||||
source authenticity.
|
||||
Unprotected content with the same tenant, plaintext SHA-256 checksum, size, and
|
||||
protection discriminator can reuse one blob. Protected content is deduplicated
|
||||
only inside the same vault/profile discriminator; ciphertext is never silently
|
||||
reused across protection boundaries. Plaintext checksums remain semantic
|
||||
version evidence, while download and integrity scans verify stored ciphertext
|
||||
before asking Encryption to open it. Neither digest proves authorship or source
|
||||
authenticity.
|
||||
|
||||
The main domain objects are:
|
||||
|
||||
@@ -42,7 +48,7 @@ The main domain objects are:
|
||||
| --- | --- | --- |
|
||||
| File asset | The user-facing file identity, owner, logical path, description, metadata, and current version | Created, organized, shared, and soft-deleted |
|
||||
| File version | A numbered snapshot of one asset and its blob | Appended by connector sync when bytes change; retained |
|
||||
| File blob | Stored bytes plus checksum, size, backend, key, reference count, and optional retention timestamp | Reused within a tenant; no automated garbage collection |
|
||||
| File blob | Stored plaintext semantics plus stored-byte integrity, backend key, optional Encryption envelope, reference count, and retention timestamp | Reused only within a tenant and matching protection boundary; no automated garbage collection |
|
||||
| Folder | An explicit logical path in a user or group space | Created, moved/renamed through organize operations, and soft-deleted |
|
||||
| Share | A grant from an asset to a user, group, tenant, or campaign with `read`, `write`, or `manage` permission | Created or updated; no public revocation endpoint yet |
|
||||
| Connector profile | A governed external endpoint, scope, optional credential link, policy, and descriptive capabilities | Created, updated, disabled, or credential-scrubbed on deletion |
|
||||
@@ -634,13 +640,17 @@ Files baseline indiscriminately.
|
||||
secrets and deployment references.
|
||||
- API metadata is recursively checked for secret-like values.
|
||||
- Downloads use sanitized attachment filenames.
|
||||
- SHA-256 and byte size are recorded for every blob/version.
|
||||
- Plaintext semantic and stored-byte SHA-256/size evidence are recorded for
|
||||
every protected blob; they are identical for unprotected blobs.
|
||||
- Upload and archive-confirm APIs can select an Encryption vault. Protected
|
||||
writes and reads fail closed if the optional Encryption capability is absent.
|
||||
|
||||
The module does **not** currently provide malware scanning, content disarm and
|
||||
reconstruction, a file-type allowlist, per-user quota, at-rest encryption for
|
||||
local blob bytes, or a dedicated preview sandbox. Deployments that require
|
||||
these controls must supply them outside Files until explicit module contracts
|
||||
exist.
|
||||
reconstruction, a file-type allowlist, per-user quota, automatic encryption
|
||||
policy assignment, client E2EE, or a dedicated preview sandbox. The optional
|
||||
server-envelope profile protects selected managed blob bytes at rest but remains
|
||||
server-decryptable. Deployments that require the other controls must supply
|
||||
them outside Files until explicit module contracts exist.
|
||||
|
||||
### Provenance
|
||||
|
||||
|
||||
@@ -16,7 +16,15 @@ def new_uuid() -> str:
|
||||
|
||||
class FileBlob(Base, TimestampMixin):
|
||||
__tablename__ = "file_blobs"
|
||||
__table_args__ = (UniqueConstraint("tenant_id", "checksum_sha256", "size_bytes", name="uq_file_blobs_tenant_checksum_size"),)
|
||||
__table_args__ = (
|
||||
UniqueConstraint(
|
||||
"tenant_id",
|
||||
"checksum_sha256",
|
||||
"size_bytes",
|
||||
"protection_discriminator",
|
||||
name="uq_file_blobs_tenant_checksum_size_protection",
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||
@@ -25,6 +33,10 @@ class FileBlob(Base, TimestampMixin):
|
||||
storage_key: Mapped[str] = mapped_column(String(1000), nullable=False)
|
||||
checksum_sha256: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
size_bytes: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
protection_discriminator: Mapped[str] = mapped_column(String(320), default="plaintext", nullable=False, index=True)
|
||||
encryption_envelope_id: Mapped[str | None] = mapped_column(String(255), nullable=True, index=True)
|
||||
storage_checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
storage_size_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
content_type: Mapped[str | None] = mapped_column(String(255))
|
||||
ref_count: Mapped[int] = mapped_column(Integer, default=1, nullable=False)
|
||||
retained_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
|
||||
@@ -6,6 +6,7 @@ from pathlib import Path
|
||||
from sqlalchemy import inspect
|
||||
|
||||
from govoplan_core.core.access import CAPABILITY_AUTH_PERMISSION_EVALUATOR, CAPABILITY_AUTH_PRINCIPAL_RESOLVER
|
||||
from govoplan_core.core.encryption import CAPABILITY_ENCRYPTION_CONTENT_CIPHER
|
||||
from govoplan_core.core.files import CAPABILITY_FILES_ACCESS
|
||||
from govoplan_core.core.module_guards import drop_table_retirement_provider, persistent_table_uninstall_guard
|
||||
from govoplan_core.core.modules import (
|
||||
@@ -275,7 +276,7 @@ manifest = ModuleManifest(
|
||||
name="Files",
|
||||
version="0.1.9",
|
||||
required_capabilities=(CAPABILITY_AUTH_PRINCIPAL_RESOLVER, CAPABILITY_AUTH_PERMISSION_EVALUATOR),
|
||||
optional_dependencies=("campaigns",),
|
||||
optional_dependencies=("campaigns", "encryption"),
|
||||
provides_interfaces=(
|
||||
ModuleInterfaceProvider(name="files.access", version="0.1.6"),
|
||||
ModuleInterfaceProvider(name="files.campaign_attachments", version="0.1.6"),
|
||||
@@ -287,6 +288,12 @@ manifest = ModuleManifest(
|
||||
version_max_exclusive="0.2.0",
|
||||
optional=True,
|
||||
),
|
||||
ModuleInterfaceRequirement(
|
||||
name=CAPABILITY_ENCRYPTION_CONTENT_CIPHER,
|
||||
version_min="1.0.0",
|
||||
version_max_exclusive="2.0.0",
|
||||
optional=True,
|
||||
),
|
||||
),
|
||||
permissions=PERMISSIONS,
|
||||
route_factory=_files_router,
|
||||
@@ -570,9 +577,9 @@ manifest = ModuleManifest(
|
||||
DocumentationTopic(
|
||||
id="files.reference.integrity-recovery-and-fail-closed-transports",
|
||||
title="Operate Files integrity, recovery, and connector transport safety",
|
||||
summary="Back up database evidence, blob bytes, and the encryption key as one recovery unit, and keep unsupported SDK transports fail-closed.",
|
||||
summary="Back up database evidence, blob ciphertext, and Encryption custody as one recovery unit, and keep unsupported SDK transports fail-closed.",
|
||||
body=(
|
||||
"Local durable storage is the operational baseline. Recover Files from a coordinated database/blob snapshot with the original master key, then run the bounded resumable integrity scan and verify representative access paths. Missing or mismatched blobs are quarantined; orphan objects are reported before dry-run-first, explicitly authorized cleanup. "
|
||||
"Local durable storage is the operational baseline. Recover Files from a coordinated database/blob snapshot with the matching Encryption tables and original deployment master key, then run the bounded resumable integrity scan and verify representative protected and unprotected access paths. Protected scans verify stored ciphertext before decryption and then verify plaintext semantic evidence. Missing or mismatched blobs are quarantined; orphan objects are reported before dry-run-first, explicitly authorized cleanup. "
|
||||
"Arbitrary external S3 managed storage/connectors and SMB connectors fail closed until botocore redirects/endpoint discovery and SMB initial connections/DFS referrals support connection-time DNS/IP pinning. Installer-owned Garage storage is supported only at the exact deployment service endpoint with its explicit trust marker. Destructive module retirement drops database tables but does not remove backend blob objects."
|
||||
),
|
||||
layer="configured",
|
||||
@@ -596,7 +603,7 @@ manifest = ModuleManifest(
|
||||
DocumentationLink(label="Create an integrity scan", href="/api/v1/files/integrity/scans", kind="api"),
|
||||
DocumentationLink(label="Files handbook", href="govoplan-files/docs/FILES_HANDBOOK.md", kind="repository"),
|
||||
),
|
||||
related_modules=("audit", "ops"),
|
||||
related_modules=("audit", "encryption", "ops"),
|
||||
unlocks=("Recoverable managed-file evidence without weakening outbound peer validation.",),
|
||||
configuration_keys=(
|
||||
"FILE_STORAGE_BACKEND",
|
||||
@@ -619,7 +626,7 @@ manifest = ModuleManifest(
|
||||
"route": "/admin?section=system-file-connectors",
|
||||
"screen": "System file connections and deployment operations",
|
||||
"section": "Storage integrity, backup/recovery, and fail-closed transports",
|
||||
"recovery_unit": ["Files database rows", "managed blob namespace", "MASTER_KEY_B64", "deployment-owned connector configuration"],
|
||||
"recovery_unit": ["Files database rows", "Encryption envelope and wrapped-key rows", "managed blob namespace", "MASTER_KEY_B64", "deployment-owned connector configuration"],
|
||||
"verification": "After restore, complete a checksum-enabled integrity scan, resolve every missing/corrupt finding, approve or retain every reported orphan, verify authorized and denied access, and test one permitted pinned HTTP connector.",
|
||||
"related_topic_ids": [
|
||||
"files.governed-connectors-and-provenance",
|
||||
|
||||
+79
@@ -0,0 +1,79 @@
|
||||
"""file content-protection metadata
|
||||
|
||||
Revision ID: d0e1f2a3b4c6
|
||||
Revises: c9d0e1f2a3b5
|
||||
Create Date: 2026-08-02 00:00:00.000000
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
revision = "d0e1f2a3b4c6"
|
||||
down_revision = "c9d0e1f2a3b5"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
with op.batch_alter_table("file_blobs") as batch_op:
|
||||
batch_op.add_column(
|
||||
sa.Column(
|
||||
"protection_discriminator",
|
||||
sa.String(length=320),
|
||||
nullable=False,
|
||||
server_default="plaintext",
|
||||
)
|
||||
)
|
||||
batch_op.add_column(
|
||||
sa.Column("encryption_envelope_id", sa.String(length=255), nullable=True)
|
||||
)
|
||||
batch_op.add_column(
|
||||
sa.Column("storage_checksum_sha256", sa.String(length=64), nullable=True)
|
||||
)
|
||||
batch_op.add_column(
|
||||
sa.Column("storage_size_bytes", sa.Integer(), nullable=True)
|
||||
)
|
||||
batch_op.drop_constraint(
|
||||
"uq_file_blobs_tenant_checksum_size",
|
||||
type_="unique",
|
||||
)
|
||||
batch_op.create_unique_constraint(
|
||||
"uq_file_blobs_tenant_checksum_size_protection",
|
||||
[
|
||||
"tenant_id",
|
||||
"checksum_sha256",
|
||||
"size_bytes",
|
||||
"protection_discriminator",
|
||||
],
|
||||
)
|
||||
batch_op.create_index(
|
||||
op.f("ix_file_blobs_protection_discriminator"),
|
||||
["protection_discriminator"],
|
||||
unique=False,
|
||||
)
|
||||
batch_op.create_index(
|
||||
op.f("ix_file_blobs_encryption_envelope_id"),
|
||||
["encryption_envelope_id"],
|
||||
unique=False,
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
with op.batch_alter_table("file_blobs") as batch_op:
|
||||
batch_op.drop_index(op.f("ix_file_blobs_encryption_envelope_id"))
|
||||
batch_op.drop_index(op.f("ix_file_blobs_protection_discriminator"))
|
||||
batch_op.drop_constraint(
|
||||
"uq_file_blobs_tenant_checksum_size_protection",
|
||||
type_="unique",
|
||||
)
|
||||
batch_op.create_unique_constraint(
|
||||
"uq_file_blobs_tenant_checksum_size",
|
||||
["tenant_id", "checksum_sha256", "size_bytes"],
|
||||
)
|
||||
batch_op.drop_column("storage_size_bytes")
|
||||
batch_op.drop_column("storage_checksum_sha256")
|
||||
batch_op.drop_column("encryption_envelope_id")
|
||||
batch_op.drop_column("protection_discriminator")
|
||||
@@ -224,6 +224,7 @@ def confirm_archive_upload(
|
||||
source_provenance_json: str | None = Form(default=None),
|
||||
source_revision: str | None = Form(default=None),
|
||||
connector_policy_json: str | None = Form(default=None),
|
||||
encryption_vault_id: str | None = Form(default=None),
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(require_scope("files:file:upload")),
|
||||
):
|
||||
@@ -283,6 +284,7 @@ def confirm_archive_upload(
|
||||
conflict_resolutions=upload_resolutions,
|
||||
metadata=metadata,
|
||||
is_admin=_is_admin(principal),
|
||||
encryption_vault_id=encryption_vault_id,
|
||||
max_entries=settings.file_archive_max_entries,
|
||||
max_file_bytes=settings.file_upload_max_bytes,
|
||||
max_expanded_bytes=settings.file_archive_max_expanded_bytes,
|
||||
@@ -329,6 +331,7 @@ def upload_files(
|
||||
source_provenance_json: str | None = Form(default=None),
|
||||
source_revision: str | None = Form(default=None),
|
||||
connector_policy_json: str | None = Form(default=None),
|
||||
encryption_vault_id: str | None = Form(default=None),
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(require_scope("files:file:upload")),
|
||||
):
|
||||
@@ -371,6 +374,7 @@ def upload_files(
|
||||
conflict_resolutions=upload_resolutions,
|
||||
metadata=metadata,
|
||||
is_admin=_is_admin(principal),
|
||||
encryption_vault_id=encryption_vault_id,
|
||||
max_file_bytes=settings.file_upload_max_bytes,
|
||||
max_total_bytes=settings.file_upload_zip_max_bytes,
|
||||
)
|
||||
@@ -394,6 +398,7 @@ def upload_files(
|
||||
conflict_resolutions=upload_resolutions,
|
||||
metadata=metadata,
|
||||
is_admin=_is_admin(principal),
|
||||
encryption_vault_id=encryption_vault_id,
|
||||
)
|
||||
uploaded_assets.append(stored.asset)
|
||||
_audit_connector_imports(session, principal, uploaded_assets)
|
||||
@@ -426,6 +431,7 @@ def upload_zip(
|
||||
source_provenance_json: str | None = Form(default=None),
|
||||
source_revision: str | None = Form(default=None),
|
||||
connector_policy_json: str | None = Form(default=None),
|
||||
encryption_vault_id: str | None = Form(default=None),
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(require_scope("files:file:upload")),
|
||||
):
|
||||
@@ -458,6 +464,7 @@ def upload_zip(
|
||||
conflict_resolutions=upload_resolutions,
|
||||
metadata=metadata,
|
||||
is_admin=_is_admin(principal),
|
||||
encryption_vault_id=encryption_vault_id,
|
||||
max_file_bytes=settings.file_upload_max_bytes,
|
||||
max_total_bytes=settings.file_upload_zip_max_bytes,
|
||||
)
|
||||
|
||||
@@ -185,6 +185,7 @@ def extract_archive_upload(
|
||||
conflict_resolutions: Iterable[FileConflictResolution] | None = None,
|
||||
metadata: dict[str, Any] | None = None,
|
||||
is_admin: bool = False,
|
||||
encryption_vault_id: str | None = None,
|
||||
max_entries: int = ARCHIVE_UPLOAD_MAX_ENTRIES,
|
||||
max_file_bytes: int = 50 * 1024 * 1024,
|
||||
max_expanded_bytes: int = 2 * 1024 * 1024 * 1024,
|
||||
@@ -235,6 +236,7 @@ def extract_archive_upload(
|
||||
conflict_resolutions=conflict_resolutions,
|
||||
metadata=metadata,
|
||||
is_admin=is_admin,
|
||||
encryption_vault_id=encryption_vault_id,
|
||||
)
|
||||
|
||||
|
||||
@@ -252,6 +254,7 @@ def extract_zip_upload(
|
||||
conflict_resolutions: Iterable[FileConflictResolution] | None = None,
|
||||
metadata: dict[str, Any] | None = None,
|
||||
is_admin: bool = False,
|
||||
encryption_vault_id: str | None = None,
|
||||
max_files: int = ZIP_UPLOAD_MAX_FILES,
|
||||
max_file_bytes: int = 50 * 1024 * 1024,
|
||||
max_total_bytes: int = 250 * 1024 * 1024,
|
||||
@@ -272,6 +275,7 @@ def extract_zip_upload(
|
||||
conflict_resolutions=conflict_resolutions,
|
||||
metadata=metadata,
|
||||
is_admin=is_admin,
|
||||
encryption_vault_id=encryption_vault_id,
|
||||
max_entries=max_files,
|
||||
max_file_bytes=max_file_bytes,
|
||||
max_expanded_bytes=max_total_bytes,
|
||||
@@ -584,6 +588,7 @@ def _store_archive_members(
|
||||
conflict_resolutions: Iterable[FileConflictResolution] | None,
|
||||
metadata: dict[str, Any] | None,
|
||||
is_admin: bool,
|
||||
encryption_vault_id: str | None,
|
||||
) -> list[UploadedStoredFile]:
|
||||
uploaded: list[UploadedStoredFile] = []
|
||||
base_folder = normalize_folder(folder)
|
||||
@@ -608,6 +613,7 @@ def _store_archive_members(
|
||||
conflict_strategy=conflict_strategy,
|
||||
conflict_resolutions=conflict_resolutions,
|
||||
is_admin=is_admin,
|
||||
encryption_vault_id=encryption_vault_id,
|
||||
)
|
||||
)
|
||||
return uploaded
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from govoplan_core.core.encryption import (
|
||||
ContentProtectionRequest,
|
||||
ContentUnprotectionRequest,
|
||||
ProtectedContent,
|
||||
encryption_content_cipher,
|
||||
)
|
||||
from govoplan_files.backend.runtime import get_registry
|
||||
from govoplan_files.backend.storage.common import FileStorageError
|
||||
|
||||
|
||||
FILES_PROTECTION_PROFILE = "files-server-envelope-v1"
|
||||
|
||||
|
||||
def protect_blob_content(
|
||||
session: object,
|
||||
*,
|
||||
tenant_id: str,
|
||||
blob_id: str,
|
||||
vault_id: str,
|
||||
ciphertext_ref: str,
|
||||
plaintext: bytes,
|
||||
actor_id: str,
|
||||
content_type: str | None,
|
||||
) -> ProtectedContent:
|
||||
capability = encryption_content_cipher(get_registry())
|
||||
if capability is None:
|
||||
raise FileStorageError(
|
||||
"File encryption was requested, but the Encryption module is unavailable."
|
||||
)
|
||||
try:
|
||||
return capability.protect_content(
|
||||
session,
|
||||
request=ContentProtectionRequest(
|
||||
tenant_id=tenant_id,
|
||||
owner_module="files",
|
||||
resource_type="file_blob",
|
||||
resource_id=blob_id,
|
||||
profile_id=FILES_PROTECTION_PROFILE,
|
||||
vault_id=vault_id,
|
||||
ciphertext_ref=ciphertext_ref,
|
||||
plaintext=plaintext,
|
||||
policy_decision_ref="files:explicit-vault-selection:v1",
|
||||
idempotency_key=f"file-blob:{blob_id}:content:v1",
|
||||
actor_id=actor_id,
|
||||
metadata={
|
||||
"content_type": content_type or "application/octet-stream",
|
||||
},
|
||||
),
|
||||
)
|
||||
except Exception as exc:
|
||||
raise FileStorageError(
|
||||
"Managed file content could not be protected by the configured vault."
|
||||
) from exc
|
||||
|
||||
|
||||
def unprotect_blob_content(
|
||||
session: object,
|
||||
*,
|
||||
tenant_id: str,
|
||||
blob_id: str,
|
||||
envelope_id: str,
|
||||
ciphertext: bytes,
|
||||
) -> bytes:
|
||||
capability = encryption_content_cipher(get_registry())
|
||||
if capability is None:
|
||||
raise FileStorageError(
|
||||
"This file is encrypted and cannot be read while Encryption is unavailable."
|
||||
)
|
||||
try:
|
||||
return capability.unprotect_content(
|
||||
session,
|
||||
request=ContentUnprotectionRequest(
|
||||
tenant_id=tenant_id,
|
||||
owner_module="files",
|
||||
resource_type="file_blob",
|
||||
resource_id=blob_id,
|
||||
envelope_id=envelope_id,
|
||||
ciphertext=ciphertext,
|
||||
),
|
||||
)
|
||||
except Exception as exc:
|
||||
raise FileStorageError(
|
||||
"Managed file content could not be opened with its protection envelope."
|
||||
) from exc
|
||||
|
||||
|
||||
__all__ = [
|
||||
"FILES_PROTECTION_PROFILE",
|
||||
"protect_blob_content",
|
||||
"unprotect_blob_content",
|
||||
]
|
||||
@@ -72,12 +72,16 @@ def _get_or_create_blob(
|
||||
data: bytes,
|
||||
filename: str,
|
||||
content_type: str | None,
|
||||
actor_id: str,
|
||||
encryption_vault_id: str | None = None,
|
||||
) -> FileBlob:
|
||||
checksum = hashlib.sha256(data).hexdigest()
|
||||
size = len(data)
|
||||
vault_id = str(encryption_vault_id or "").strip() or None
|
||||
protection_discriminator = f"vault:{vault_id}" if vault_id else "plaintext"
|
||||
blob = (
|
||||
session.query(FileBlob)
|
||||
.filter(FileBlob.tenant_id == tenant_id, FileBlob.checksum_sha256 == checksum, FileBlob.size_bytes == size)
|
||||
.filter(FileBlob.tenant_id == tenant_id, FileBlob.checksum_sha256 == checksum, FileBlob.size_bytes == size, FileBlob.protection_discriminator == protection_discriminator)
|
||||
.one_or_none()
|
||||
)
|
||||
if blob:
|
||||
@@ -93,8 +97,28 @@ def _get_or_create_blob(
|
||||
except StorageBackendError as exc:
|
||||
raise FileStorageError(str(exc)) from exc
|
||||
if repair_required:
|
||||
stored_data = data
|
||||
if vault_id:
|
||||
from govoplan_files.backend.storage.content_protection import protect_blob_content
|
||||
|
||||
protected = protect_blob_content(
|
||||
session,
|
||||
tenant_id=tenant_id,
|
||||
blob_id=blob.id,
|
||||
vault_id=vault_id,
|
||||
ciphertext_ref=blob.storage_key,
|
||||
plaintext=data,
|
||||
actor_id=actor_id,
|
||||
content_type=content_type,
|
||||
)
|
||||
if blob.encryption_envelope_id not in {None, protected.envelope.envelope_id}:
|
||||
raise FileStorageError("The existing encrypted blob has another protection envelope.")
|
||||
stored_data = protected.ciphertext
|
||||
blob.encryption_envelope_id = protected.envelope.envelope_id
|
||||
blob.storage_checksum_sha256 = hashlib.sha256(stored_data).hexdigest()
|
||||
blob.storage_size_bytes = len(stored_data)
|
||||
try:
|
||||
backend.put_bytes(blob.storage_key, data, content_type=content_type)
|
||||
backend.put_bytes(blob.storage_key, stored_data, content_type="application/octet-stream" if vault_id else content_type)
|
||||
except StorageBackendError as exc:
|
||||
raise FileStorageError(str(exc)) from exc
|
||||
blob.integrity_status = "verified"
|
||||
@@ -105,19 +129,42 @@ def _get_or_create_blob(
|
||||
session.add(blob)
|
||||
return blob
|
||||
|
||||
blob_id = str(uuid4())
|
||||
storage_key = _storage_key(tenant_id=tenant_id, checksum=checksum, filename=filename)
|
||||
stored_data = data
|
||||
envelope_id = None
|
||||
if vault_id:
|
||||
from govoplan_files.backend.storage.content_protection import protect_blob_content
|
||||
|
||||
protected = protect_blob_content(
|
||||
session,
|
||||
tenant_id=tenant_id,
|
||||
blob_id=blob_id,
|
||||
vault_id=vault_id,
|
||||
ciphertext_ref=storage_key,
|
||||
plaintext=data,
|
||||
actor_id=actor_id,
|
||||
content_type=content_type,
|
||||
)
|
||||
stored_data = protected.ciphertext
|
||||
envelope_id = protected.envelope.envelope_id
|
||||
backend = get_storage_backend()
|
||||
try:
|
||||
backend.put_bytes(storage_key, data, content_type=content_type)
|
||||
backend.put_bytes(storage_key, stored_data, content_type="application/octet-stream" if vault_id else content_type)
|
||||
except StorageBackendError as exc:
|
||||
raise FileStorageError(str(exc)) from exc
|
||||
blob = FileBlob(
|
||||
id=blob_id,
|
||||
tenant_id=tenant_id,
|
||||
storage_backend=_storage_backend_name(),
|
||||
storage_bucket=_storage_bucket_name(),
|
||||
storage_key=storage_key,
|
||||
checksum_sha256=checksum,
|
||||
size_bytes=size,
|
||||
protection_discriminator=protection_discriminator,
|
||||
encryption_envelope_id=envelope_id,
|
||||
storage_checksum_sha256=hashlib.sha256(stored_data).hexdigest() if vault_id else None,
|
||||
storage_size_bytes=len(stored_data) if vault_id else None,
|
||||
content_type=content_type,
|
||||
ref_count=1,
|
||||
integrity_status="verified",
|
||||
@@ -146,6 +193,7 @@ def create_file_asset(
|
||||
conflict_strategy: str = "reject",
|
||||
conflict_resolutions: Iterable[FileConflictResolution] | None = None,
|
||||
is_admin: bool = False,
|
||||
encryption_vault_id: str | None = None,
|
||||
) -> UploadedStoredFile:
|
||||
owner_type = owner_type.lower().strip()
|
||||
ensure_owner_access(session, tenant_id=tenant_id, owner_type=owner_type, owner_id=owner_id, user_id=user_id, is_admin=is_admin)
|
||||
@@ -173,7 +221,7 @@ def create_file_asset(
|
||||
elif action == "rename":
|
||||
logical_path = _next_available_logical_path(session, tenant_id=tenant_id, owner_type=owner_type, owner_id=owner_id, desired_path=logical_path)
|
||||
|
||||
blob = _get_or_create_blob(session, tenant_id=tenant_id, data=data, filename=safe_filename, content_type=content_type)
|
||||
blob = _get_or_create_blob(session, tenant_id=tenant_id, data=data, filename=safe_filename, content_type=content_type, actor_id=user_id, encryption_vault_id=encryption_vault_id)
|
||||
asset = FileAsset(
|
||||
tenant_id=tenant_id,
|
||||
owner_type=owner_type,
|
||||
@@ -305,6 +353,7 @@ def update_file_asset_content(
|
||||
data: bytes,
|
||||
content_type: str | None,
|
||||
metadata: dict[str, Any],
|
||||
encryption_vault_id: str | None = None,
|
||||
) -> tuple[UploadedStoredFile, str]:
|
||||
if asset.tenant_id != tenant_id or asset.deleted_at is not None:
|
||||
raise FileStorageError("File not found")
|
||||
@@ -315,10 +364,23 @@ def update_file_asset_content(
|
||||
checksum = hashlib.sha256(data).hexdigest()
|
||||
asset.metadata_ = metadata
|
||||
session.add(asset)
|
||||
if current_blob.checksum_sha256 == checksum and current_blob.size_bytes == len(data):
|
||||
inherited_vault_id = encryption_vault_id
|
||||
if inherited_vault_id is None and current_blob.encryption_envelope_id:
|
||||
prefix = "vault:"
|
||||
if current_blob.protection_discriminator.startswith(prefix):
|
||||
inherited_vault_id = current_blob.protection_discriminator[len(prefix) :]
|
||||
inherited_vault_id = str(inherited_vault_id or "").strip() or None
|
||||
target_protection = (
|
||||
f"vault:{inherited_vault_id}" if inherited_vault_id else "plaintext"
|
||||
)
|
||||
if (
|
||||
current_blob.checksum_sha256 == checksum
|
||||
and current_blob.size_bytes == len(data)
|
||||
and current_blob.protection_discriminator == target_protection
|
||||
):
|
||||
return UploadedStoredFile(asset=asset, version=current_version, blob=current_blob), "unchanged"
|
||||
|
||||
blob = _get_or_create_blob(session, tenant_id=tenant_id, data=data, filename=safe_filename, content_type=content_type)
|
||||
blob = _get_or_create_blob(session, tenant_id=tenant_id, data=data, filename=safe_filename, content_type=content_type, actor_id=user_id, encryption_vault_id=inherited_vault_id)
|
||||
version = FileVersion(
|
||||
tenant_id=tenant_id,
|
||||
file_asset_id=asset.id,
|
||||
|
||||
@@ -3,7 +3,7 @@ from __future__ import annotations
|
||||
import hashlib
|
||||
from dataclasses import dataclass
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy.orm import Session, object_session
|
||||
|
||||
from govoplan_files.backend.db.models import (
|
||||
FileBlob,
|
||||
@@ -115,11 +115,13 @@ def inspect_blob(
|
||||
backend: StorageBackend,
|
||||
verify_checksum: bool = True,
|
||||
) -> BlobInspection:
|
||||
expected_size = blob.storage_size_bytes if blob.storage_size_bytes is not None else blob.size_bytes
|
||||
expected_checksum = blob.storage_checksum_sha256 if blob.storage_checksum_sha256 is not None else blob.checksum_sha256
|
||||
try:
|
||||
info = backend.stat(blob.storage_key)
|
||||
except StorageObjectMissing:
|
||||
return BlobInspection(valid=False, kind="missing")
|
||||
if info.size_bytes != blob.size_bytes:
|
||||
if info.size_bytes != expected_size:
|
||||
return BlobInspection(
|
||||
valid=False,
|
||||
kind="size_mismatch",
|
||||
@@ -137,14 +139,14 @@ def inspect_blob(
|
||||
observed_size += len(chunk)
|
||||
digest.update(chunk)
|
||||
observed_checksum = digest.hexdigest()
|
||||
if observed_size != blob.size_bytes:
|
||||
if observed_size != expected_size:
|
||||
return BlobInspection(
|
||||
valid=False,
|
||||
kind="size_mismatch",
|
||||
observed_size_bytes=observed_size,
|
||||
observed_checksum_sha256=observed_checksum,
|
||||
)
|
||||
if observed_checksum != blob.checksum_sha256:
|
||||
if observed_checksum != expected_checksum:
|
||||
return BlobInspection(
|
||||
valid=False,
|
||||
kind="checksum_mismatch",
|
||||
@@ -196,14 +198,33 @@ def read_verified_blob_bytes(
|
||||
data = backend.get_bytes(blob.storage_key)
|
||||
except StorageBackendError as exc:
|
||||
raise FileStorageError("Managed file content is not available") from exc
|
||||
if len(data) != blob.size_bytes:
|
||||
expected_storage_size = blob.storage_size_bytes if blob.storage_size_bytes is not None else blob.size_bytes
|
||||
expected_storage_checksum = blob.storage_checksum_sha256 if blob.storage_checksum_sha256 is not None else blob.checksum_sha256
|
||||
if len(data) != expected_storage_size:
|
||||
raise FileStorageError(
|
||||
"Managed file content failed its recorded size verification"
|
||||
)
|
||||
if hashlib.sha256(data).hexdigest() != blob.checksum_sha256:
|
||||
if hashlib.sha256(data).hexdigest() != expected_storage_checksum:
|
||||
raise FileStorageError(
|
||||
"Managed file content failed its recorded checksum verification"
|
||||
)
|
||||
if blob.encryption_envelope_id:
|
||||
session = object_session(blob)
|
||||
if session is None:
|
||||
raise FileStorageError("Encrypted managed content requires an attached database session")
|
||||
from govoplan_files.backend.storage.content_protection import unprotect_blob_content
|
||||
|
||||
data = unprotect_blob_content(
|
||||
session,
|
||||
tenant_id=blob.tenant_id,
|
||||
blob_id=blob.id,
|
||||
envelope_id=blob.encryption_envelope_id,
|
||||
ciphertext=data,
|
||||
)
|
||||
if len(data) != blob.size_bytes:
|
||||
raise FileStorageError("Decrypted managed file content failed its recorded size verification")
|
||||
if hashlib.sha256(data).hexdigest() != blob.checksum_sha256:
|
||||
raise FileStorageError("Decrypted managed file content failed its recorded checksum verification")
|
||||
return data
|
||||
|
||||
|
||||
@@ -432,8 +453,8 @@ def _record_blob_finding(
|
||||
kind=inspection.kind,
|
||||
blob_id=blob.id,
|
||||
storage_key=blob.storage_key,
|
||||
expected_size_bytes=blob.size_bytes,
|
||||
expected_checksum_sha256=blob.checksum_sha256,
|
||||
expected_size_bytes=blob.storage_size_bytes if blob.storage_size_bytes is not None else blob.size_bytes,
|
||||
expected_checksum_sha256=blob.storage_checksum_sha256 if blob.storage_checksum_sha256 is not None else blob.checksum_sha256,
|
||||
)
|
||||
_update_finding_from_inspection(finding, inspection)
|
||||
session.add(finding)
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
import hashlib
|
||||
import unittest
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from govoplan_core.core.encryption import (
|
||||
CAPABILITY_ENCRYPTION_CONTENT_CIPHER,
|
||||
ContentProtectionEnvelope,
|
||||
ProtectedContent,
|
||||
)
|
||||
from govoplan_files.backend.db.models import FileBlob
|
||||
from govoplan_files.backend.runtime import configure_runtime
|
||||
from govoplan_files.backend.storage.common import FileStorageError
|
||||
from govoplan_files.backend.storage.content_protection import protect_blob_content
|
||||
from govoplan_files.backend.storage.integrity import read_verified_blob_bytes
|
||||
|
||||
|
||||
class Registry:
|
||||
def __init__(self, capability=None) -> None:
|
||||
self.capability_value = capability
|
||||
|
||||
def has_capability(self, name: str) -> bool:
|
||||
return (
|
||||
name == CAPABILITY_ENCRYPTION_CONTENT_CIPHER
|
||||
and self.capability_value is not None
|
||||
)
|
||||
|
||||
def capability(self, _name: str):
|
||||
return self.capability_value
|
||||
|
||||
|
||||
class Cipher:
|
||||
def protect_content(self, _session, *, request):
|
||||
ciphertext = b"protected:" + request.plaintext
|
||||
envelope = ContentProtectionEnvelope(
|
||||
envelope_id="envelope-file-1",
|
||||
tenant_id=request.tenant_id,
|
||||
owner_module=request.owner_module,
|
||||
resource_type=request.resource_type,
|
||||
resource_id=request.resource_id,
|
||||
profile_kind="server_envelope",
|
||||
profile_id=request.profile_id,
|
||||
provider_id="test",
|
||||
vault_id=request.vault_id,
|
||||
key_version=1,
|
||||
algorithm_suite="AES-256-GCM",
|
||||
ciphertext_ref=request.ciphertext_ref,
|
||||
ciphertext_digest=f"sha256:{hashlib.sha256(ciphertext).hexdigest()}",
|
||||
authenticated_context_digest="sha256:context",
|
||||
state="active",
|
||||
created_at=datetime.now(tz=UTC),
|
||||
wrapped_key_refs=("wrapped:1",),
|
||||
)
|
||||
return ProtectedContent(envelope=envelope, ciphertext=ciphertext)
|
||||
|
||||
def unprotect_content(self, _session, *, request):
|
||||
if not request.ciphertext.startswith(b"protected:"):
|
||||
raise ValueError("invalid fixture ciphertext")
|
||||
return request.ciphertext.removeprefix(b"protected:")
|
||||
|
||||
def execute_rewrap(self, *_args, **_kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
def prepare_reencryption(self, *_args, **_kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class Backend:
|
||||
def __init__(self, data: bytes) -> None:
|
||||
self.data = data
|
||||
|
||||
def get_bytes(self, _key: str) -> bytes:
|
||||
return self.data
|
||||
|
||||
|
||||
class FileContentProtectionTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.engine = create_engine("sqlite+pysqlite:///:memory:")
|
||||
FileBlob.__table__.create(self.engine)
|
||||
self.session = Session(self.engine)
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self.session.close()
|
||||
self.engine.dispose()
|
||||
|
||||
def test_encrypted_blob_is_opened_after_stored_object_integrity_check(self) -> None:
|
||||
configure_runtime(registry=Registry(Cipher()), settings=object())
|
||||
plaintext = b"monthly records"
|
||||
protected = protect_blob_content(
|
||||
self.session,
|
||||
tenant_id="tenant-1",
|
||||
blob_id="blob-1",
|
||||
vault_id="vault-1",
|
||||
ciphertext_ref="tenants/tenant-1/files/blob-1",
|
||||
plaintext=plaintext,
|
||||
actor_id="account-1",
|
||||
content_type="text/plain",
|
||||
)
|
||||
blob = FileBlob(
|
||||
id="blob-1",
|
||||
tenant_id="tenant-1",
|
||||
storage_backend="test",
|
||||
storage_key="tenants/tenant-1/files/blob-1",
|
||||
checksum_sha256=hashlib.sha256(plaintext).hexdigest(),
|
||||
size_bytes=len(plaintext),
|
||||
protection_discriminator="vault:vault-1",
|
||||
encryption_envelope_id=protected.envelope.envelope_id,
|
||||
storage_checksum_sha256=hashlib.sha256(protected.ciphertext).hexdigest(),
|
||||
storage_size_bytes=len(protected.ciphertext),
|
||||
content_type="text/plain",
|
||||
ref_count=1,
|
||||
integrity_status="verified",
|
||||
)
|
||||
self.session.add(blob)
|
||||
self.session.flush()
|
||||
self.assertEqual(
|
||||
plaintext,
|
||||
read_verified_blob_bytes(blob, backend=Backend(protected.ciphertext)),
|
||||
)
|
||||
|
||||
def test_encrypted_blob_fails_closed_without_encryption_capability(self) -> None:
|
||||
configure_runtime(registry=Registry(), settings=object())
|
||||
ciphertext = b"protected:monthly records"
|
||||
blob = FileBlob(
|
||||
id="blob-1",
|
||||
tenant_id="tenant-1",
|
||||
storage_backend="test",
|
||||
storage_key="tenants/tenant-1/files/blob-1",
|
||||
checksum_sha256=hashlib.sha256(b"monthly records").hexdigest(),
|
||||
size_bytes=len(b"monthly records"),
|
||||
protection_discriminator="vault:vault-1",
|
||||
encryption_envelope_id="envelope-file-1",
|
||||
storage_checksum_sha256=hashlib.sha256(ciphertext).hexdigest(),
|
||||
storage_size_bytes=len(ciphertext),
|
||||
ref_count=1,
|
||||
integrity_status="verified",
|
||||
)
|
||||
self.session.add(blob)
|
||||
self.session.flush()
|
||||
with self.assertRaisesRegex(FileStorageError, "Encryption is unavailable"):
|
||||
read_verified_blob_bytes(blob, backend=Backend(ciphertext))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -10,6 +10,7 @@ STATIC_TOPIC_IDS = {
|
||||
"files.workflow.delete-managed-files",
|
||||
"files.governed-connectors-and-provenance",
|
||||
"files.reference.integrity-recovery-and-fail-closed-transports",
|
||||
"files.reference.shared-storage-profile",
|
||||
"files.reference.snapshot-provenance-and-capabilities",
|
||||
"files.assurance.process-and-release-readiness",
|
||||
}
|
||||
@@ -157,6 +158,7 @@ class FilesManifestDocumentationTests(unittest.TestCase):
|
||||
self.assertIn("bounded resumable integrity scan", topic.body)
|
||||
self.assertIn("quarantined", topic.body)
|
||||
self.assertIn("MASTER_KEY_B64", topic.metadata["recovery_unit"])
|
||||
self.assertIn("Encryption envelope and wrapped-key rows", topic.metadata["recovery_unit"])
|
||||
self.assertTrue(topic.metadata["verification"])
|
||||
self.assertIn(
|
||||
"/api/v1/files/integrity/scans",
|
||||
|
||||
Reference in New Issue
Block a user