GovOPlaN/govoplan-files
2
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

Add governed Files integrity operations UI

Browse Source
This commit is contained in:
zemion
2026-08-04 01:04:39 +02:00
parent 04882f1628
commit 7e6be4b017
15 changed files with 841 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/govoplan_files/backend/db/models.py
+2
View File
@@ -54,6 +54,7 @@ class FileIntegrityScan(Base, TimestampMixin):
storage_backend: Mapped[str] = mapped_column(String(50), nullable=False)
storage_prefix: Mapped[str] = mapped_column(String(1000), nullable=False)
status: Mapped[str] = mapped_column(String(30), default="pending", nullable=False, index=True)
revision: Mapped[int] = mapped_column(Integer, default=1, nullable=False)
phase: Mapped[str] = mapped_column(String(30), default="blobs", nullable=False)
verify_checksums: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False)
batch_size: Mapped[int] = mapped_column(Integer, default=100, nullable=False)
@@ -81,6 +82,7 @@ class FileIntegrityFinding(Base, TimestampMixin):
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
kind: Mapped[str] = mapped_column(String(40), nullable=False, index=True)
state: Mapped[str] = mapped_column(String(30), default="open", nullable=False, index=True)
revision: Mapped[int] = mapped_column(Integer, default=1, nullable=False)
blob_id: Mapped[str | None] = mapped_column(ForeignKey("file_blobs.id", ondelete="SET NULL"), nullable=True, index=True)
storage_key: Mapped[str] = mapped_column(String(1000), nullable=False)
expected_size_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True)
src/govoplan_files/backend/manifest.py
+3 -1
View File
@@ -321,6 +321,7 @@ manifest = ModuleManifest(
view_surfaces=(
ViewSurface(id="files.admin.system-connectors", module_id="files", kind="section", label="System file connections", order=75),
ViewSurface(id="files.admin.tenant-connectors", module_id="files", kind="section", label="Tenant file connections", order=65),
ViewSurface(id="files.admin.tenant-integrity", module_id="files", kind="section", label="File integrity", order=66),
ViewSurface(id="files.admin.group-connectors", module_id="files", kind="section", label="Group file connections", order=65),
ViewSurface(id="files.admin.user-connectors", module_id="files", kind="section", label="User file connections", order=65),
ViewSurface(id="files.settings.connectors", module_id="files", kind="section", label="Personal file connections", order=20),
@@ -588,7 +589,7 @@ manifest = ModuleManifest(
title="Operate Files integrity, recovery, and connector transport safety",
summary="Back up database evidence, blob ciphertext, and Encryption custody as one recovery unit, and keep unsupported SDK transports fail-closed.",
body=(
"Local durable storage is the operational baseline. Recover Files from a coordinated database/blob snapshot with the matching Encryption tables and original deployment master key, then run the bounded resumable integrity scan and verify representative protected and unprotected access paths. Protected scans verify stored ciphertext before decryption and then verify plaintext semantic evidence. Managed blob creation/repair and applied orphan cleanup commit lease-fenced Core recovery intent before object effects; success, compensation, and forward completion require independent database and object checks, while mismatch is quarantined and unresolved work remains visible in Ops. Missing or mismatched blobs are quarantined; orphan objects are reported before dry-run-first, explicitly authorized cleanup. "
"Local durable storage is the operational baseline. Recover Files from a coordinated database/blob snapshot with the matching Encryption tables and original deployment master key, then run the bounded resumable integrity scan from Administration and verify representative protected and unprotected access paths. Each scan batch and finding action requires the revision shown to the operator, so a stale screen cannot recheck or delete after concurrent reconciliation. Protected scans verify stored ciphertext before decryption and then verify plaintext semantic evidence. Managed blob creation/repair and applied orphan cleanup commit lease-fenced Core recovery intent before object effects; success, compensation, and forward completion require independent database and object checks, while mismatch is quarantined and unresolved work remains visible in Ops. Missing or mismatched blobs are quarantined; orphan objects are reported before dry-run-first, explicitly authorized cleanup. "
"Arbitrary external S3 managed storage/connectors and SMB connectors fail closed until botocore redirects/endpoint discovery and SMB initial connections/DFS referrals support connection-time DNS/IP pinning. Installer-owned Garage storage is supported only at the exact deployment service endpoint with its explicit trust marker. Destructive module retirement drops database tables but does not remove backend blob objects."
),
layer="configured",
@@ -608,6 +609,7 @@ manifest = ModuleManifest(
),
links=(
DocumentationLink(label="System file connections", href="/admin?section=system-file-connectors", kind="runtime"),
DocumentationLink(label="File integrity operations", href="/admin?section=tenant-file-integrity", kind="runtime"),
DocumentationLink(label="Connector provider status", href="/api/v1/files/connectors/providers", kind="api"),
DocumentationLink(label="Create an integrity scan", href="/api/v1/files/integrity/scans", kind="api"),
DocumentationLink(label="Files handbook", href="govoplan-files/docs/FILES_HANDBOOK.md", kind="repository"),
src/govoplan_files/backend/migrations/versions/f1a2b3c4d5e7_file_integrity_action_revisions.py
+34
View File
@@ -0,0 +1,34 @@
"""add stale-action revisions to Files integrity operations
Revision ID: f1a2b3c4d5e7
Revises: d0e1f2a3b4c6
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
revision = "f1a2b3c4d5e7"
down_revision = "d0e1f2a3b4c6"
branch_labels = None
depends_on = None
def upgrade() -> None:
with op.batch_alter_table("file_integrity_scans") as batch_op:
batch_op.add_column(
sa.Column("revision", sa.Integer(), nullable=False, server_default="1")
)
with op.batch_alter_table("file_integrity_findings") as batch_op:
batch_op.add_column(
sa.Column("revision", sa.Integer(), nullable=False, server_default="1")
)
def downgrade() -> None:
with op.batch_alter_table("file_integrity_findings") as batch_op:
batch_op.drop_column("revision")
with op.batch_alter_table("file_integrity_scans") as batch_op:
batch_op.drop_column("revision")
src/govoplan_files/backend/routes/integrity.py
+56 -6
View File
@@ -18,6 +18,7 @@ from govoplan_files.backend.schemas import (
FileIntegrityFindingResponse,
FileIntegrityFindingsResponse,
FileIntegrityScanCreateRequest,
FileIntegrityScanRunRequest,
FileIntegrityScanResponse,
FileIntegrityScansResponse,
)
@@ -93,10 +94,17 @@ def create_scan(
@router.post("/scans/{scan_id}/run", response_model=FileIntegrityScanResponse)
def run_scan_batch(
scan_id: str,
payload: FileIntegrityScanRunRequest,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(require_scope("files:file:admin")),
) -> FileIntegrityScanResponse:
scan = _scan_for_tenant(session, scan_id, principal.tenant_id)
scan = _scan_for_tenant(
session,
scan_id,
principal.tenant_id,
for_update=True,
)
_assert_expected_revision(scan.revision, payload.expected_revision)
previous_status = scan.status
try:
run_integrity_scan_batch(session, scan)
@@ -117,7 +125,12 @@ def run_scan_batch(
return _scan_response(scan)
except (FileStorageError, StorageBackendError) as exc:
session.rollback()
scan = _scan_for_tenant(session, scan_id, principal.tenant_id)
scan = _scan_for_tenant(
session,
scan_id,
principal.tenant_id,
for_update=True,
)
mark_integrity_scan_failed(scan, error=exc)
audit_from_principal(
session,
@@ -172,7 +185,13 @@ def recheck_finding(
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(require_scope("files:file:admin")),
) -> FileIntegrityActionResponse:
finding = _finding_for_tenant(session, finding_id, principal.tenant_id)
finding = _finding_for_tenant(
session,
finding_id,
principal.tenant_id,
for_update=True,
)
_assert_expected_revision(finding.revision, payload.expected_revision)
try:
result = recheck_integrity_finding(
session,
@@ -198,7 +217,13 @@ def cleanup_finding(
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(require_scope("files:file:admin")),
) -> FileIntegrityActionResponse:
finding = _finding_for_tenant(session, finding_id, principal.tenant_id)
finding = _finding_for_tenant(
session,
finding_id,
principal.tenant_id,
for_update=True,
)
_assert_expected_revision(finding.revision, payload.expected_revision)
try:
result = cleanup_orphan_finding(
session,
@@ -218,8 +243,13 @@ def _scan_for_tenant(
session: Session,
scan_id: str,
tenant_id: str,
*,
for_update: bool = False,
) -> FileIntegrityScan:
scan = session.get(FileIntegrityScan, scan_id)
query = session.query(FileIntegrityScan).filter(FileIntegrityScan.id == scan_id)
if for_update:
query = query.populate_existing().with_for_update()
scan = query.one_or_none()
if scan is None or scan.tenant_id != tenant_id:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
@@ -232,8 +262,15 @@ def _finding_for_tenant(
session: Session,
finding_id: str,
tenant_id: str,
*,
for_update: bool = False,
) -> FileIntegrityFinding:
finding = session.get(FileIntegrityFinding, finding_id)
query = session.query(FileIntegrityFinding).filter(
FileIntegrityFinding.id == finding_id
)
if for_update:
query = query.populate_existing().with_for_update()
finding = query.one_or_none()
if finding is None or finding.tenant_id != tenant_id:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
@@ -242,6 +279,17 @@ def _finding_for_tenant(
return finding
def _assert_expected_revision(current: int, expected: int) -> None:
if current != expected:
raise HTTPException(
status_code=status.HTTP_409_CONFLICT,
detail=(
"The integrity record changed after it was loaded; reload before "
"performing this action."
),
)
def _audit_integrity_action(session, principal, result) -> None:
audit_from_principal(
session,
@@ -271,6 +319,7 @@ def _scan_response(scan: FileIntegrityScan) -> FileIntegrityScanResponse:
storage_backend=scan.storage_backend,
storage_prefix=scan.storage_prefix,
status=scan.status,
revision=scan.revision,
phase=scan.phase,
verify_checksums=scan.verify_checksums,
batch_size=scan.batch_size,
@@ -299,6 +348,7 @@ def _finding_response(
tenant_id=finding.tenant_id,
kind=finding.kind,
state=finding.state,
revision=finding.revision,
blob_id=finding.blob_id,
storage_key=finding.storage_key,
expected_size_bytes=finding.expected_size_bytes,
src/govoplan_files/backend/schemas.py
+7
View File
@@ -101,6 +101,7 @@ class FileIntegrityScanResponse(BaseModel):
storage_backend: str
storage_prefix: str
status: str
revision: int
phase: str
verify_checksums: bool
batch_size: int
@@ -127,6 +128,7 @@ class FileIntegrityFindingResponse(BaseModel):
tenant_id: str
kind: str
state: str
revision: int
blob_id: str | None = None
storage_key: str
expected_size_bytes: int | None = None
@@ -145,6 +147,11 @@ class FileIntegrityFindingsResponse(BaseModel):
class FileIntegrityActionRequest(BaseModel):
dry_run: bool = True
expected_revision: int = Field(ge=1)
class FileIntegrityScanRunRequest(BaseModel):
expected_revision: int = Field(ge=1)
class FileIntegrityActionResponse(BaseModel):
src/govoplan_files/backend/storage/integrity.py
+8
View File
@@ -99,6 +99,7 @@ def run_integrity_scan_batch(
scan.phase = "completed"
scan.status = "completed"
scan.completed_at = utcnow()
scan.revision += 1
session.add(scan)
return scan
@@ -110,6 +111,7 @@ def mark_integrity_scan_failed(
) -> None:
scan.status = "failed"
scan.last_error = type(error).__name__[:255]
scan.revision += 1
def inspect_blob(
@@ -269,6 +271,7 @@ def recheck_integrity_finding(
finding.resolved_at = utcnow()
finding.resolved_by_user_id = user_id
session.add(finding)
finding.revision += 1
changed = previous != (
blob.integrity_status,
blob.quarantined_at,
@@ -312,6 +315,7 @@ def cleanup_orphan_finding(
finding.state = "resolved"
finding.resolved_at = utcnow()
finding.resolved_by_user_id = user_id
finding.revision += 1
session.add(finding)
return IntegrityActionResult(
action="retained_referenced",
@@ -360,6 +364,7 @@ def cleanup_orphan_finding(
finding.state = "deleted"
finding.resolved_at = utcnow()
finding.resolved_by_user_id = user_id
finding.revision += 1
session.add(finding)
return IntegrityActionResult(
action=action,
@@ -465,6 +470,8 @@ def _record_blob_finding(
expected_size_bytes=blob.storage_size_bytes if blob.storage_size_bytes is not None else blob.size_bytes,
expected_checksum_sha256=blob.storage_checksum_sha256 if blob.storage_checksum_sha256 is not None else blob.checksum_sha256,
)
else:
finding.revision += 1
_update_finding_from_inspection(finding, inspection)
session.add(finding)
return finding
@@ -514,6 +521,7 @@ def _resolve_scan_blob_findings(
):
finding.state = "resolved"
finding.resolved_at = utcnow()
finding.revision += 1
session.add(finding)