diff --git a/README.md b/README.md index 60b55eb..23c2950 100644 --- a/README.md +++ b/README.md @@ -166,6 +166,12 @@ database and streamed object evidence, while rollback compensates only a newly reserved unreferenced key. New object keys are opaque and do not retain the uploaded filename. Uncertain or mismatched effects remain visible through Ops. +Operators with `files:file:admin` can run bounded, resumable integrity scans in +Administration. Scan batches and finding actions carry monotonic revisions; +stale resume, recheck, or cleanup requests fail before touching object storage. +Orphan cleanup always requires a dry-run preview followed by separate +confirmation and records recovery-ledger evidence. + Bulk rename and transfer APIs are owner-scoped: callers must provide the active user or group file space with `owner_type` and `owner_id`. The storage layer keeps a named legacy file-only helper for historical callers that lack owner diff --git a/docs/FILES_HANDBOOK.md b/docs/FILES_HANDBOOK.md index b25bece..7de475b 100644 --- a/docs/FILES_HANDBOOK.md +++ b/docs/FILES_HANDBOOK.md @@ -509,10 +509,13 @@ snapshots so database references and objects represent the same recovery point. The integrity API verifies a restored set, but it does not replace a coordinated backup. -Create a scan with `POST /api/v1/files/integrity/scans`, then call -`POST /api/v1/files/integrity/scans/{scan_id}/run` until it reports -`completed`. Each call advances at most the persisted batch size, so a stopped -operator or worker can resume from the committed blob/object cursors. +Operators normally use **Administration > File integrity**. The equivalent API +creates a scan with `POST /api/v1/files/integrity/scans`, then calls +`POST /api/v1/files/integrity/scans/{scan_id}/run` with the scan's current +`expected_revision` until it reports `completed`. Each call advances at most +the persisted batch size, so a stopped operator or worker can resume from the +committed blob/object cursors. Concurrent or stale actions receive `409` before +the storage backend is invoked; reload the scan and inspect the newer state. Findings distinguish: @@ -524,10 +527,14 @@ Findings distinguish: Missing or corrupt blobs fail closed for ordinary downloads and Campaign attachment materialization. After restoring the expected bytes, use the finding -`recheck` action first in dry-run mode and then apply it. Orphan cleanup is also -dry-run by default, rechecks that no database reference exists, remains scoped -to the scanned tenant prefix, and is idempotent. Both applied and dry-run -actions emit audit evidence. +`recheck` action with its current `expected_revision`. Orphan cleanup starts +with a dry-run preview and requires separate destructive confirmation. The +confirmation reuses the finding revision from that preview, rechecks that no +database reference exists, remains scoped to the scanned tenant prefix, and is +idempotent. Both applied and dry-run actions emit audit evidence. A shared +reference blocks deletion. Files currently has no legal-hold or hard-purge +model, so retention-controlled objects must not be treated as cleanup +candidates until those controls are implemented. ### Recovery ledger for object effects diff --git a/src/govoplan_files/backend/db/models.py b/src/govoplan_files/backend/db/models.py index c9165ab..bb3feed 100644 --- a/src/govoplan_files/backend/db/models.py +++ b/src/govoplan_files/backend/db/models.py @@ -54,6 +54,7 @@ class FileIntegrityScan(Base, TimestampMixin): storage_backend: Mapped[str] = mapped_column(String(50), nullable=False) storage_prefix: Mapped[str] = mapped_column(String(1000), nullable=False) status: Mapped[str] = mapped_column(String(30), default="pending", nullable=False, index=True) + revision: Mapped[int] = mapped_column(Integer, default=1, nullable=False) phase: Mapped[str] = mapped_column(String(30), default="blobs", nullable=False) verify_checksums: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False) batch_size: Mapped[int] = mapped_column(Integer, default=100, nullable=False) @@ -81,6 +82,7 @@ class FileIntegrityFinding(Base, TimestampMixin): tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True) kind: Mapped[str] = mapped_column(String(40), nullable=False, index=True) state: Mapped[str] = mapped_column(String(30), default="open", nullable=False, index=True) + revision: Mapped[int] = mapped_column(Integer, default=1, nullable=False) blob_id: Mapped[str | None] = mapped_column(ForeignKey("file_blobs.id", ondelete="SET NULL"), nullable=True, index=True) storage_key: Mapped[str] = mapped_column(String(1000), nullable=False) expected_size_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True) diff --git a/src/govoplan_files/backend/manifest.py b/src/govoplan_files/backend/manifest.py index 88a2f55..7bf1781 100644 --- a/src/govoplan_files/backend/manifest.py +++ b/src/govoplan_files/backend/manifest.py @@ -321,6 +321,7 @@ manifest = ModuleManifest( view_surfaces=( ViewSurface(id="files.admin.system-connectors", module_id="files", kind="section", label="System file connections", order=75), ViewSurface(id="files.admin.tenant-connectors", module_id="files", kind="section", label="Tenant file connections", order=65), + ViewSurface(id="files.admin.tenant-integrity", module_id="files", kind="section", label="File integrity", order=66), ViewSurface(id="files.admin.group-connectors", module_id="files", kind="section", label="Group file connections", order=65), ViewSurface(id="files.admin.user-connectors", module_id="files", kind="section", label="User file connections", order=65), ViewSurface(id="files.settings.connectors", module_id="files", kind="section", label="Personal file connections", order=20), @@ -588,7 +589,7 @@ manifest = ModuleManifest( title="Operate Files integrity, recovery, and connector transport safety", summary="Back up database evidence, blob ciphertext, and Encryption custody as one recovery unit, and keep unsupported SDK transports fail-closed.", body=( - "Local durable storage is the operational baseline. Recover Files from a coordinated database/blob snapshot with the matching Encryption tables and original deployment master key, then run the bounded resumable integrity scan and verify representative protected and unprotected access paths. Protected scans verify stored ciphertext before decryption and then verify plaintext semantic evidence. Managed blob creation/repair and applied orphan cleanup commit lease-fenced Core recovery intent before object effects; success, compensation, and forward completion require independent database and object checks, while mismatch is quarantined and unresolved work remains visible in Ops. Missing or mismatched blobs are quarantined; orphan objects are reported before dry-run-first, explicitly authorized cleanup. " + "Local durable storage is the operational baseline. Recover Files from a coordinated database/blob snapshot with the matching Encryption tables and original deployment master key, then run the bounded resumable integrity scan from Administration and verify representative protected and unprotected access paths. Each scan batch and finding action requires the revision shown to the operator, so a stale screen cannot recheck or delete after concurrent reconciliation. Protected scans verify stored ciphertext before decryption and then verify plaintext semantic evidence. Managed blob creation/repair and applied orphan cleanup commit lease-fenced Core recovery intent before object effects; success, compensation, and forward completion require independent database and object checks, while mismatch is quarantined and unresolved work remains visible in Ops. Missing or mismatched blobs are quarantined; orphan objects are reported before dry-run-first, explicitly authorized cleanup. " "Arbitrary external S3 managed storage/connectors and SMB connectors fail closed until botocore redirects/endpoint discovery and SMB initial connections/DFS referrals support connection-time DNS/IP pinning. Installer-owned Garage storage is supported only at the exact deployment service endpoint with its explicit trust marker. Destructive module retirement drops database tables but does not remove backend blob objects." ), layer="configured", @@ -608,6 +609,7 @@ manifest = ModuleManifest( ), links=( DocumentationLink(label="System file connections", href="/admin?section=system-file-connectors", kind="runtime"), + DocumentationLink(label="File integrity operations", href="/admin?section=tenant-file-integrity", kind="runtime"), DocumentationLink(label="Connector provider status", href="/api/v1/files/connectors/providers", kind="api"), DocumentationLink(label="Create an integrity scan", href="/api/v1/files/integrity/scans", kind="api"), DocumentationLink(label="Files handbook", href="govoplan-files/docs/FILES_HANDBOOK.md", kind="repository"), diff --git a/src/govoplan_files/backend/migrations/versions/f1a2b3c4d5e7_file_integrity_action_revisions.py b/src/govoplan_files/backend/migrations/versions/f1a2b3c4d5e7_file_integrity_action_revisions.py new file mode 100644 index 0000000..9954859 --- /dev/null +++ b/src/govoplan_files/backend/migrations/versions/f1a2b3c4d5e7_file_integrity_action_revisions.py @@ -0,0 +1,34 @@ +"""add stale-action revisions to Files integrity operations + +Revision ID: f1a2b3c4d5e7 +Revises: d0e1f2a3b4c6 +""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op + + +revision = "f1a2b3c4d5e7" +down_revision = "d0e1f2a3b4c6" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + with op.batch_alter_table("file_integrity_scans") as batch_op: + batch_op.add_column( + sa.Column("revision", sa.Integer(), nullable=False, server_default="1") + ) + with op.batch_alter_table("file_integrity_findings") as batch_op: + batch_op.add_column( + sa.Column("revision", sa.Integer(), nullable=False, server_default="1") + ) + + +def downgrade() -> None: + with op.batch_alter_table("file_integrity_findings") as batch_op: + batch_op.drop_column("revision") + with op.batch_alter_table("file_integrity_scans") as batch_op: + batch_op.drop_column("revision") diff --git a/src/govoplan_files/backend/routes/integrity.py b/src/govoplan_files/backend/routes/integrity.py index 78641b0..f711d62 100644 --- a/src/govoplan_files/backend/routes/integrity.py +++ b/src/govoplan_files/backend/routes/integrity.py @@ -18,6 +18,7 @@ from govoplan_files.backend.schemas import ( FileIntegrityFindingResponse, FileIntegrityFindingsResponse, FileIntegrityScanCreateRequest, + FileIntegrityScanRunRequest, FileIntegrityScanResponse, FileIntegrityScansResponse, ) @@ -93,10 +94,17 @@ def create_scan( @router.post("/scans/{scan_id}/run", response_model=FileIntegrityScanResponse) def run_scan_batch( scan_id: str, + payload: FileIntegrityScanRunRequest, session: Session = Depends(get_session), principal: ApiPrincipal = Depends(require_scope("files:file:admin")), ) -> FileIntegrityScanResponse: - scan = _scan_for_tenant(session, scan_id, principal.tenant_id) + scan = _scan_for_tenant( + session, + scan_id, + principal.tenant_id, + for_update=True, + ) + _assert_expected_revision(scan.revision, payload.expected_revision) previous_status = scan.status try: run_integrity_scan_batch(session, scan) @@ -117,7 +125,12 @@ def run_scan_batch( return _scan_response(scan) except (FileStorageError, StorageBackendError) as exc: session.rollback() - scan = _scan_for_tenant(session, scan_id, principal.tenant_id) + scan = _scan_for_tenant( + session, + scan_id, + principal.tenant_id, + for_update=True, + ) mark_integrity_scan_failed(scan, error=exc) audit_from_principal( session, @@ -172,7 +185,13 @@ def recheck_finding( session: Session = Depends(get_session), principal: ApiPrincipal = Depends(require_scope("files:file:admin")), ) -> FileIntegrityActionResponse: - finding = _finding_for_tenant(session, finding_id, principal.tenant_id) + finding = _finding_for_tenant( + session, + finding_id, + principal.tenant_id, + for_update=True, + ) + _assert_expected_revision(finding.revision, payload.expected_revision) try: result = recheck_integrity_finding( session, @@ -198,7 +217,13 @@ def cleanup_finding( session: Session = Depends(get_session), principal: ApiPrincipal = Depends(require_scope("files:file:admin")), ) -> FileIntegrityActionResponse: - finding = _finding_for_tenant(session, finding_id, principal.tenant_id) + finding = _finding_for_tenant( + session, + finding_id, + principal.tenant_id, + for_update=True, + ) + _assert_expected_revision(finding.revision, payload.expected_revision) try: result = cleanup_orphan_finding( session, @@ -218,8 +243,13 @@ def _scan_for_tenant( session: Session, scan_id: str, tenant_id: str, + *, + for_update: bool = False, ) -> FileIntegrityScan: - scan = session.get(FileIntegrityScan, scan_id) + query = session.query(FileIntegrityScan).filter(FileIntegrityScan.id == scan_id) + if for_update: + query = query.populate_existing().with_for_update() + scan = query.one_or_none() if scan is None or scan.tenant_id != tenant_id: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, @@ -232,8 +262,15 @@ def _finding_for_tenant( session: Session, finding_id: str, tenant_id: str, + *, + for_update: bool = False, ) -> FileIntegrityFinding: - finding = session.get(FileIntegrityFinding, finding_id) + query = session.query(FileIntegrityFinding).filter( + FileIntegrityFinding.id == finding_id + ) + if for_update: + query = query.populate_existing().with_for_update() + finding = query.one_or_none() if finding is None or finding.tenant_id != tenant_id: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, @@ -242,6 +279,17 @@ def _finding_for_tenant( return finding +def _assert_expected_revision(current: int, expected: int) -> None: + if current != expected: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + "The integrity record changed after it was loaded; reload before " + "performing this action." + ), + ) + + def _audit_integrity_action(session, principal, result) -> None: audit_from_principal( session, @@ -271,6 +319,7 @@ def _scan_response(scan: FileIntegrityScan) -> FileIntegrityScanResponse: storage_backend=scan.storage_backend, storage_prefix=scan.storage_prefix, status=scan.status, + revision=scan.revision, phase=scan.phase, verify_checksums=scan.verify_checksums, batch_size=scan.batch_size, @@ -299,6 +348,7 @@ def _finding_response( tenant_id=finding.tenant_id, kind=finding.kind, state=finding.state, + revision=finding.revision, blob_id=finding.blob_id, storage_key=finding.storage_key, expected_size_bytes=finding.expected_size_bytes, diff --git a/src/govoplan_files/backend/schemas.py b/src/govoplan_files/backend/schemas.py index 1701a2f..487d8cf 100644 --- a/src/govoplan_files/backend/schemas.py +++ b/src/govoplan_files/backend/schemas.py @@ -101,6 +101,7 @@ class FileIntegrityScanResponse(BaseModel): storage_backend: str storage_prefix: str status: str + revision: int phase: str verify_checksums: bool batch_size: int @@ -127,6 +128,7 @@ class FileIntegrityFindingResponse(BaseModel): tenant_id: str kind: str state: str + revision: int blob_id: str | None = None storage_key: str expected_size_bytes: int | None = None @@ -145,6 +147,11 @@ class FileIntegrityFindingsResponse(BaseModel): class FileIntegrityActionRequest(BaseModel): dry_run: bool = True + expected_revision: int = Field(ge=1) + + +class FileIntegrityScanRunRequest(BaseModel): + expected_revision: int = Field(ge=1) class FileIntegrityActionResponse(BaseModel): diff --git a/src/govoplan_files/backend/storage/integrity.py b/src/govoplan_files/backend/storage/integrity.py index c588912..d47f28a 100644 --- a/src/govoplan_files/backend/storage/integrity.py +++ b/src/govoplan_files/backend/storage/integrity.py @@ -99,6 +99,7 @@ def run_integrity_scan_batch( scan.phase = "completed" scan.status = "completed" scan.completed_at = utcnow() + scan.revision += 1 session.add(scan) return scan @@ -110,6 +111,7 @@ def mark_integrity_scan_failed( ) -> None: scan.status = "failed" scan.last_error = type(error).__name__[:255] + scan.revision += 1 def inspect_blob( @@ -269,6 +271,7 @@ def recheck_integrity_finding( finding.resolved_at = utcnow() finding.resolved_by_user_id = user_id session.add(finding) + finding.revision += 1 changed = previous != ( blob.integrity_status, blob.quarantined_at, @@ -312,6 +315,7 @@ def cleanup_orphan_finding( finding.state = "resolved" finding.resolved_at = utcnow() finding.resolved_by_user_id = user_id + finding.revision += 1 session.add(finding) return IntegrityActionResult( action="retained_referenced", @@ -360,6 +364,7 @@ def cleanup_orphan_finding( finding.state = "deleted" finding.resolved_at = utcnow() finding.resolved_by_user_id = user_id + finding.revision += 1 session.add(finding) return IntegrityActionResult( action=action, @@ -465,6 +470,8 @@ def _record_blob_finding( expected_size_bytes=blob.storage_size_bytes if blob.storage_size_bytes is not None else blob.size_bytes, expected_checksum_sha256=blob.storage_checksum_sha256 if blob.storage_checksum_sha256 is not None else blob.checksum_sha256, ) + else: + finding.revision += 1 _update_finding_from_inspection(finding, inspection) session.add(finding) return finding @@ -514,6 +521,7 @@ def _resolve_scan_blob_findings( ): finding.state = "resolved" finding.resolved_at = utcnow() + finding.revision += 1 session.add(finding) diff --git a/tests/test_integrity_reconciliation.py b/tests/test_integrity_reconciliation.py index 67db99c..daae563 100644 --- a/tests/test_integrity_reconciliation.py +++ b/tests/test_integrity_reconciliation.py @@ -6,6 +6,7 @@ import unittest from pathlib import Path from unittest.mock import patch +from fastapi import HTTPException from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker @@ -16,6 +17,7 @@ from govoplan_files.backend.db.models import ( FileIntegrityFinding, FileIntegrityScan, ) +from govoplan_files.backend.routes.integrity import _assert_expected_revision from govoplan_files.backend.storage.backends import LocalFilesystemStorageBackend from govoplan_files.backend.storage.common import FileStorageError from govoplan_files.backend.storage.integrity import ( @@ -82,6 +84,7 @@ class IntegrityReconciliationTests(unittest.TestCase): backend=self.backend, ) self.session.commit() + self.assertEqual(1, scan.revision) invocations = 0 while scan.status != "completed": @@ -99,6 +102,7 @@ class IntegrityReconciliationTests(unittest.TestCase): self.fail("Integrity scan did not complete") self.assertGreater(invocations, 3) + self.assertEqual(1 + invocations, scan.revision) self.assertEqual(3, scan.scanned_blob_count) self.assertEqual(1, scan.verified_blob_count) self.assertEqual(2, scan.quarantined_blob_count) @@ -183,6 +187,13 @@ class IntegrityReconciliationTests(unittest.TestCase): self.assertEqual("already_deleted", repeated.action) self.assertFalse(repeated.changed) + def test_stale_integrity_action_revision_is_rejected(self) -> None: + with self.assertRaises(HTTPException) as captured: + _assert_expected_revision(3, 2) + + self.assertEqual(409, captured.exception.status_code) + self.assertIn("reload", str(captured.exception.detail).lower()) + def _blob(blob_id: str, filename: str, expected_data: bytes) -> FileBlob: return FileBlob( diff --git a/tests/test_manifest_documentation.py b/tests/test_manifest_documentation.py index ef228bb..0800de7 100644 --- a/tests/test_manifest_documentation.py +++ b/tests/test_manifest_documentation.py @@ -175,6 +175,14 @@ class FilesManifestDocumentationTests(unittest.TestCase): "/api/v1/files/integrity/scans", {link.href for link in topic.links}, ) + self.assertIn( + "/admin?section=tenant-file-integrity", + {link.href for link in topic.links}, + ) + self.assertIn( + "files.admin.tenant-integrity", + {surface.id for surface in self.manifest.frontend.view_surfaces}, + ) self.assertIn( "GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS", topic.configuration_keys ) diff --git a/webui/scripts/test-interface-pattern-language.mjs b/webui/scripts/test-interface-pattern-language.mjs index aa5d839..695e391 100644 --- a/webui/scripts/test-interface-pattern-language.mjs +++ b/webui/scripts/test-interface-pattern-language.mjs @@ -7,6 +7,8 @@ function read(relativePath) { } const connector = read("../src/features/files/FileConnectorSettingsPanel.tsx"); +const integrity = read("../src/features/files/FileIntegrityPanel.tsx"); +const filesApi = read("../src/api/files.ts"); const filesPage = read("../src/features/files/FilesPage.tsx"); const moduleSource = read("../src/module.ts"); const styles = read("../src/styles/file-manager.css"); @@ -23,6 +25,14 @@ assert.match(connector, /(settings, "/api/v1/files/transfer", { method: "POST", body: JSON.stringify(payload) }); } +export async function listFileIntegrityScans( + settings: ApiSettings, + limit = 100 +): Promise { + const response = await apiFetch<{ scans: FileIntegrityScan[] }>( + settings, + `/api/v1/files/integrity/scans?limit=${Math.max(1, Math.min(limit, 200))}` + ); + return response.scans; +} + +export function createFileIntegrityScan( + settings: ApiSettings, + payload: { verify_checksums: boolean; batch_size: number } +): Promise { + return apiFetch(settings, "/api/v1/files/integrity/scans", { + method: "POST", + body: JSON.stringify(payload) + }); +} + +export function runFileIntegrityScanBatch( + settings: ApiSettings, + scan: Pick +): Promise { + return apiFetch( + settings, + `/api/v1/files/integrity/scans/${encodeURIComponent(scan.id)}/run`, + { + method: "POST", + body: JSON.stringify({ expected_revision: scan.revision }) + } + ); +} + +export async function listFileIntegrityFindings( + settings: ApiSettings, + scanId: string, + state?: string +): Promise { + const search = new URLSearchParams({ limit: "1000" }); + if (state) search.set("state", state); + const response = await apiFetch<{ findings: FileIntegrityFinding[] }>( + settings, + `/api/v1/files/integrity/scans/${encodeURIComponent(scanId)}/findings?${search}` + ); + return response.findings; +} + +export function recheckFileIntegrityFinding( + settings: ApiSettings, + finding: Pick, + dryRun = false +): Promise { + return apiFetch( + settings, + `/api/v1/files/integrity/findings/${encodeURIComponent(finding.id)}/recheck`, + { + method: "POST", + body: JSON.stringify({ + dry_run: dryRun, + expected_revision: finding.revision + }) + } + ); +} + +export function cleanupFileIntegrityFinding( + settings: ApiSettings, + finding: Pick, + dryRun: boolean +): Promise { + return apiFetch( + settings, + `/api/v1/files/integrity/findings/${encodeURIComponent(finding.id)}/cleanup`, + { + method: "POST", + body: JSON.stringify({ + dry_run: dryRun, + expected_revision: finding.revision + }) + } + ); +} + export async function downloadFile(settings: ApiSettings, file: ManagedFile): Promise { const response = await fetch(apiUrl(settings, `/api/v1/files/${file.id}/download`), { headers: authHeaders(settings), credentials: "include" }); if (!response.ok) throw new Error(`${response.status} ${response.statusText}: ${await response.text()}`); diff --git a/webui/src/features/files/FileIntegrityPanel.tsx b/webui/src/features/files/FileIntegrityPanel.tsx new file mode 100644 index 0000000..1814beb --- /dev/null +++ b/webui/src/features/files/FileIntegrityPanel.tsx @@ -0,0 +1,527 @@ +import { useEffect, useMemo, useState } from "react"; +import { + AdminPageLayout, + adminErrorMessage, + Button, + Card, + ConfirmDialog, + DataGrid, + Dialog, + DismissibleAlert, + DocumentationHelpLink, + FormField, + MetricCard, + StatusBadge, + TableActionGroup, + ToggleSwitch, + type ApiSettings, + type DataGridColumn +} from "@govoplan/core-webui"; +import { + CheckCircle2, + Eye, + Play, + Plus, + RefreshCw, + RotateCw, + Trash2 +} from "lucide-react"; +import { + cleanupFileIntegrityFinding, + createFileIntegrityScan, + listFileIntegrityFindings, + listFileIntegrityScans, + recheckFileIntegrityFinding, + runFileIntegrityScanBatch, + type FileIntegrityActionResult, + type FileIntegrityFinding, + type FileIntegrityScan +} from "../../api/files"; + +type Props = { + settings: ApiSettings; + canWrite: boolean; +}; + +const DOCUMENTATION = { + topicId: "files.reference.integrity-recovery-and-fail-closed-transports", + documentationType: "admin" as const +}; + +export default function FileIntegrityPanel({ settings, canWrite }: Props) { + const [scans, setScans] = useState([]); + const [selectedScanId, setSelectedScanId] = useState(""); + const [findings, setFindings] = useState([]); + const [loading, setLoading] = useState(true); + const [busy, setBusy] = useState(false); + const [error, setError] = useState(""); + const [success, setSuccess] = useState(""); + const [createOpen, setCreateOpen] = useState(false); + const [verifyChecksums, setVerifyChecksums] = useState(true); + const [batchSize, setBatchSize] = useState(100); + const [cleanupPreview, setCleanupPreview] = useState(null); + + const selectedScan = scans.find((scan) => scan.id === selectedScanId) ?? null; + + useEffect(() => { + void loadScans(); + }, [settings.accessToken, settings.apiBaseUrl, settings.apiKey]); + + useEffect(() => { + if (!selectedScanId) { + setFindings([]); + return; + } + void loadFindings(selectedScanId); + }, [selectedScanId, settings.accessToken, settings.apiBaseUrl, settings.apiKey]); + + async function loadScans() { + setLoading(true); + setError(""); + try { + const loaded = await listFileIntegrityScans(settings); + setScans(loaded); + setSelectedScanId((current) => ( + loaded.some((scan) => scan.id === current) ? current : loaded[0]?.id ?? "" + )); + } catch (err) { + setError(adminErrorMessage(err)); + } finally { + setLoading(false); + } + } + + async function loadFindings(scanId: string) { + setError(""); + try { + setFindings(await listFileIntegrityFindings(settings, scanId)); + } catch (err) { + setError(adminErrorMessage(err)); + } + } + + function replaceScan(next: FileIntegrityScan) { + setScans((current) => [ + next, + ...current.filter((scan) => scan.id !== next.id) + ].sort((left, right) => right.created_at.localeCompare(left.created_at))); + } + + function replaceFinding(next: FileIntegrityFinding) { + setFindings((current) => current.map((finding) => ( + finding.id === next.id ? next : finding + ))); + } + + async function createScan() { + setBusy(true); + setError(""); + setSuccess(""); + try { + const scan = await createFileIntegrityScan(settings, { + verify_checksums: verifyChecksums, + batch_size: batchSize + }); + replaceScan(scan); + setSelectedScanId(scan.id); + setCreateOpen(false); + setSuccess("Integrity scan created. Run its first bounded batch when ready."); + } catch (err) { + setError(adminErrorMessage(err)); + } finally { + setBusy(false); + } + } + + async function runBatch(scan: FileIntegrityScan) { + setBusy(true); + setError(""); + setSuccess(""); + try { + const next = await runFileIntegrityScanBatch(settings, scan); + replaceScan(next); + setSelectedScanId(next.id); + await loadFindings(next.id); + setSuccess( + next.status === "completed" + ? "Integrity scan completed. Review and resolve every finding." + : `Completed one ${next.batch_size}-item batch; the scan can be resumed.` + ); + } catch (err) { + setError(adminErrorMessage(err)); + await loadScans(); + } finally { + setBusy(false); + } + } + + async function recheck(finding: FileIntegrityFinding) { + setBusy(true); + setError(""); + setSuccess(""); + try { + const result = await recheckFileIntegrityFinding(settings, finding); + replaceFinding(result.finding); + setSuccess( + result.inspection_valid + ? "The stored object now matches its recorded integrity evidence." + : "The object still fails integrity verification and remains quarantined." + ); + } catch (err) { + setError(adminErrorMessage(err)); + if (selectedScanId) await loadFindings(selectedScanId); + } finally { + setBusy(false); + } + } + + async function previewCleanup(finding: FileIntegrityFinding) { + setBusy(true); + setError(""); + setSuccess(""); + try { + const preview = await cleanupFileIntegrityFinding(settings, finding, true); + setCleanupPreview(preview); + if (preview.action !== "would_delete") { + replaceFinding(preview.finding); + setSuccess(cleanupActionMessage(preview.action)); + } + } catch (err) { + setError(adminErrorMessage(err)); + if (selectedScanId) await loadFindings(selectedScanId); + } finally { + setBusy(false); + } + } + + async function confirmCleanup() { + if (!cleanupPreview) return; + setBusy(true); + setError(""); + setSuccess(""); + try { + const result = await cleanupFileIntegrityFinding( + settings, + cleanupPreview.finding, + false + ); + replaceFinding(result.finding); + setCleanupPreview(null); + setSuccess(cleanupActionMessage(result.action)); + } catch (err) { + setCleanupPreview(null); + setError(adminErrorMessage(err)); + if (selectedScanId) await loadFindings(selectedScanId); + } finally { + setBusy(false); + } + } + + const scanColumns = useMemo[]>(() => [ + { + id: "created", + header: "Created", + width: 180, + minWidth: 150, + sortable: true, + value: (scan) => scan.created_at, + render: (scan) => formatDateTime(scan.created_at) + }, + { + id: "status", + header: "Status", + width: 130, + minWidth: 110, + sortable: true, + filterable: true, + filterType: "list", + value: (scan) => scan.status, + render: (scan) => + }, + { + id: "phase", + header: "Phase", + width: 110, + minWidth: 90, + value: (scan) => scan.phase + }, + { + id: "progress", + header: "Progress", + minWidth: 250, + resizable: true, + value: (scan) => `${scan.scanned_blob_count} blobs, ${scan.scanned_object_count} objects`, + render: (scan) => ( + + {scan.scanned_blob_count} blobs ({scan.verified_blob_count} verified, {scan.quarantined_blob_count} quarantined), {" "} + {scan.scanned_object_count} objects ({scan.orphan_object_count} orphaned) + + ) + }, + { + id: "backend", + header: "Storage", + width: 120, + minWidth: 100, + value: (scan) => scan.storage_backend + }, + { + id: "actions", + header: "Actions", + width: 105, + minWidth: 105, + sticky: "end", + align: "right", + render: (scan) => ( + , + onClick: () => setSelectedScanId(scan.id) + }, + { + id: "run", + label: scan.status === "failed" ? "Resume next batch" : "Run next batch", + icon: , + onClick: () => void runBatch(scan), + disabled: busy || ["completed", "cancelled"].includes(scan.status), + disabledReason: ["completed", "cancelled"].includes(scan.status) + ? "This scan is complete." + : undefined + } + ]} + /> + ) + } + ], [busy]); + + const findingColumns = useMemo[]>(() => [ + { + id: "kind", + header: "Finding", + width: 180, + minWidth: 145, + sortable: true, + filterable: true, + value: (finding) => finding.kind.replaceAll("_", " ") + }, + { + id: "state", + header: "State", + width: 115, + minWidth: 95, + sortable: true, + filterable: true, + value: (finding) => finding.state, + render: (finding) => + }, + { + id: "object", + header: "Affected object", + minWidth: 260, + resizable: true, + filterable: true, + value: (finding) => finding.storage_key, + render: (finding) => ( + + {finding.blob_id ? `Blob ${finding.blob_id}` : finding.storage_key} + + ) + }, + { + id: "evidence", + header: "Expected / observed", + minWidth: 230, + resizable: true, + value: (finding) => evidenceLabel(finding) + }, + { + id: "updated", + header: "Updated", + width: 175, + minWidth: 145, + sortable: true, + value: (finding) => finding.updated_at, + render: (finding) => formatDateTime(finding.updated_at) + }, + { + id: "actions", + header: "Actions", + width: 70, + minWidth: 70, + sticky: "end", + align: "right", + render: (finding) => ( + , + variant: "danger", + onClick: () => void previewCleanup(finding), + disabled: busy || finding.state === "deleted", + disabledReason: finding.state === "deleted" ? "The object is already absent." : undefined + }] + : [{ + id: "recheck", + label: "Recheck stored object", + icon: , + onClick: () => void recheck(finding), + disabled: busy + }]} + /> + ) + } + ], [busy]); + + return ( + <> + + + + + + )} + > + + Cleanup is available only for objects with no managed database reference and always starts with a dry-run preview. Files does not yet implement legal-hold or hard-purge policy; such objects must remain outside cleanup until those controls exist. + + + +
+ scan.id} + initialFit="container" + rowClassName={(scan) => scan.id === selectedScanId ? "is-selected" : undefined} + emptyText="No integrity scans have been created." + /> +
+ + + {selectedScan && ( + <> +
+ + + + finding.state === "open").length} tone={findings.some((finding) => finding.state === "open") ? "warning" : "good"} /> +
+ {selectedScan.last_error && ( + + The last batch failed with {selectedScan.last_error}. Verify storage availability, then resume from the recorded cursor. + + )} + +
+ finding.id} + initialFit="container" + emptyText={selectedScan.status === "completed" ? "No findings were recorded." : "No findings in completed batches yet."} + /> +
+ + + )} + + + setCreateOpen(false)} + closeDisabled={busy} + footer={( + <> + + + + )} + > +
+ + + setBatchSize(Math.max(1, Math.min(1000, Number(event.target.value) || 1)))} + disabled={busy} + /> + +
+ + + void confirmCleanup()} + onCancel={() => setCleanupPreview(null)} + /> + + ); +} + +function statusTone(value: string): string { + if (["completed", "resolved", "verified"].includes(value)) return "success"; + if (["failed", "deleted", "missing", "checksum_mismatch", "size_mismatch"].includes(value)) return "danger"; + if (["running", "pending", "open"].includes(value)) return "warning"; + return "neutral"; +} + +function evidenceLabel(finding: FileIntegrityFinding): string { + const expected = finding.expected_size_bytes == null ? "-" : `${finding.expected_size_bytes} B`; + const observed = finding.observed_size_bytes == null ? "-" : `${finding.observed_size_bytes} B`; + return `${expected} / ${observed}`; +} + +function cleanupActionMessage(action: string): string { + if (action === "retained_referenced") return "Cleanup was blocked because the object has a managed database reference."; + if (action === "already_deleted" || action === "already_absent") return "The object was already absent; the finding is reconciled."; + if (action === "deleted") return "The unreferenced storage object was deleted and recovery evidence was recorded."; + return `Integrity cleanup result: ${action.replaceAll("_", " ")}.`; +} + +function formatDateTime(value: string | null | undefined): string { + if (!value) return "-"; + const parsed = new Date(value); + return Number.isNaN(parsed.getTime()) ? value : parsed.toLocaleString(); +} diff --git a/webui/src/index.ts b/webui/src/index.ts index 5c3d1b7..694aabe 100644 --- a/webui/src/index.ts +++ b/webui/src/index.ts @@ -1,6 +1,7 @@ export { default } from "./module"; export * from "./module"; export { default as FilesPage } from "./features/files/FilesPage"; +export { default as FileIntegrityPanel } from "./features/files/FileIntegrityPanel"; export * from "./api/files"; export type { PlatformWebModule, PlatformNavItem, PlatformRouteContribution, PlatformRouteContext } from "@govoplan/core-webui"; export { FolderTree } from "./features/files/components/FileManagerComponents"; diff --git a/webui/src/module.ts b/webui/src/module.ts index 3fe4c07..0418c1e 100644 --- a/webui/src/module.ts +++ b/webui/src/module.ts @@ -16,6 +16,7 @@ import { generatedTranslations } from "./i18n/generatedTranslations"; import "./styles/file-manager.css"; const FilesPage = lazy(() => import("./features/files/FilesPage")); +const FileIntegrityPanel = lazy(() => import("./features/files/FileIntegrityPanel")); const fileRead = ["files:file:read"]; const translations = { @@ -97,6 +98,20 @@ const fileConnectorAdminSections: AdminSectionsUiCapability = { scopeType: "tenant", canWrite: hasScope(auth, "admin:settings:write") || hasScope(auth, "files:file:admin") }) + }, + { + id: "tenant-file-integrity", + moduleId: "files", + kind: "operations", + surfaceId: "files.admin.tenant-integrity", + label: "File integrity", + group: "TENANT", + order: 66, + allOf: ["files:file:admin"], + render: ({ settings, auth }) => createElement(FileIntegrityPanel, { + settings, + canWrite: hasScope(auth, "files:file:admin") + }) }] }; @@ -110,6 +125,7 @@ export const filesModule: PlatformWebModule = { viewSurfaces: [ { id: "files.admin.system-connectors", moduleId: "files", kind: "section", label: "System file connections", order: 75 }, { id: "files.admin.tenant-connectors", moduleId: "files", kind: "section", label: "Tenant file connections", order: 65 }, + { id: "files.admin.tenant-integrity", moduleId: "files", kind: "section", label: "File integrity", order: 66 }, { id: "files.admin.group-connectors", moduleId: "files", kind: "section", label: "Group file connections", order: 65 }, { id: "files.admin.user-connectors", moduleId: "files", kind: "section", label: "User file connections", order: 65 }, { id: "files.settings.connectors", moduleId: "files", kind: "section", label: "Personal file connections", order: 20 },