388 lines
13 KiB
Python
388 lines
13 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
|
|
from fastapi import APIRouter, Depends, HTTPException, Query, status
|
|
from sqlalchemy.orm import Session
|
|
|
|
from govoplan_core.auth import ApiPrincipal, require_scope
|
|
from govoplan_core.audit.logging import audit_from_principal
|
|
from govoplan_core.db.session import get_session
|
|
from govoplan_files.backend.db.models import (
|
|
FileIntegrityFinding,
|
|
FileIntegrityScan,
|
|
)
|
|
from govoplan_files.backend.schemas import (
|
|
FileIntegrityActionRequest,
|
|
FileIntegrityActionResponse,
|
|
FileIntegrityFindingResponse,
|
|
FileIntegrityFindingsResponse,
|
|
FileIntegrityScanCreateRequest,
|
|
FileIntegrityScanRunRequest,
|
|
FileIntegrityScanResponse,
|
|
FileIntegrityScansResponse,
|
|
)
|
|
from govoplan_files.backend.storage.backends import StorageBackendError
|
|
from govoplan_files.backend.storage.common import FileStorageError
|
|
from govoplan_files.backend.storage.integrity import (
|
|
cleanup_orphan_finding,
|
|
create_integrity_scan,
|
|
mark_integrity_scan_failed,
|
|
recheck_integrity_finding,
|
|
run_integrity_scan_batch,
|
|
)
|
|
|
|
|
|
router = APIRouter(prefix="/files/integrity", tags=["files-integrity"])
|
|
|
|
|
|
@router.get("/scans", response_model=FileIntegrityScansResponse)
|
|
def list_integrity_scans(
|
|
limit: int = Query(default=50, ge=1, le=200),
|
|
session: Session = Depends(get_session),
|
|
principal: ApiPrincipal = Depends(require_scope("files:file:admin")),
|
|
) -> FileIntegrityScansResponse:
|
|
rows = (
|
|
session.query(FileIntegrityScan)
|
|
.filter(FileIntegrityScan.tenant_id == principal.tenant_id)
|
|
.order_by(FileIntegrityScan.created_at.desc(), FileIntegrityScan.id.desc())
|
|
.limit(limit)
|
|
.all()
|
|
)
|
|
return FileIntegrityScansResponse(
|
|
scans=[_scan_response(row) for row in rows]
|
|
)
|
|
|
|
|
|
@router.post(
|
|
"/scans",
|
|
response_model=FileIntegrityScanResponse,
|
|
status_code=status.HTTP_201_CREATED,
|
|
)
|
|
def create_scan(
|
|
payload: FileIntegrityScanCreateRequest,
|
|
session: Session = Depends(get_session),
|
|
principal: ApiPrincipal = Depends(require_scope("files:file:admin")),
|
|
) -> FileIntegrityScanResponse:
|
|
try:
|
|
scan = create_integrity_scan(
|
|
session,
|
|
tenant_id=principal.tenant_id,
|
|
user_id=principal.user.id,
|
|
verify_checksums=payload.verify_checksums,
|
|
batch_size=payload.batch_size,
|
|
)
|
|
audit_from_principal(
|
|
session,
|
|
principal,
|
|
action="files.integrity.scan_created",
|
|
object_type="file_integrity_scan",
|
|
object_id=scan.id,
|
|
details={
|
|
"storage_backend": scan.storage_backend,
|
|
"verify_checksums": scan.verify_checksums,
|
|
"batch_size": scan.batch_size,
|
|
},
|
|
)
|
|
session.commit()
|
|
return _scan_response(scan)
|
|
except (FileStorageError, StorageBackendError) as exc:
|
|
session.rollback()
|
|
raise _integrity_http_error(exc) from exc
|
|
|
|
|
|
@router.post("/scans/{scan_id}/run", response_model=FileIntegrityScanResponse)
|
|
def run_scan_batch(
|
|
scan_id: str,
|
|
payload: FileIntegrityScanRunRequest,
|
|
session: Session = Depends(get_session),
|
|
principal: ApiPrincipal = Depends(require_scope("files:file:admin")),
|
|
) -> FileIntegrityScanResponse:
|
|
scan = _scan_for_tenant(
|
|
session,
|
|
scan_id,
|
|
principal.tenant_id,
|
|
for_update=True,
|
|
)
|
|
_assert_expected_revision(scan.revision, payload.expected_revision)
|
|
previous_status = scan.status
|
|
try:
|
|
run_integrity_scan_batch(session, scan)
|
|
if scan.status == "completed" and previous_status != "completed":
|
|
audit_from_principal(
|
|
session,
|
|
principal,
|
|
action="files.integrity.scan_completed",
|
|
object_type="file_integrity_scan",
|
|
object_id=scan.id,
|
|
details={
|
|
"verified_blob_count": scan.verified_blob_count,
|
|
"quarantined_blob_count": scan.quarantined_blob_count,
|
|
"orphan_object_count": scan.orphan_object_count,
|
|
},
|
|
)
|
|
session.commit()
|
|
return _scan_response(scan)
|
|
except (FileStorageError, StorageBackendError) as exc:
|
|
session.rollback()
|
|
scan = _scan_for_tenant(
|
|
session,
|
|
scan_id,
|
|
principal.tenant_id,
|
|
for_update=True,
|
|
)
|
|
mark_integrity_scan_failed(scan, error=exc)
|
|
audit_from_principal(
|
|
session,
|
|
principal,
|
|
action="files.integrity.scan_failed",
|
|
object_type="file_integrity_scan",
|
|
object_id=scan.id,
|
|
details={"error_type": type(exc).__name__},
|
|
)
|
|
session.commit()
|
|
raise _integrity_http_error(exc) from exc
|
|
|
|
|
|
@router.get(
|
|
"/scans/{scan_id}/findings",
|
|
response_model=FileIntegrityFindingsResponse,
|
|
)
|
|
def list_integrity_findings(
|
|
scan_id: str,
|
|
state_filter: str | None = Query(default=None, alias="state"),
|
|
limit: int = Query(default=500, ge=1, le=1000),
|
|
session: Session = Depends(get_session),
|
|
principal: ApiPrincipal = Depends(require_scope("files:file:admin")),
|
|
) -> FileIntegrityFindingsResponse:
|
|
scan = _scan_for_tenant(session, scan_id, principal.tenant_id)
|
|
query = session.query(FileIntegrityFinding).filter(
|
|
FileIntegrityFinding.scan_id == scan.id,
|
|
FileIntegrityFinding.tenant_id == principal.tenant_id,
|
|
)
|
|
if state_filter:
|
|
query = query.filter(FileIntegrityFinding.state == state_filter)
|
|
rows = (
|
|
query.order_by(
|
|
FileIntegrityFinding.created_at.asc(),
|
|
FileIntegrityFinding.id.asc(),
|
|
)
|
|
.limit(limit)
|
|
.all()
|
|
)
|
|
return FileIntegrityFindingsResponse(
|
|
findings=[_finding_response(row) for row in rows]
|
|
)
|
|
|
|
|
|
@router.post(
|
|
"/findings/{finding_id}/recheck",
|
|
response_model=FileIntegrityActionResponse,
|
|
)
|
|
def recheck_finding(
|
|
finding_id: str,
|
|
payload: FileIntegrityActionRequest,
|
|
session: Session = Depends(get_session),
|
|
principal: ApiPrincipal = Depends(require_scope("files:file:admin")),
|
|
) -> FileIntegrityActionResponse:
|
|
finding = _finding_for_tenant(
|
|
session,
|
|
finding_id,
|
|
principal.tenant_id,
|
|
for_update=True,
|
|
)
|
|
_assert_expected_revision(finding.revision, payload.expected_revision)
|
|
try:
|
|
result = recheck_integrity_finding(
|
|
session,
|
|
finding,
|
|
user_id=principal.user.id,
|
|
dry_run=payload.dry_run,
|
|
)
|
|
_audit_integrity_action(session, principal, result)
|
|
session.commit()
|
|
return _action_response(result)
|
|
except (FileStorageError, StorageBackendError) as exc:
|
|
session.rollback()
|
|
raise _integrity_http_error(exc) from exc
|
|
|
|
|
|
@router.post(
|
|
"/findings/{finding_id}/cleanup",
|
|
response_model=FileIntegrityActionResponse,
|
|
)
|
|
def cleanup_finding(
|
|
finding_id: str,
|
|
payload: FileIntegrityActionRequest,
|
|
session: Session = Depends(get_session),
|
|
principal: ApiPrincipal = Depends(require_scope("files:file:admin")),
|
|
) -> FileIntegrityActionResponse:
|
|
finding = _finding_for_tenant(
|
|
session,
|
|
finding_id,
|
|
principal.tenant_id,
|
|
for_update=True,
|
|
)
|
|
_assert_expected_revision(finding.revision, payload.expected_revision)
|
|
try:
|
|
result = cleanup_orphan_finding(
|
|
session,
|
|
finding,
|
|
user_id=principal.user.id,
|
|
dry_run=payload.dry_run,
|
|
)
|
|
_audit_integrity_action(session, principal, result)
|
|
session.commit()
|
|
return _action_response(result)
|
|
except (FileStorageError, StorageBackendError) as exc:
|
|
session.rollback()
|
|
raise _integrity_http_error(exc) from exc
|
|
|
|
|
|
def _scan_for_tenant(
|
|
session: Session,
|
|
scan_id: str,
|
|
tenant_id: str,
|
|
*,
|
|
for_update: bool = False,
|
|
) -> FileIntegrityScan:
|
|
query = session.query(FileIntegrityScan).filter(FileIntegrityScan.id == scan_id)
|
|
if for_update:
|
|
query = query.populate_existing().with_for_update()
|
|
scan = query.one_or_none()
|
|
if scan is None or scan.tenant_id != tenant_id:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_404_NOT_FOUND,
|
|
detail="Integrity scan not found",
|
|
)
|
|
return scan
|
|
|
|
|
|
def _finding_for_tenant(
|
|
session: Session,
|
|
finding_id: str,
|
|
tenant_id: str,
|
|
*,
|
|
for_update: bool = False,
|
|
) -> FileIntegrityFinding:
|
|
query = session.query(FileIntegrityFinding).filter(
|
|
FileIntegrityFinding.id == finding_id
|
|
)
|
|
if for_update:
|
|
query = query.populate_existing().with_for_update()
|
|
finding = query.one_or_none()
|
|
if finding is None or finding.tenant_id != tenant_id:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_404_NOT_FOUND,
|
|
detail="Integrity finding not found",
|
|
)
|
|
return finding
|
|
|
|
|
|
def _assert_expected_revision(current: int, expected: int) -> None:
|
|
if current != expected:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_409_CONFLICT,
|
|
detail=(
|
|
"The integrity record changed after it was loaded; reload before "
|
|
"performing this action."
|
|
),
|
|
)
|
|
|
|
|
|
def _audit_integrity_action(session, principal, result) -> None:
|
|
audit_from_principal(
|
|
session,
|
|
principal,
|
|
action=f"files.integrity.{result.action}",
|
|
object_type="file_integrity_finding",
|
|
object_id=result.finding.id,
|
|
details={
|
|
"scan_id": result.finding.scan_id,
|
|
"kind": result.finding.kind,
|
|
"dry_run": result.dry_run,
|
|
"changed": result.changed,
|
|
"storage_key_sha256": hashlib.sha256(
|
|
result.finding.storage_key.encode("utf-8")
|
|
).hexdigest(),
|
|
"inspection": result.inspection.kind
|
|
if result.inspection
|
|
else None,
|
|
},
|
|
)
|
|
|
|
|
|
def _scan_response(scan: FileIntegrityScan) -> FileIntegrityScanResponse:
|
|
return FileIntegrityScanResponse(
|
|
id=scan.id,
|
|
tenant_id=scan.tenant_id,
|
|
storage_backend=scan.storage_backend,
|
|
storage_prefix=scan.storage_prefix,
|
|
status=scan.status,
|
|
revision=scan.revision,
|
|
phase=scan.phase,
|
|
verify_checksums=scan.verify_checksums,
|
|
batch_size=scan.batch_size,
|
|
scanned_blob_count=scan.scanned_blob_count,
|
|
verified_blob_count=scan.verified_blob_count,
|
|
quarantined_blob_count=scan.quarantined_blob_count,
|
|
scanned_object_count=scan.scanned_object_count,
|
|
orphan_object_count=scan.orphan_object_count,
|
|
created_by_user_id=scan.created_by_user_id,
|
|
started_at=scan.started_at.isoformat() if scan.started_at else None,
|
|
completed_at=scan.completed_at.isoformat()
|
|
if scan.completed_at
|
|
else None,
|
|
last_error=scan.last_error,
|
|
created_at=scan.created_at.isoformat(),
|
|
updated_at=scan.updated_at.isoformat(),
|
|
)
|
|
|
|
|
|
def _finding_response(
|
|
finding: FileIntegrityFinding,
|
|
) -> FileIntegrityFindingResponse:
|
|
return FileIntegrityFindingResponse(
|
|
id=finding.id,
|
|
scan_id=finding.scan_id,
|
|
tenant_id=finding.tenant_id,
|
|
kind=finding.kind,
|
|
state=finding.state,
|
|
revision=finding.revision,
|
|
blob_id=finding.blob_id,
|
|
storage_key=finding.storage_key,
|
|
expected_size_bytes=finding.expected_size_bytes,
|
|
observed_size_bytes=finding.observed_size_bytes,
|
|
expected_checksum_sha256=finding.expected_checksum_sha256,
|
|
observed_checksum_sha256=finding.observed_checksum_sha256,
|
|
resolved_at=finding.resolved_at.isoformat()
|
|
if finding.resolved_at
|
|
else None,
|
|
resolved_by_user_id=finding.resolved_by_user_id,
|
|
created_at=finding.created_at.isoformat(),
|
|
updated_at=finding.updated_at.isoformat(),
|
|
)
|
|
|
|
|
|
def _action_response(result) -> FileIntegrityActionResponse:
|
|
return FileIntegrityActionResponse(
|
|
action=result.action,
|
|
changed=result.changed,
|
|
dry_run=result.dry_run,
|
|
finding=_finding_response(result.finding),
|
|
inspection_kind=result.inspection.kind if result.inspection else None,
|
|
inspection_valid=result.inspection.valid if result.inspection else None,
|
|
)
|
|
|
|
|
|
def _integrity_http_error(exc: Exception) -> HTTPException:
|
|
if isinstance(exc, FileStorageError):
|
|
return HTTPException(
|
|
status_code=status.HTTP_409_CONFLICT,
|
|
detail=str(exc),
|
|
)
|
|
return HTTPException(
|
|
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
|
detail="The configured file storage backend could not complete the integrity operation",
|
|
)
|