439 lines
16 KiB
Python
439 lines
16 KiB
Python
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
import hashlib
|
|
from typing import Any, Mapping
|
|
|
|
from govoplan_core.core.recovery import (
|
|
RecoveryGuaranteeError,
|
|
RecoveryMode,
|
|
RecoveryPlan,
|
|
RecoveryStatus,
|
|
)
|
|
from govoplan_core.core.recovery_runtime import (
|
|
RecoveryOperationBusy,
|
|
RecoveryOperationStateConflict,
|
|
begin_durable_recovery_operation,
|
|
)
|
|
from govoplan_core.core.runtime_coordination import process_runtime_identity
|
|
from govoplan_core.db.session import get_database
|
|
from govoplan_files.backend.storage.common import FileStorageError
|
|
from govoplan_files.backend.storage.connector_browse import (
|
|
ConnectorBrowseError,
|
|
ConnectorBrowseUnsupported,
|
|
_clean,
|
|
_s3_bucket,
|
|
_s3_client,
|
|
_s3_object_key,
|
|
normalize_connector_browse_path,
|
|
)
|
|
from govoplan_files.backend.storage.connector_profiles import ConnectorProfile
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class ConnectorWriteResult:
|
|
recovery_operation_id: str
|
|
status: str
|
|
replayed: bool
|
|
provider: str
|
|
remote_path: str
|
|
revision: str | None
|
|
checksum_sha256: str
|
|
size_bytes: int
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class _HeadProbe:
|
|
observed: Mapping[str, Any] | None
|
|
verified: bool
|
|
exception_type: str | None = None
|
|
|
|
|
|
def write_connector_file(
|
|
profile: ConnectorProfile,
|
|
*,
|
|
tenant_id: str,
|
|
library_id: str | None,
|
|
remote_path: str,
|
|
data: bytes,
|
|
content_type: str | None,
|
|
expected_revision: str | None,
|
|
idempotency_key: str,
|
|
) -> ConnectorWriteResult:
|
|
if profile.provider != "s3" or "write" not in profile.capabilities:
|
|
raise FileStorageError(
|
|
"Remote writes require an S3 connection with the write capability enabled"
|
|
)
|
|
try:
|
|
normalized_path = normalize_connector_browse_path(remote_path)
|
|
except ConnectorBrowseError as exc:
|
|
raise FileStorageError(str(exc)) from exc
|
|
if not normalized_path:
|
|
raise FileStorageError("Remote write requires an object path")
|
|
try:
|
|
bucket = _s3_bucket(profile, library_id)
|
|
key = _s3_object_key(profile, normalized_path)
|
|
except ConnectorBrowseError as exc:
|
|
raise FileStorageError(str(exc)) from exc
|
|
if not bucket or not key:
|
|
raise FileStorageError("S3 remote write requires a bucket and object path")
|
|
checksum = hashlib.sha256(data).hexdigest()
|
|
target_digest = hashlib.sha256(f"{bucket}:{key}".encode("utf-8")).hexdigest()
|
|
request = {
|
|
"tenant_id": tenant_id,
|
|
"connector_profile_id": profile.id,
|
|
"provider": profile.provider,
|
|
"remote_target_sha256": target_digest,
|
|
"content_sha256": checksum,
|
|
"size_bytes": len(data),
|
|
"expected_revision": expected_revision,
|
|
}
|
|
try:
|
|
started = begin_durable_recovery_operation(
|
|
get_database().SessionLocal,
|
|
identity=process_runtime_identity(),
|
|
module_id="files",
|
|
operation_type="connector-s3-write",
|
|
idempotency_key=f"files-connector-write:{idempotency_key}",
|
|
request=request,
|
|
recovery_plan=RecoveryPlan(
|
|
mode=RecoveryMode.FORWARD_RECOVERY,
|
|
preconditions=(
|
|
"the connector space is explicitly configured for two-way writes",
|
|
"the connection and inherited policy allow the exact remote path",
|
|
"a conditional create or expected remote revision prevents blind overwrite",
|
|
),
|
|
forward_recovery_steps=(
|
|
"inspect provider metadata for the request and content digests",
|
|
"resolve success only for an exact digest match or retry after verified absence",
|
|
),
|
|
verification_steps=(
|
|
"read S3 object metadata after the write",
|
|
"match the recorded request and content digests without downloading content",
|
|
),
|
|
),
|
|
precondition_evidence={
|
|
"remote_target_sha256": target_digest,
|
|
"content_sha256": checksum,
|
|
"size_bytes": len(data),
|
|
"conditional_write": True,
|
|
},
|
|
lease_resource_key=f"files:connector:{tenant_id}:{profile.id}:{target_digest[:40]}",
|
|
lease_ttl_seconds=15 * 60,
|
|
resource_type="file_connector_object",
|
|
resource_id=target_digest,
|
|
metadata={
|
|
"resources": ["postgresql", "s3-connector"],
|
|
"provider": "s3",
|
|
"connector_profile_id": profile.id,
|
|
},
|
|
block_unresolved_resource=True,
|
|
)
|
|
except (RecoveryOperationBusy, RecoveryOperationStateConflict) as exc:
|
|
raise FileStorageError(
|
|
"This remote object is owned by another write or unresolved recovery operation"
|
|
) from exc
|
|
except (RecoveryGuaranteeError, RuntimeError, ValueError) as exc:
|
|
raise FileStorageError(
|
|
"The Files recovery ledger is unavailable; the remote object was not changed"
|
|
) from exc
|
|
if started.replayed or started.operation is None:
|
|
return ConnectorWriteResult(
|
|
recovery_operation_id=started.operation_id,
|
|
status=started.status,
|
|
replayed=True,
|
|
provider="s3",
|
|
remote_path=normalized_path,
|
|
revision=None,
|
|
checksum_sha256=checksum,
|
|
size_bytes=len(data),
|
|
)
|
|
operation = started.operation
|
|
try:
|
|
client = _s3_client(profile)
|
|
except (ConnectorBrowseError, ConnectorBrowseUnsupported) as exc:
|
|
operation.reject(
|
|
summary="The S3 client was unavailable before the remote effect",
|
|
evidence=_verified(
|
|
{"external_effect_started": False}, exception_type=type(exc).__name__
|
|
),
|
|
)
|
|
raise FileStorageError(str(exc)) from exc
|
|
try:
|
|
try:
|
|
before = _head_object(client, bucket=bucket, key=key)
|
|
except Exception as exc:
|
|
operation.reject(
|
|
summary="Remote preconditions could not be inspected before the effect",
|
|
evidence=_verified(
|
|
{"external_effect_started": False},
|
|
exception_type=type(exc).__name__,
|
|
),
|
|
)
|
|
raise FileStorageError("S3 remote revision lookup failed") from exc
|
|
expected = _normalize_revision(expected_revision)
|
|
if before is not None and expected is None:
|
|
operation.reject(
|
|
summary="A blind remote overwrite was rejected",
|
|
evidence=_verified(
|
|
{"remote_object_exists": True, "expected_revision_supplied": False}
|
|
),
|
|
)
|
|
raise FileStorageError(
|
|
"The remote object already exists; reload it and supply its expected revision"
|
|
)
|
|
if expected is not None and (
|
|
before is None or expected not in _observed_revisions(before)
|
|
):
|
|
operation.reject(
|
|
summary="The expected remote revision did not match",
|
|
evidence=_verified(
|
|
{
|
|
"remote_object_exists": before is not None,
|
|
"expected_revision_matches": False,
|
|
}
|
|
),
|
|
)
|
|
raise FileStorageError("The remote object changed; reload before writing")
|
|
params: dict[str, object] = {
|
|
"Bucket": bucket,
|
|
"Key": key,
|
|
"Body": data,
|
|
"Metadata": {
|
|
"govoplan-sha256": checksum,
|
|
"govoplan-request-id": started.operation_id,
|
|
},
|
|
}
|
|
if content_type:
|
|
params["ContentType"] = content_type
|
|
if before is None:
|
|
params["IfNoneMatch"] = "*"
|
|
else:
|
|
params["IfMatch"] = str(before.get("ETag") or expected_revision or "")
|
|
try:
|
|
client.put_object(**params)
|
|
except Exception as exc:
|
|
probe = _probe_head_object(client, bucket=bucket, key=key)
|
|
if probe.verified and _matches_effect(
|
|
probe.observed,
|
|
checksum=checksum,
|
|
operation_id=started.operation_id,
|
|
):
|
|
operation.succeed(
|
|
evidence=_success_evidence(
|
|
probe.observed,
|
|
checksum=checksum,
|
|
operation_id=started.operation_id,
|
|
)
|
|
)
|
|
return _result(
|
|
started.operation_id,
|
|
normalized_path,
|
|
checksum,
|
|
len(data),
|
|
probe.observed,
|
|
)
|
|
operation.unresolved(
|
|
status=RecoveryStatus.OUTCOME_UNKNOWN,
|
|
summary="The S3 write outcome requires reconciliation",
|
|
evidence={
|
|
"remote_target_sha256": target_digest,
|
|
"observed": _public_observation(probe),
|
|
"exception_type": type(exc).__name__,
|
|
},
|
|
failure_summary="Inspect the remote object metadata before retrying this path",
|
|
)
|
|
return ConnectorWriteResult(
|
|
recovery_operation_id=started.operation_id,
|
|
status=RecoveryStatus.OUTCOME_UNKNOWN.value,
|
|
replayed=False,
|
|
provider="s3",
|
|
remote_path=normalized_path,
|
|
revision=_revision(probe.observed),
|
|
checksum_sha256=checksum,
|
|
size_bytes=len(data),
|
|
)
|
|
probe = _probe_head_object(client, bucket=bucket, key=key)
|
|
if not probe.verified:
|
|
operation.unresolved(
|
|
status=RecoveryStatus.OUTCOME_UNKNOWN,
|
|
summary="The S3 write returned but provider evidence could not be queried",
|
|
evidence={
|
|
"remote_target_sha256": target_digest,
|
|
"observed": _public_observation(probe),
|
|
},
|
|
failure_summary="Inspect the remote object metadata before retrying this path",
|
|
)
|
|
return ConnectorWriteResult(
|
|
recovery_operation_id=started.operation_id,
|
|
status=RecoveryStatus.OUTCOME_UNKNOWN.value,
|
|
replayed=False,
|
|
provider="s3",
|
|
remote_path=normalized_path,
|
|
revision=None,
|
|
checksum_sha256=checksum,
|
|
size_bytes=len(data),
|
|
)
|
|
observed = probe.observed
|
|
if not _matches_effect(
|
|
observed, checksum=checksum, operation_id=started.operation_id
|
|
):
|
|
operation.unresolved(
|
|
status=RecoveryStatus.RECOVERY_REQUIRED,
|
|
summary="The S3 write returned but exact provider evidence did not match",
|
|
evidence={
|
|
"remote_target_sha256": target_digest,
|
|
"observed": _public_observation(probe),
|
|
},
|
|
failure_summary="Reconcile the remote object before another write",
|
|
)
|
|
return ConnectorWriteResult(
|
|
recovery_operation_id=started.operation_id,
|
|
status=RecoveryStatus.RECOVERY_REQUIRED.value,
|
|
replayed=False,
|
|
provider="s3",
|
|
remote_path=normalized_path,
|
|
revision=_revision(observed),
|
|
checksum_sha256=checksum,
|
|
size_bytes=len(data),
|
|
)
|
|
operation.succeed(
|
|
evidence=_success_evidence(
|
|
observed, checksum=checksum, operation_id=started.operation_id
|
|
)
|
|
)
|
|
return _result(
|
|
started.operation_id,
|
|
normalized_path,
|
|
checksum,
|
|
len(data),
|
|
observed,
|
|
)
|
|
finally:
|
|
close = getattr(client, "close", None)
|
|
if callable(close):
|
|
close()
|
|
|
|
|
|
def _head_object(client: Any, *, bucket: str, key: str) -> Mapping[str, Any] | None:
|
|
try:
|
|
response = client.head_object(Bucket=bucket, Key=key)
|
|
except Exception as exc:
|
|
if _is_not_found(exc):
|
|
return None
|
|
raise
|
|
if not isinstance(response, Mapping):
|
|
raise FileStorageError("S3 connector returned invalid object metadata")
|
|
return response
|
|
|
|
|
|
def _probe_head_object(client: Any, *, bucket: str, key: str) -> _HeadProbe:
|
|
try:
|
|
return _HeadProbe(
|
|
observed=_head_object(client, bucket=bucket, key=key), verified=True
|
|
)
|
|
except Exception as exc:
|
|
return _HeadProbe(
|
|
observed=None,
|
|
verified=False,
|
|
exception_type=type(exc).__name__,
|
|
)
|
|
|
|
|
|
def _is_not_found(exc: Exception) -> bool:
|
|
response = getattr(exc, "response", None)
|
|
if not isinstance(response, Mapping):
|
|
return False
|
|
error = response.get("Error")
|
|
code = error.get("Code") if isinstance(error, Mapping) else None
|
|
status = response.get("ResponseMetadata")
|
|
http_status = status.get("HTTPStatusCode") if isinstance(status, Mapping) else None
|
|
return str(code).casefold() in {"404", "nosuchkey", "notfound"} or http_status == 404
|
|
|
|
|
|
def _metadata(observed: Mapping[str, Any] | None) -> Mapping[str, Any]:
|
|
value = observed.get("Metadata") if observed else None
|
|
return value if isinstance(value, Mapping) else {}
|
|
|
|
|
|
def _matches_effect(
|
|
observed: Mapping[str, Any] | None, *, checksum: str, operation_id: str
|
|
) -> bool:
|
|
metadata = _metadata(observed)
|
|
return bool(
|
|
observed is not None
|
|
and _clean(metadata.get("govoplan-sha256")) == checksum
|
|
and _clean(metadata.get("govoplan-request-id")) == operation_id
|
|
)
|
|
|
|
|
|
def _revision(observed: Mapping[str, Any] | None) -> str | None:
|
|
if observed is None:
|
|
return None
|
|
return _clean(observed.get("VersionId") or observed.get("ETag"))
|
|
|
|
|
|
def _normalize_revision(value: str | None) -> str | None:
|
|
cleaned = _clean(value)
|
|
return cleaned.strip('"') if cleaned else None
|
|
|
|
|
|
def _observed_revisions(observed: Mapping[str, Any]) -> set[str]:
|
|
return {
|
|
normalized
|
|
for value in (observed.get("VersionId"), observed.get("ETag"))
|
|
if (normalized := _normalize_revision(_clean(value))) is not None
|
|
}
|
|
|
|
|
|
def _success_evidence(
|
|
observed: Mapping[str, Any] | None, *, checksum: str, operation_id: str
|
|
) -> dict[str, object]:
|
|
return _verified(
|
|
{
|
|
"remote_object_present": observed is not None,
|
|
"content_digest_matches": _clean(_metadata(observed).get("govoplan-sha256")) == checksum,
|
|
"request_marker_matches": _clean(_metadata(observed).get("govoplan-request-id")) == operation_id,
|
|
},
|
|
revision=_revision(observed),
|
|
content_sha256=checksum,
|
|
)
|
|
|
|
|
|
def _public_observation(probe: _HeadProbe) -> dict[str, object]:
|
|
observed = probe.observed
|
|
return {
|
|
"probe_verified": probe.verified,
|
|
"probe_exception_type": probe.exception_type,
|
|
"present": observed is not None,
|
|
"revision": _revision(observed),
|
|
"has_content_digest": bool(_clean(_metadata(observed).get("govoplan-sha256"))),
|
|
"has_request_marker": bool(_clean(_metadata(observed).get("govoplan-request-id"))),
|
|
}
|
|
|
|
|
|
def _verified(checks: dict[str, object], **details: object) -> dict[str, object]:
|
|
return {"verified": True, "checks": checks, **details}
|
|
|
|
|
|
def _result(
|
|
operation_id: str,
|
|
remote_path: str,
|
|
checksum: str,
|
|
size_bytes: int,
|
|
observed: Mapping[str, Any] | None,
|
|
) -> ConnectorWriteResult:
|
|
return ConnectorWriteResult(
|
|
recovery_operation_id=operation_id,
|
|
status=RecoveryStatus.SUCCEEDED.value,
|
|
replayed=False,
|
|
provider="s3",
|
|
remote_path=remote_path,
|
|
revision=_revision(observed),
|
|
checksum_sha256=checksum,
|
|
size_bytes=size_bytes,
|
|
)
|
|
|
|
|
|
__all__ = ["ConnectorWriteResult", "write_connector_file"]
|