GovOPlaN/govoplan-core
2
0
Fork 0
Code Issues 4 Pull Requests Actions Packages Projects Releases Wiki Activity

Implement durable module recovery operations

Browse Source
This commit is contained in:
zemion
2026-08-03 03:07:42 +02:00
parent 21c1fa49b6
commit bca6a7c8aa
6 changed files with 744 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/govoplan_core/core/recovery_runtime.py
+446
View File
@@ -0,0 +1,446 @@
from __future__ import annotations
from collections.abc import Callable
from dataclasses import dataclass
from typing import Any
from sqlalchemy import select
from sqlalchemy.orm import Session
from govoplan_core.core.recovery import (
RecoveryGuaranteeError,
RecoveryMode,
RecoveryOperation,
RecoveryPlan,
RecoveryStatus,
plan_recovery_operation,
prepare_recovery_operation,
record_recovery_checkpoint,
start_recovery_operation,
transition_recovery_operation,
verify_recovery_evidence_chain,
)
from govoplan_core.core.runtime_coordination import (
LeaseClaim,
RuntimeIdentity,
acquire_lease,
release_lease,
renew_lease,
)
SessionFactory = Callable[[], Session]
class RecoveryOperationBusy(RecoveryGuaranteeError):
pass
class RecoveryOperationStateConflict(RecoveryGuaranteeError):
def __init__(self, operation_id: str, status: str) -> None:
self.operation_id = operation_id
self.status = status
super().__init__(
f"Recovery operation {operation_id} is already {status}; "
"reconcile it before starting another effect"
)
@dataclass(frozen=True, slots=True)
class DurableRecoveryStart:
operation_id: str
status: str
replayed: bool
operation: DurableRecoveryOperation | None
@dataclass(slots=True)
class DurableRecoveryOperation:
"""Append checkpoints in independent, committed transactions.
The caller's business transaction may roll back without erasing evidence
that an object, queue, filesystem, or provider effect already occurred.
"""
session_factory: SessionFactory
operation_id: str
lease_claim: LeaseClaim
lease_ttl_seconds: int
closed: bool = False
def checkpoint(
self,
*,
kind: str,
summary: str,
evidence: dict[str, Any],
) -> None:
with self.session_factory() as session:
operation, claim = self._locked_and_renewed(session)
record_recovery_checkpoint(
session,
operation,
kind=kind,
summary=summary,
evidence=evidence,
lease_claim=claim,
)
self._verify_chain(session)
session.commit()
def succeed(self, *, evidence: dict[str, Any]) -> None:
with self.session_factory() as session:
operation, claim = self._locked_and_renewed(session)
transition_recovery_operation(
session,
operation,
status=RecoveryStatus.SUCCEEDED,
kind="verified-success",
summary="Operation effects and authoritative state were verified",
evidence=evidence,
lease_claim=claim,
)
self._verify_chain(session)
release_lease(session, claim)
session.commit()
self.closed = True
def compensate(
self,
*,
failure_summary: str,
failure_evidence: dict[str, Any],
recovery_evidence: dict[str, Any],
) -> None:
with self.session_factory() as session:
operation, claim = self._locked_and_renewed(session)
if operation.status == RecoveryStatus.RUNNING.value:
operation = transition_recovery_operation(
session,
operation,
status=RecoveryStatus.RECOVERY_REQUIRED,
kind="compensation-required",
summary="The started operation requires explicit compensation",
evidence=failure_evidence,
failure_summary=failure_summary,
lease_claim=claim,
)
if operation.status == RecoveryStatus.RECOVERY_REQUIRED.value:
operation = transition_recovery_operation(
session,
operation,
status=RecoveryStatus.RECOVERING,
kind="compensation-started",
summary="Compensation started",
evidence={"failure_summary": failure_summary},
lease_claim=claim,
)
transition_recovery_operation(
session,
operation,
status=RecoveryStatus.RECOVERED,
kind="compensation-verified",
summary="Compensation restored the declared invariant",
evidence=recovery_evidence,
lease_claim=claim,
)
self._verify_chain(session)
release_lease(session, claim)
session.commit()
self.closed = True
def unresolved(
self,
*,
status: RecoveryStatus,
summary: str,
evidence: dict[str, Any],
failure_summary: str,
) -> None:
if status not in {
RecoveryStatus.OUTCOME_UNKNOWN,
RecoveryStatus.RECOVERY_REQUIRED,
}:
raise ValueError("Unresolved operations require an unresolved status")
with self.session_factory() as session:
operation, claim = self._locked_and_renewed(session)
transition_recovery_operation(
session,
operation,
status=status,
kind="unresolved-effect",
summary=summary,
evidence=evidence,
failure_summary=failure_summary,
lease_claim=claim,
)
self._verify_chain(session)
release_lease(session, claim)
session.commit()
self.closed = True
def release_unresolved(self) -> None:
"""Release authority after a process-local exception.
This does not alter the operation state. A later recovery claim treats a
stale `running` operation according to its declared recovery mode.
"""
if self.closed:
return
with self.session_factory() as session:
operation, claim = self._locked_and_renewed(session)
record_recovery_checkpoint(
session,
operation,
kind="authority-released",
summary="Execution authority was released without a terminal claim",
evidence={"status": operation.status},
lease_claim=claim,
)
self._verify_chain(session)
release_lease(session, claim)
session.commit()
self.closed = True
def _locked_and_renewed(
self,
session: Session,
) -> tuple[RecoveryOperation, LeaseClaim]:
if self.closed:
raise RecoveryGuaranteeError("Recovery operation handle is closed")
claim = renew_lease(
session,
self.lease_claim,
ttl_seconds=self.lease_ttl_seconds,
)
operation = session.execute(
select(RecoveryOperation)
.where(RecoveryOperation.id == self.operation_id)
.with_for_update()
).scalar_one()
self.lease_claim = claim
return operation, claim
def _verify_chain(self, session: Session) -> None:
if not verify_recovery_evidence_chain(session, self.operation_id):
raise RecoveryGuaranteeError(
"Recovery checkpoint chain verification failed"
)
def begin_durable_recovery_operation(
session_factory: SessionFactory,
*,
identity: RuntimeIdentity,
module_id: str,
operation_type: str,
idempotency_key: str,
request: dict[str, Any],
recovery_plan: RecoveryPlan,
precondition_evidence: dict[str, Any],
lease_resource_key: str,
lease_ttl_seconds: int = 300,
resource_type: str | None = None,
resource_id: str | None = None,
metadata: dict[str, Any] | None = None,
) -> DurableRecoveryStart:
if lease_ttl_seconds < 1:
raise ValueError("Recovery lease TTL must be at least one second")
with session_factory() as session:
claim = acquire_lease(
session,
installation_id=identity.installation_id,
resource_key=lease_resource_key,
holder_node_id=identity.node_id,
holder_incarnation=identity.incarnation,
ttl_seconds=lease_ttl_seconds,
metadata={
"module_id": module_id,
"operation_type": operation_type,
},
)
if claim is None:
raise RecoveryOperationBusy(
f"Another runtime owns the recovery fence for {lease_resource_key}"
)
existing = session.execute(
select(RecoveryOperation).where(
RecoveryOperation.installation_id == identity.installation_id,
RecoveryOperation.module_id == module_id,
RecoveryOperation.idempotency_key == idempotency_key,
)
).scalar_one_or_none()
operation = plan_recovery_operation(
session,
installation_id=identity.installation_id,
module_id=module_id,
operation_type=operation_type,
idempotency_key=idempotency_key,
request=request,
recovery_plan=recovery_plan,
resource_type=resource_type,
resource_id=resource_id,
lease_claim=claim,
metadata=metadata,
)
if existing is not None:
if operation.status == RecoveryStatus.SUCCEEDED.value:
release_lease(session, claim)
session.commit()
return DurableRecoveryStart(
operation_id=operation.id,
status=operation.status,
replayed=True,
operation=None,
)
session.rollback()
raise RecoveryOperationStateConflict(operation.id, operation.status)
prepare_recovery_operation(
session,
operation,
evidence=precondition_evidence,
lease_claim=claim,
)
start_recovery_operation(
session,
operation,
evidence={"lease_resource_key": lease_resource_key},
lease_claim=claim,
)
if not verify_recovery_evidence_chain(session, operation.id):
raise RecoveryGuaranteeError(
"Recovery checkpoint chain verification failed before side effects"
)
session.commit()
return DurableRecoveryStart(
operation_id=operation.id,
status=operation.status,
replayed=False,
operation=DurableRecoveryOperation(
session_factory=session_factory,
operation_id=operation.id,
lease_claim=claim,
lease_ttl_seconds=lease_ttl_seconds,
),
)
def claim_durable_recovery_operation(
session_factory: SessionFactory,
*,
identity: RuntimeIdentity,
operation_id: str,
lease_ttl_seconds: int = 300,
) -> DurableRecoveryOperation:
with session_factory() as session:
candidate = session.get(RecoveryOperation, operation_id)
if candidate is None:
raise RecoveryGuaranteeError("Recovery operation was not found")
if candidate.status in {
RecoveryStatus.SUCCEEDED.value,
RecoveryStatus.FAILED.value,
RecoveryStatus.RECOVERED.value,
RecoveryStatus.MANUAL_INTERVENTION.value,
}:
raise RecoveryOperationStateConflict(candidate.id, candidate.status)
if not candidate.lease_resource_key:
raise RecoveryGuaranteeError(
"Recovery takeover requires an operation-bound lease resource"
)
claim = acquire_lease(
session,
installation_id=identity.installation_id,
resource_key=candidate.lease_resource_key,
holder_node_id=identity.node_id,
holder_incarnation=identity.incarnation,
ttl_seconds=lease_ttl_seconds,
metadata={"recovery_operation_id": candidate.id},
)
if claim is None:
raise RecoveryOperationBusy(
f"Another runtime owns recovery operation {candidate.id}"
)
operation = session.execute(
select(RecoveryOperation)
.where(RecoveryOperation.id == operation_id)
.with_for_update()
).scalar_one()
previous_fence = {
"holder_node_id": operation.holder_node_id,
"holder_incarnation": operation.holder_incarnation,
"fence_number": operation.fencing_token,
}
operation.holder_node_id = claim.holder_node_id
operation.holder_incarnation = claim.holder_incarnation
operation.fencing_token = claim.fencing_token
session.add(operation)
record_recovery_checkpoint(
session,
operation,
kind="fence-takeover",
summary="A new runtime claimed explicit recovery authority",
evidence={
"previous_fence": previous_fence,
"new_fence_number": claim.fencing_token,
},
lease_claim=claim,
)
if operation.status == RecoveryStatus.RUNNING.value:
mode = RecoveryMode(operation.mode)
if mode == RecoveryMode.ATOMIC:
transition_recovery_operation(
session,
operation,
status=RecoveryStatus.FAILED,
kind="stale-atomic-operation",
summary="The stale database-only transaction rolled back",
evidence={"previous_fence": previous_fence},
failure_summary="Execution authority expired before commit",
lease_claim=claim,
)
elif mode in {RecoveryMode.COMPENSATION, RecoveryMode.SNAPSHOT_RESTORE}:
transition_recovery_operation(
session,
operation,
status=RecoveryStatus.RECOVERY_REQUIRED,
kind="stale-effect-requires-recovery",
summary="Execution authority expired after effects may have started",
evidence={"previous_fence": previous_fence},
failure_summary="Execution authority expired during a non-atomic operation",
lease_claim=claim,
)
else:
transition_recovery_operation(
session,
operation,
status=RecoveryStatus.OUTCOME_UNKNOWN,
kind="stale-effect-outcome-unknown",
summary="Execution authority expired after an external effect may have started",
evidence={"previous_fence": previous_fence},
failure_summary="External effect outcome requires reconciliation",
lease_claim=claim,
)
if not verify_recovery_evidence_chain(session, operation.id):
raise RecoveryGuaranteeError("Recovery checkpoint chain verification failed")
if operation.status == RecoveryStatus.FAILED.value:
release_lease(session, claim)
session.commit()
raise RecoveryOperationStateConflict(operation.id, operation.status)
session.commit()
return DurableRecoveryOperation(
session_factory=session_factory,
operation_id=operation.id,
lease_claim=claim,
lease_ttl_seconds=lease_ttl_seconds,
)
__all__ = [
"DurableRecoveryOperation",
"DurableRecoveryStart",
"RecoveryOperationBusy",
"RecoveryOperationStateConflict",
"begin_durable_recovery_operation",
"claim_durable_recovery_operation",
]
src/govoplan_core/server/default_config.py
+19
View File
@@ -12,6 +12,7 @@ from govoplan_core.db.bootstrap import bootstrap_dev_data, create_all_tables
from govoplan_core.db.session import get_database
from govoplan_core.server.config import GovoplanServerConfig
from govoplan_core.server.runtime_agent import RuntimeNodeAgent
from govoplan_core.core.runtime_coordination import RuntimeIdentity, runtime_identity
from govoplan_core.settings import Settings, settings
@@ -49,11 +50,19 @@ async def lifespan(app: FastAPI):
if registry is not None
else ()
)
configured_identity = getattr(
app.state,
"govoplan_runtime_identity",
None,
)
runtime_agent = RuntimeNodeAgent(
settings=settings,
software_version=app.version,
module_ids=module_ids,
metadata={"process": "api"},
identity=configured_identity
if isinstance(configured_identity, RuntimeIdentity)
else None,
)
await runtime_agent.start()
app.state.govoplan_runtime_agent = runtime_agent
@@ -73,6 +82,16 @@ def register_health_details(
active_settings = (
config_settings if isinstance(config_settings, Settings) else settings
)
module_ids = tuple(manifest.id for manifest in registry.manifests())
if not isinstance(
getattr(app.state, "govoplan_runtime_identity", None),
RuntimeIdentity,
):
app.state.govoplan_runtime_identity = runtime_identity(
active_settings,
software_version=app.version,
module_ids=module_ids,
)
@app.get("/health/details")
def health_details(
src/govoplan_core/server/runtime_agent.py
+18 -2
View File
@@ -17,6 +17,21 @@ from govoplan_core.db.session import get_database
logger = logging.getLogger("govoplan.runtime")
def application_runtime_identity(app: object) -> RuntimeIdentity:
"""Return the registered identity used to fence request-owned effects."""
state = getattr(app, "state", None)
agent = getattr(state, "govoplan_runtime_agent", None)
identity = getattr(agent, "identity", None) or getattr(
state,
"govoplan_runtime_identity",
None,
)
if not isinstance(identity, RuntimeIdentity):
raise RuntimeError("The application runtime identity is not available")
return identity
class RuntimeNodeAgent:
"""Register one process in the shared runtime directory and heartbeat it."""
@@ -30,9 +45,10 @@ class RuntimeNodeAgent:
node_id: str | None = None,
queues: tuple[str, ...] | None = None,
metadata: dict[str, Any] | None = None,
identity: RuntimeIdentity | None = None,
) -> None:
self.settings = settings
self.identity: RuntimeIdentity = runtime_identity(
self.identity: RuntimeIdentity = identity or runtime_identity(
settings,
software_version=software_version,
module_ids=module_ids,
@@ -126,4 +142,4 @@ class RuntimeNodeAgent:
return True
__all__ = ["RuntimeNodeAgent"]
__all__ = ["RuntimeNodeAgent", "application_runtime_identity"]