Add durable external-effect runtime identity

This commit is contained in:
2026-08-03 03:47:41 +02:00
parent 6c2940aebc
commit 01f91154e0
9 changed files with 186 additions and 7 deletions
+12
View File
@@ -9,6 +9,7 @@ import time
from celery import Celery
from celery.signals import (
heartbeat_sent,
task_prerun,
worker_process_init,
worker_ready,
worker_shutdown,
@@ -68,6 +69,7 @@ from govoplan_core.core.registry import PlatformRegistry
from govoplan_core.core.runtime import configure_runtime
from govoplan_core.core.runtime_coordination import (
RuntimeIdentity,
bind_process_runtime_identity,
heartbeat_runtime_node,
register_runtime_node,
runtime_identity,
@@ -208,6 +210,7 @@ def _worker_runtime_identity(sender: object | None = None) -> RuntimeIdentity:
role="worker",
node_id=hostname,
)
bind_process_runtime_identity(_worker_identity)
return _worker_identity
@@ -222,10 +225,19 @@ def _worker_metadata() -> dict[str, object]:
@worker_process_init.connect
def _reset_worker_process_database(**_kwargs) -> None:
global _worker_identity
# SQLAlchemy pools must not be shared across prefork child processes.
_worker_identity = None
bind_process_runtime_identity(None)
configure_database(settings.database_url, dispose_previous=True)
@task_prerun.connect
def _bind_worker_effect_identity(task=None, **_kwargs) -> None:
_worker_runtime_identity(task)
@worker_ready.connect
def _register_worker_runtime(sender=None, **_kwargs) -> None:
global _worker_consumer
+14 -3
View File
@@ -41,6 +41,7 @@ class RecoveryStatus(StrEnum):
PREPARED = "prepared"
RUNNING = "running"
SUCCEEDED = "succeeded"
REJECTED = "rejected"
FAILED = "failed"
OUTCOME_UNKNOWN = "outcome_unknown"
RECOVERY_REQUIRED = "recovery_required"
@@ -52,6 +53,7 @@ class RecoveryStatus(StrEnum):
TERMINAL_RECOVERY_STATUSES = frozenset(
{
RecoveryStatus.SUCCEEDED.value,
RecoveryStatus.REJECTED.value,
RecoveryStatus.FAILED.value,
RecoveryStatus.RECOVERED.value,
RecoveryStatus.MANUAL_INTERVENTION.value,
@@ -69,6 +71,7 @@ _TRANSITIONS: dict[str, frozenset[str]] = {
RecoveryStatus.RUNNING.value: frozenset(
{
RecoveryStatus.SUCCEEDED.value,
RecoveryStatus.REJECTED.value,
RecoveryStatus.FAILED.value,
RecoveryStatus.OUTCOME_UNKNOWN.value,
RecoveryStatus.RECOVERY_REQUIRED.value,
@@ -431,7 +434,11 @@ def transition_recovery_operation(
elif status == RecoveryStatus.RECOVERED:
locked.recovered_at = observed_at
locked.completed_at = observed_at
elif status in {RecoveryStatus.FAILED, RecoveryStatus.MANUAL_INTERVENTION}:
elif status in {
RecoveryStatus.REJECTED,
RecoveryStatus.FAILED,
RecoveryStatus.MANUAL_INTERVENTION,
}:
locked.completed_at = observed_at
session.add(locked)
record_recovery_checkpoint(
@@ -591,7 +598,11 @@ def _validate_transition_evidence(
evidence: dict[str, Any],
failure_summary: str | None,
) -> None:
if status in {RecoveryStatus.SUCCEEDED, RecoveryStatus.RECOVERED}:
if status in {
RecoveryStatus.SUCCEEDED,
RecoveryStatus.REJECTED,
RecoveryStatus.RECOVERED,
}:
checks = evidence.get("checks")
if (
evidence.get("verified") is not True
@@ -602,7 +613,7 @@ def _validate_transition_evidence(
or not checks
):
raise RecoveryGuaranteeError(
"Successful recovery transitions require verified evidence and check results"
"Verified terminal transitions require verified evidence and check results"
)
if status == RecoveryStatus.MANUAL_INTERVENTION and not failure_summary:
raise RecoveryGuaranteeError(
@@ -105,6 +105,46 @@ class DurableRecoveryOperation:
session.commit()
self.closed = True
def fail(self, *, summary: str, evidence: dict[str, Any]) -> None:
"""Finish a verified, ordinary failure that needs no recovery."""
with self.session_factory() as session:
operation, claim = self._locked_and_renewed(session)
transition_recovery_operation(
session,
operation,
status=RecoveryStatus.FAILED,
kind="verified-failure",
summary=summary,
evidence=evidence,
failure_summary=summary,
lease_claim=claim,
)
self._verify_chain(session)
release_lease(session, claim)
session.commit()
self.closed = True
def reject(self, *, summary: str, evidence: dict[str, Any]) -> None:
"""Finish an operation with a verified definitive rejection."""
with self.session_factory() as session:
operation, claim = self._locked_and_renewed(session)
transition_recovery_operation(
session,
operation,
status=RecoveryStatus.REJECTED,
kind="verified-rejection",
summary=summary,
evidence=evidence,
failure_summary=summary,
lease_claim=claim,
)
self._verify_chain(session)
release_lease(session, claim)
session.commit()
self.closed = True
def compensate(
self,
*,
@@ -339,6 +379,7 @@ def claim_durable_recovery_operation(
raise RecoveryGuaranteeError("Recovery operation was not found")
if candidate.status in {
RecoveryStatus.SUCCEEDED.value,
RecoveryStatus.REJECTED.value,
RecoveryStatus.FAILED.value,
RecoveryStatus.RECOVERED.value,
RecoveryStatus.MANUAL_INTERVENTION.value,
@@ -122,6 +122,26 @@ class RuntimeIdentity:
queues: tuple[str, ...] = ()
_process_runtime_identity: RuntimeIdentity | None = None
def bind_process_runtime_identity(identity: RuntimeIdentity | None) -> None:
"""Bind the authority identity used by effects in this OS process."""
global _process_runtime_identity
_process_runtime_identity = identity
def process_runtime_identity() -> RuntimeIdentity:
"""Return the process authority or fail before a consequential effect."""
if _process_runtime_identity is None:
raise RuntimeCoordinationError(
"No runtime identity is bound to the current process"
)
return _process_runtime_identity
@dataclass(frozen=True, slots=True)
class LeaseClaim:
installation_id: str
+6 -1
View File
@@ -12,7 +12,11 @@ from govoplan_core.db.bootstrap import bootstrap_dev_data, create_all_tables
from govoplan_core.db.session import get_database
from govoplan_core.server.config import GovoplanServerConfig
from govoplan_core.server.runtime_agent import RuntimeNodeAgent
from govoplan_core.core.runtime_coordination import RuntimeIdentity, runtime_identity
from govoplan_core.core.runtime_coordination import (
RuntimeIdentity,
bind_process_runtime_identity,
runtime_identity,
)
from govoplan_core.settings import Settings, settings
@@ -92,6 +96,7 @@ def register_health_details(
software_version=app.version,
module_ids=module_ids,
)
bind_process_runtime_identity(app.state.govoplan_runtime_identity)
@app.get("/health/details")
def health_details(