Add durable external-effect runtime identity
This commit is contained in:
@@ -9,6 +9,7 @@ import time
|
||||
from celery import Celery
|
||||
from celery.signals import (
|
||||
heartbeat_sent,
|
||||
task_prerun,
|
||||
worker_process_init,
|
||||
worker_ready,
|
||||
worker_shutdown,
|
||||
@@ -68,6 +69,7 @@ from govoplan_core.core.registry import PlatformRegistry
|
||||
from govoplan_core.core.runtime import configure_runtime
|
||||
from govoplan_core.core.runtime_coordination import (
|
||||
RuntimeIdentity,
|
||||
bind_process_runtime_identity,
|
||||
heartbeat_runtime_node,
|
||||
register_runtime_node,
|
||||
runtime_identity,
|
||||
@@ -208,6 +210,7 @@ def _worker_runtime_identity(sender: object | None = None) -> RuntimeIdentity:
|
||||
role="worker",
|
||||
node_id=hostname,
|
||||
)
|
||||
bind_process_runtime_identity(_worker_identity)
|
||||
return _worker_identity
|
||||
|
||||
|
||||
@@ -222,10 +225,19 @@ def _worker_metadata() -> dict[str, object]:
|
||||
|
||||
@worker_process_init.connect
|
||||
def _reset_worker_process_database(**_kwargs) -> None:
|
||||
global _worker_identity
|
||||
|
||||
# SQLAlchemy pools must not be shared across prefork child processes.
|
||||
_worker_identity = None
|
||||
bind_process_runtime_identity(None)
|
||||
configure_database(settings.database_url, dispose_previous=True)
|
||||
|
||||
|
||||
@task_prerun.connect
|
||||
def _bind_worker_effect_identity(task=None, **_kwargs) -> None:
|
||||
_worker_runtime_identity(task)
|
||||
|
||||
|
||||
@worker_ready.connect
|
||||
def _register_worker_runtime(sender=None, **_kwargs) -> None:
|
||||
global _worker_consumer
|
||||
|
||||
@@ -41,6 +41,7 @@ class RecoveryStatus(StrEnum):
|
||||
PREPARED = "prepared"
|
||||
RUNNING = "running"
|
||||
SUCCEEDED = "succeeded"
|
||||
REJECTED = "rejected"
|
||||
FAILED = "failed"
|
||||
OUTCOME_UNKNOWN = "outcome_unknown"
|
||||
RECOVERY_REQUIRED = "recovery_required"
|
||||
@@ -52,6 +53,7 @@ class RecoveryStatus(StrEnum):
|
||||
TERMINAL_RECOVERY_STATUSES = frozenset(
|
||||
{
|
||||
RecoveryStatus.SUCCEEDED.value,
|
||||
RecoveryStatus.REJECTED.value,
|
||||
RecoveryStatus.FAILED.value,
|
||||
RecoveryStatus.RECOVERED.value,
|
||||
RecoveryStatus.MANUAL_INTERVENTION.value,
|
||||
@@ -69,6 +71,7 @@ _TRANSITIONS: dict[str, frozenset[str]] = {
|
||||
RecoveryStatus.RUNNING.value: frozenset(
|
||||
{
|
||||
RecoveryStatus.SUCCEEDED.value,
|
||||
RecoveryStatus.REJECTED.value,
|
||||
RecoveryStatus.FAILED.value,
|
||||
RecoveryStatus.OUTCOME_UNKNOWN.value,
|
||||
RecoveryStatus.RECOVERY_REQUIRED.value,
|
||||
@@ -431,7 +434,11 @@ def transition_recovery_operation(
|
||||
elif status == RecoveryStatus.RECOVERED:
|
||||
locked.recovered_at = observed_at
|
||||
locked.completed_at = observed_at
|
||||
elif status in {RecoveryStatus.FAILED, RecoveryStatus.MANUAL_INTERVENTION}:
|
||||
elif status in {
|
||||
RecoveryStatus.REJECTED,
|
||||
RecoveryStatus.FAILED,
|
||||
RecoveryStatus.MANUAL_INTERVENTION,
|
||||
}:
|
||||
locked.completed_at = observed_at
|
||||
session.add(locked)
|
||||
record_recovery_checkpoint(
|
||||
@@ -591,7 +598,11 @@ def _validate_transition_evidence(
|
||||
evidence: dict[str, Any],
|
||||
failure_summary: str | None,
|
||||
) -> None:
|
||||
if status in {RecoveryStatus.SUCCEEDED, RecoveryStatus.RECOVERED}:
|
||||
if status in {
|
||||
RecoveryStatus.SUCCEEDED,
|
||||
RecoveryStatus.REJECTED,
|
||||
RecoveryStatus.RECOVERED,
|
||||
}:
|
||||
checks = evidence.get("checks")
|
||||
if (
|
||||
evidence.get("verified") is not True
|
||||
@@ -602,7 +613,7 @@ def _validate_transition_evidence(
|
||||
or not checks
|
||||
):
|
||||
raise RecoveryGuaranteeError(
|
||||
"Successful recovery transitions require verified evidence and check results"
|
||||
"Verified terminal transitions require verified evidence and check results"
|
||||
)
|
||||
if status == RecoveryStatus.MANUAL_INTERVENTION and not failure_summary:
|
||||
raise RecoveryGuaranteeError(
|
||||
|
||||
@@ -105,6 +105,46 @@ class DurableRecoveryOperation:
|
||||
session.commit()
|
||||
self.closed = True
|
||||
|
||||
def fail(self, *, summary: str, evidence: dict[str, Any]) -> None:
|
||||
"""Finish a verified, ordinary failure that needs no recovery."""
|
||||
|
||||
with self.session_factory() as session:
|
||||
operation, claim = self._locked_and_renewed(session)
|
||||
transition_recovery_operation(
|
||||
session,
|
||||
operation,
|
||||
status=RecoveryStatus.FAILED,
|
||||
kind="verified-failure",
|
||||
summary=summary,
|
||||
evidence=evidence,
|
||||
failure_summary=summary,
|
||||
lease_claim=claim,
|
||||
)
|
||||
self._verify_chain(session)
|
||||
release_lease(session, claim)
|
||||
session.commit()
|
||||
self.closed = True
|
||||
|
||||
def reject(self, *, summary: str, evidence: dict[str, Any]) -> None:
|
||||
"""Finish an operation with a verified definitive rejection."""
|
||||
|
||||
with self.session_factory() as session:
|
||||
operation, claim = self._locked_and_renewed(session)
|
||||
transition_recovery_operation(
|
||||
session,
|
||||
operation,
|
||||
status=RecoveryStatus.REJECTED,
|
||||
kind="verified-rejection",
|
||||
summary=summary,
|
||||
evidence=evidence,
|
||||
failure_summary=summary,
|
||||
lease_claim=claim,
|
||||
)
|
||||
self._verify_chain(session)
|
||||
release_lease(session, claim)
|
||||
session.commit()
|
||||
self.closed = True
|
||||
|
||||
def compensate(
|
||||
self,
|
||||
*,
|
||||
@@ -339,6 +379,7 @@ def claim_durable_recovery_operation(
|
||||
raise RecoveryGuaranteeError("Recovery operation was not found")
|
||||
if candidate.status in {
|
||||
RecoveryStatus.SUCCEEDED.value,
|
||||
RecoveryStatus.REJECTED.value,
|
||||
RecoveryStatus.FAILED.value,
|
||||
RecoveryStatus.RECOVERED.value,
|
||||
RecoveryStatus.MANUAL_INTERVENTION.value,
|
||||
|
||||
@@ -122,6 +122,26 @@ class RuntimeIdentity:
|
||||
queues: tuple[str, ...] = ()
|
||||
|
||||
|
||||
_process_runtime_identity: RuntimeIdentity | None = None
|
||||
|
||||
|
||||
def bind_process_runtime_identity(identity: RuntimeIdentity | None) -> None:
|
||||
"""Bind the authority identity used by effects in this OS process."""
|
||||
|
||||
global _process_runtime_identity
|
||||
_process_runtime_identity = identity
|
||||
|
||||
|
||||
def process_runtime_identity() -> RuntimeIdentity:
|
||||
"""Return the process authority or fail before a consequential effect."""
|
||||
|
||||
if _process_runtime_identity is None:
|
||||
raise RuntimeCoordinationError(
|
||||
"No runtime identity is bound to the current process"
|
||||
)
|
||||
return _process_runtime_identity
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class LeaseClaim:
|
||||
installation_id: str
|
||||
|
||||
@@ -12,7 +12,11 @@ from govoplan_core.db.bootstrap import bootstrap_dev_data, create_all_tables
|
||||
from govoplan_core.db.session import get_database
|
||||
from govoplan_core.server.config import GovoplanServerConfig
|
||||
from govoplan_core.server.runtime_agent import RuntimeNodeAgent
|
||||
from govoplan_core.core.runtime_coordination import RuntimeIdentity, runtime_identity
|
||||
from govoplan_core.core.runtime_coordination import (
|
||||
RuntimeIdentity,
|
||||
bind_process_runtime_identity,
|
||||
runtime_identity,
|
||||
)
|
||||
from govoplan_core.settings import Settings, settings
|
||||
|
||||
|
||||
@@ -92,6 +96,7 @@ def register_health_details(
|
||||
software_version=app.version,
|
||||
module_ids=module_ids,
|
||||
)
|
||||
bind_process_runtime_identity(app.state.govoplan_runtime_identity)
|
||||
|
||||
@app.get("/health/details")
|
||||
def health_details(
|
||||
|
||||
Reference in New Issue
Block a user