GovOPlaN/govoplan-core
2
0
Fork 0
Code Issues 4 Pull Requests Actions Packages Projects Releases Wiki Activity

Implement durable module recovery operations

Browse Source
This commit is contained in:
zemion
2026-08-03 03:07:42 +02:00
parent 21c1fa49b6
commit bca6a7c8aa
6 changed files with 744 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_api_smoke.py
+54 -9
View File
@@ -44,6 +44,11 @@ from govoplan_core.db.migrations import alembic_config
from govoplan_core.db.session import configure_database, set_database
from govoplan_core.core.change_sequence import decode_sequence_watermark, prune_sequence_entries
from govoplan_core.core.pagination import encode_keyset_cursor, keyset_query_fingerprint
from govoplan_core.core.recovery import (
RecoveryOperation,
RecoveryStatus,
verify_recovery_evidence_chain,
)
from govoplan_core.tenancy.scope import create_scope_tables, scope_registry
from govoplan_access.backend.permissions.catalog import permission_catalog as access_permission_catalog
@@ -101,6 +106,14 @@ class ApiSmokeTests(unittest.TestCase):
payload = response.json()
return {"Authorization": f"Bearer {payload['access_token']}"}, payload
def _stored_campaign_eml(self, job: object) -> bytes:
from govoplan_files.backend.storage.backends import get_storage_backend
self.assertIsNone(getattr(job, "eml_local_path", None))
storage_key = getattr(job, "eml_storage_key", None)
self.assertTrue(storage_key)
return get_storage_backend().get_bytes(str(storage_key))
def _create_test_mail_profile(
self,
headers: dict[str, str],
@@ -3304,6 +3317,29 @@ class ApiSmokeTests(unittest.TestCase):
json={"write_eml": False},
)
self.assertEqual(built.status_code, 200, built.text)
replayed_build = self.client.post(
f"/api/v1/campaigns/versions/{version_id}/build",
headers=headers,
json={"write_eml": False},
)
self.assertEqual(replayed_build.status_code, 200, replayed_build.text)
with SessionLocal() as session:
operations = (
session.query(RecoveryOperation)
.filter(
RecoveryOperation.module_id == "campaigns",
RecoveryOperation.resource_id == version_id,
)
.all()
)
self.assertEqual(1, len(operations))
self.assertEqual(
RecoveryStatus.SUCCEEDED.value,
operations[0].status,
)
self.assertTrue(
verify_recovery_evidence_chain(session, operations[0].id)
)
self.assertEqual(built.json()["built_count"], 1)
mocked = self.client.post(
@@ -3742,7 +3778,8 @@ class ApiSmokeTests(unittest.TestCase):
with SessionLocal() as session:
job = session.query(CampaignJob).filter(CampaignJob.campaign_version_id == version_id).one()
self.assertTrue(job.eml_local_path)
self.assertIsNone(job.eml_local_path)
self.assertTrue(job.eml_storage_key)
built_use = (
session.query(CampaignAttachmentUse)
.filter(
@@ -3979,7 +4016,9 @@ class ApiSmokeTests(unittest.TestCase):
job = session.query(CampaignJob).filter(CampaignJob.campaign_id == campaign_id).one()
self.assertEqual([item["email"] for item in job.resolved_recipients["from_all"]], ["local-from@example.org"])
self.assertEqual([item["email"] for item in job.resolved_recipients["to"]], ["global-to@example.org", "local-to@example.org"])
message = BytesParser(policy=policy.default).parsebytes(Path(job.eml_local_path).read_bytes())
message = BytesParser(policy=policy.default).parsebytes(
self._stored_campaign_eml(job)
)
self.assertIsNone(message["Sender"])
self.assertEqual([address.addr_spec for address in message["From"].addresses], ["local-from@example.org"])
self.assertEqual([address.addr_spec for address in message["To"].addresses], ["global-to@example.org", "local-to@example.org"])
@@ -4229,8 +4268,9 @@ class ApiSmokeTests(unittest.TestCase):
with SessionLocal() as session:
job = session.query(CampaignJob).filter(CampaignJob.campaign_id == campaign_id).one()
eml_path = Path(job.eml_local_path)
message = BytesParser(policy=policy.default).parsebytes(eml_path.read_bytes())
message = BytesParser(policy=policy.default).parsebytes(
self._stored_campaign_eml(job)
)
uses = (
session.query(CampaignAttachmentUse)
.filter(
@@ -4337,8 +4377,7 @@ class ApiSmokeTests(unittest.TestCase):
with SessionLocal() as session:
job = session.query(CampaignJob).filter(CampaignJob.campaign_version_id == version_id).one()
self.assertIsNotNone(job.eml_local_path)
generated_eml = Path(job.eml_local_path).read_bytes()
generated_eml = self._stored_campaign_eml(job)
sent = self.client.post(
f"/api/v1/campaigns/{campaign_id}/send-now",
@@ -4368,13 +4407,19 @@ class ApiSmokeTests(unittest.TestCase):
)
from govoplan_campaign.backend.db.models import CampaignJob
from govoplan_files.backend.storage.backends import get_storage_backend
from govoplan_mail.backend.dev.mock_mailbox import list_records
with SessionLocal() as session:
job = session.query(CampaignJob).filter(CampaignJob.campaign_version_id == version_id).one()
self.assertIsNotNone(job.eml_local_path)
eml_path = Path(job.eml_local_path)
eml_path.write_bytes(eml_path.read_bytes() + b"\r\nX-Tampered: true\r\n")
storage_key = job.eml_storage_key
generated_eml = self._stored_campaign_eml(job)
assert storage_key is not None
get_storage_backend().put_bytes(
storage_key,
generated_eml + b"\r\nX-Tampered: true\r\n",
content_type="message/rfc822",
)
sent = self.client.post(
f"/api/v1/campaigns/{campaign_id}/send-now",
tests/test_recovery_runtime.py
+182
View File
@@ -0,0 +1,182 @@
from __future__ import annotations
from datetime import datetime, timedelta, timezone
from sqlalchemy import create_engine, select
from sqlalchemy.orm import sessionmaker
import pytest
from govoplan_core.core.recovery import (
RecoveryCheckpoint,
RecoveryGuaranteeError,
RecoveryMode,
RecoveryOperation,
RecoveryPlan,
RecoveryStatus,
verify_recovery_evidence_chain,
)
from govoplan_core.core.recovery_runtime import (
RecoveryOperationBusy,
RecoveryOperationStateConflict,
begin_durable_recovery_operation,
claim_durable_recovery_operation,
)
from govoplan_core.core.runtime_coordination import DistributedLease, RuntimeIdentity
from govoplan_core.db.base import Base
def _fixture():
engine = create_engine("sqlite+pysqlite:///:memory:")
Base.metadata.create_all(
engine,
tables=[
DistributedLease.__table__,
RecoveryOperation.__table__,
RecoveryCheckpoint.__table__,
],
)
return engine, sessionmaker(bind=engine, expire_on_commit=False)
def _identity(node: str, incarnation: str) -> RuntimeIdentity:
return RuntimeIdentity(
installation_id="installation-1",
node_id=node,
incarnation=incarnation,
role="worker",
software_version="test",
composition_hash="a" * 64,
)
def _start(factory, identity, *, key: str = "build-1"):
return begin_durable_recovery_operation(
factory,
identity=identity,
module_id="campaigns",
operation_type="build-artifacts",
idempotency_key=key,
request={"version_id": "version-1", "write_eml": True},
recovery_plan=RecoveryPlan(
mode=RecoveryMode.COMPENSATION,
preconditions=("validated version is locked",),
compensation_steps=("delete build object prefix",),
verification_steps=("compare database and object manifests",),
),
precondition_evidence={"validation_sha256": "b" * 64},
lease_resource_key="campaign:build:version-1",
resource_type="campaign_version",
resource_id="version-1",
)
def test_durable_operation_commits_before_caller_effect_and_replays_success() -> None:
engine, factory = _fixture()
try:
started = _start(factory, _identity("worker-1", "incarnation-1"))
assert started.operation is not None
with factory() as session:
persisted = session.get(RecoveryOperation, started.operation_id)
assert persisted is not None
assert persisted.status == RecoveryStatus.RUNNING.value
assert persisted.checkpoint_count == 3
started.operation.checkpoint(
kind="object-prefix-reserved",
summary="Build object prefix reserved",
evidence={"prefix": "campaign-artifacts/build-1/"},
)
started.operation.succeed(
evidence={
"verified": True,
"checks": {"database_manifest": "matched", "object_manifest": "matched"},
}
)
replay = _start(factory, _identity("worker-2", "incarnation-2"))
assert replay.replayed is True
assert replay.operation is None
with factory() as session:
assert verify_recovery_evidence_chain(session, started.operation_id)
finally:
engine.dispose()
def test_same_fence_cannot_start_duplicate_running_operation() -> None:
engine, factory = _fixture()
try:
identity = _identity("worker-1", "incarnation-1")
started = _start(factory, identity)
with pytest.raises(RecoveryOperationStateConflict, match="already running"):
_start(factory, identity)
assert started.operation is not None
started.operation.release_unresolved()
finally:
engine.dispose()
def test_other_runtime_cannot_use_an_active_fence() -> None:
engine, factory = _fixture()
try:
started = _start(factory, _identity("worker-1", "incarnation-1"))
with pytest.raises(RecoveryOperationBusy):
_start(factory, _identity("worker-2", "incarnation-2"), key="build-2")
assert started.operation is not None
started.operation.release_unresolved()
finally:
engine.dispose()
def test_expired_crash_fence_is_taken_over_as_recovery_required() -> None:
engine, factory = _fixture()
try:
started = _start(factory, _identity("worker-1", "incarnation-1"))
assert started.operation is not None
with factory() as session:
lease = session.execute(select(DistributedLease)).scalar_one()
lease.expires_at = datetime.now(timezone.utc) - timedelta(seconds=1)
session.add(lease)
session.commit()
recovery = claim_durable_recovery_operation(
factory,
identity=_identity("worker-2", "incarnation-2"),
operation_id=started.operation_id,
)
with factory() as session:
operation = session.get(RecoveryOperation, started.operation_id)
assert operation is not None
assert operation.status == RecoveryStatus.RECOVERY_REQUIRED.value
assert operation.fencing_token == 2
assert verify_recovery_evidence_chain(session, operation.id)
recovery.compensate(
failure_summary="worker stopped during object publication",
failure_evidence={"object_prefix": "campaign-artifacts/build-1/"},
recovery_evidence={
"verified": True,
"checks": {"object_prefix_empty": True},
},
)
finally:
engine.dispose()
def test_tampered_checkpoint_blocks_verified_success() -> None:
engine, factory = _fixture()
try:
started = _start(factory, _identity("worker-1", "incarnation-1"))
assert started.operation is not None
with factory() as session:
checkpoint = session.execute(
select(RecoveryCheckpoint).order_by(RecoveryCheckpoint.sequence)
).scalars().first()
assert checkpoint is not None
checkpoint.summary = "tampered"
session.add(checkpoint)
session.commit()
with pytest.raises(RecoveryGuaranteeError, match="chain verification failed"):
started.operation.succeed(
evidence={"verified": True, "checks": {"objects": "matched"}}
)
finally:
engine.dispose()