GovOPlaN/govoplan-core
2
0
Fork 0
Code Issues 4 Pull Requests Actions Packages Projects Releases Wiki Activity

Fence and reconcile module lifecycle effects

Browse Source
This commit is contained in:
zemion
2026-08-03 07:02:07 +02:00
parent b962f6756e
commit 729b84d3af
12 changed files with 1166 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_module_system.py
+105 -3
View File
@@ -20,7 +20,7 @@ from pathlib import Path
from types import SimpleNamespace
from unittest.mock import patch
from sqlalchemy import Column, Integer, MetaData, Table, create_engine, insert, inspect
from sqlalchemy import Column, Integer, MetaData, Table, create_engine, insert, inspect, select
from sqlalchemy.orm import Session
# Keep the default app import side effect from bootstrapping a development DB.
@@ -93,6 +93,13 @@ from govoplan_core.core.configuration_packages import (
validate_configuration_package_catalog,
)
from govoplan_core.core.module_license import issue_module_license, module_license_decision, module_license_diagnostics, validate_module_license
from govoplan_core.core.recovery import (
RecoveryCheckpoint,
RecoveryOperation,
RecoveryStatus,
verify_recovery_evidence_chain,
)
from govoplan_core.core.runtime_coordination import DistributedLease
from govoplan_core.core.module_package_catalog import (
module_package_catalog,
record_module_package_catalog_acceptance,
@@ -2607,7 +2614,15 @@ finally:
settings = _settings(root)
configure_database(settings.database_url)
database = get_database()
Base.metadata.create_all(bind=database.engine, tables=[SystemSettings.__table__])
Base.metadata.create_all(
bind=database.engine,
tables=[
SystemSettings.__table__,
DistributedLease.__table__,
RecoveryOperation.__table__,
RecoveryCheckpoint.__table__,
],
)
metadata = MetaData()
table = Table("retirement_example", metadata, Column("id", Integer, primary_key=True))
metadata.create_all(bind=database.engine)
@@ -2658,6 +2673,9 @@ finally:
database_url=settings.database_url,
runtime_dir=root / "installer",
)
recovery = session.execute(select(RecoveryOperation)).scalar_one()
self.assertEqual(RecoveryStatus.SUCCEEDED.value, recovery.status)
self.assertTrue(verify_recovery_evidence_chain(session, recovery.id))
self.assertEqual("applied", result.status)
self.assertFalse(inspect(database.engine).has_table("retirement_example"))
@@ -2813,7 +2831,15 @@ finally:
settings = _settings(root)
configure_database(settings.database_url)
database = get_database()
Base.metadata.create_all(bind=database.engine, tables=[SystemSettings.__table__])
Base.metadata.create_all(
bind=database.engine,
tables=[
SystemSettings.__table__,
DistributedLease.__table__,
RecoveryOperation.__table__,
RecoveryCheckpoint.__table__,
],
)
def fake_run(*_args, **kwargs):
argv = tuple(_args[0]) if _args else ()
@@ -2846,11 +2872,78 @@ finally:
restored_desired = saved_desired_enabled_modules(session, ("tenancy", "access"))
restored_plan = saved_module_install_plan(session)
recovery = session.execute(select(RecoveryOperation)).scalar_one()
self.assertEqual(RecoveryStatus.RECOVERED.value, recovery.status)
self.assertTrue(verify_recovery_evidence_chain(session, recovery.id))
self.assertEqual("rolled-back", result.status)
self.assertEqual(("tenancy", "access"), restored_desired)
self.assertEqual(("planned",), tuple(item.status for item in restored_plan.items))
def test_module_installer_blocks_after_unresolved_package_effect(self) -> None:
root = Path(tempfile.mkdtemp(prefix="govoplan-installer-unresolved-", dir=_TEST_ROOT))
settings = _settings(root)
configure_database(settings.database_url)
database = get_database()
Base.metadata.create_all(
bind=database.engine,
tables=[
SystemSettings.__table__,
DistributedLease.__table__,
RecoveryOperation.__table__,
RecoveryCheckpoint.__table__,
],
)
def fail_package_install(*args, **_kwargs):
argv = tuple(args[0]) if args else ()
if any("govoplan-example==0.1.4" in str(item) for item in argv):
return SimpleNamespace(returncode=1, stdout="", stderr="install failed")
return SimpleNamespace(returncode=0, stdout="", stderr="")
with database.session() as session:
save_maintenance_mode(session, MaintenanceMode(enabled=True))
plan = save_module_install_plan(session, [{
"module_id": "example",
"action": "install",
"python_package": "govoplan-example",
"python_ref": "govoplan-example==0.1.4",
}])
session.commit()
with patch(
"govoplan_core.core.module_installer.subprocess.run",
side_effect=fail_package_install,
):
result = run_module_install_plan(
session=session,
plan=plan,
available=available_module_manifests(),
current_enabled=("tenancy", "access"),
desired_enabled=("tenancy", "access"),
database_url=settings.database_url,
runtime_dir=root / "installer",
)
self.assertEqual("failed", result.status)
recovery = session.execute(select(RecoveryOperation)).scalar_one()
self.assertEqual(RecoveryStatus.RECOVERY_REQUIRED.value, recovery.status)
self.assertTrue(verify_recovery_evidence_chain(session, recovery.id))
with self.assertRaisesRegex(
module_installer_module.ModuleInstallerError,
"already recovery_required",
):
run_module_install_plan(
session=session,
plan=plan,
available=available_module_manifests(),
current_enabled=("tenancy", "access"),
desired_enabled=("tenancy", "access"),
database_url=settings.database_url,
runtime_dir=root / "installer",
)
def test_module_installer_external_database_backup_command_is_recorded(self) -> None:
root = Path(tempfile.mkdtemp(prefix="govoplan-installer-external-backup-", dir=_TEST_ROOT))
settings = _settings(root)
@@ -4216,6 +4309,15 @@ finally:
app, _settings_obj = self._app_for_modules(())
lifecycle = getattr(app.state, "govoplan_lifecycle", None)
self.assertIsNotNone(lifecycle)
database = get_database()
Base.metadata.create_all(
bind=database.engine,
tables=[
DistributedLease.__table__,
RecoveryOperation.__table__,
RecoveryCheckpoint.__table__,
],
)
with TestClient(app) as client:
response = client.get("/api/v1/platform/modules")
tests/test_recovery_runtime.py
+38 -1
View File
@@ -50,7 +50,13 @@ def _identity(node: str, incarnation: str) -> RuntimeIdentity:
)
def _start(factory, identity, *, key: str = "build-1"):
def _start(
factory,
identity,
*,
key: str = "build-1",
block_unresolved_resource: bool = False,
):
return begin_durable_recovery_operation(
factory,
identity=identity,
@@ -68,6 +74,7 @@ def _start(factory, identity, *, key: str = "build-1"):
lease_resource_key="campaign:build:version-1",
resource_type="campaign_version",
resource_id="version-1",
block_unresolved_resource=block_unresolved_resource,
)
@@ -273,6 +280,36 @@ def test_other_runtime_cannot_use_an_active_fence() -> None:
engine.dispose()
def test_unresolved_predecessor_can_block_new_effects_on_same_resource() -> None:
engine, factory = _fixture()
try:
started = _start(
factory,
_identity("worker-1", "incarnation-1"),
block_unresolved_resource=True,
)
assert started.operation is not None
started.operation.unresolved(
status=RecoveryStatus.OUTCOME_UNKNOWN,
summary="Provider outcome is unknown",
evidence={"request_sent": True},
failure_summary="Reconcile before retry",
)
with pytest.raises(
RecoveryOperationStateConflict,
match="outcome_unknown",
):
_start(
factory,
_identity("worker-2", "incarnation-2"),
key="build-2",
block_unresolved_resource=True,
)
finally:
engine.dispose()
def test_expired_crash_fence_is_taken_over_as_recovery_required() -> None:
engine, factory = _fixture()
try:
tests/test_workflow_contribution_lifecycle.py
+28 -1
View File
@@ -10,11 +10,18 @@ from govoplan_core.core.modules import (
ModuleManifest,
)
from govoplan_core.core.registry import PlatformRegistry
from govoplan_core.core.recovery import RecoveryCheckpoint, RecoveryOperation
from govoplan_core.core.runtime_coordination import (
DistributedLease,
RuntimeIdentity,
bind_process_runtime_identity,
)
from govoplan_core.core.workflows import (
CAPABILITY_WORKFLOW_DEFINITION_CONTRIBUTIONS,
WorkflowDefinitionContribution,
)
from govoplan_core.db.session import configure_database, reset_database
from govoplan_core.db.base import Base
from govoplan_core.db.session import configure_database, get_database, reset_database
class _ContributionProvider:
@@ -30,8 +37,28 @@ class _ContributionProvider:
class WorkflowContributionLifecycleTests(unittest.TestCase):
def setUp(self) -> None:
configure_database("sqlite:///:memory:")
database = get_database()
Base.metadata.create_all(
bind=database.engine,
tables=[
DistributedLease.__table__,
RecoveryOperation.__table__,
RecoveryCheckpoint.__table__,
],
)
bind_process_runtime_identity(
RuntimeIdentity(
installation_id="test-installation",
node_id="test-node",
incarnation="test-incarnation",
role="test",
software_version="test",
composition_hash="0" * 64,
)
)
def tearDown(self) -> None:
bind_process_runtime_identity(None)
reset_database(dispose=True)
def test_active_graph_change_reconciles_module_workflow_baselines(self) -> None: