feat: add institutional governance and recovery contracts
This commit is contained in:
@@ -3,6 +3,7 @@ from __future__ import annotations
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
from fastapi import Depends, FastAPI
|
||||
from fastapi import HTTPException, status
|
||||
from sqlalchemy.engine import make_url
|
||||
|
||||
from govoplan_core.auth import ApiPrincipal, require_scope
|
||||
@@ -10,6 +11,7 @@ from govoplan_core.core.registry import PlatformRegistry
|
||||
from govoplan_core.db.bootstrap import bootstrap_dev_data, create_all_tables
|
||||
from govoplan_core.db.session import get_database
|
||||
from govoplan_core.server.config import GovoplanServerConfig
|
||||
from govoplan_core.server.runtime_agent import RuntimeNodeAgent
|
||||
from govoplan_core.settings import Settings, settings
|
||||
|
||||
|
||||
@@ -41,15 +43,36 @@ async def lifespan(app: FastAPI):
|
||||
"reconcile_workflow_definitions",
|
||||
):
|
||||
lifecycle.reconcile_workflow_definitions()
|
||||
yield
|
||||
registry = getattr(app.state, "govoplan_registry", None)
|
||||
module_ids = (
|
||||
tuple(manifest.id for manifest in registry.manifests())
|
||||
if registry is not None
|
||||
else ()
|
||||
)
|
||||
runtime_agent = RuntimeNodeAgent(
|
||||
settings=settings,
|
||||
software_version=app.version,
|
||||
module_ids=module_ids,
|
||||
metadata={"process": "api"},
|
||||
)
|
||||
await runtime_agent.start()
|
||||
app.state.govoplan_runtime_agent = runtime_agent
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
await runtime_agent.stop()
|
||||
|
||||
|
||||
def _cors_origins(value: str) -> list[str]:
|
||||
return [item.strip() for item in value.split(",") if item.strip()]
|
||||
|
||||
|
||||
def register_health_details(app: FastAPI, registry: PlatformRegistry, config_settings: object | None) -> None:
|
||||
active_settings = config_settings if isinstance(config_settings, Settings) else settings
|
||||
def register_health_details(
|
||||
app: FastAPI, registry: PlatformRegistry, config_settings: object | None
|
||||
) -> None:
|
||||
active_settings = (
|
||||
config_settings if isinstance(config_settings, Settings) else settings
|
||||
)
|
||||
|
||||
@app.get("/health/details")
|
||||
def health_details(
|
||||
@@ -63,13 +86,39 @@ def register_health_details(app: FastAPI, registry: PlatformRegistry, config_set
|
||||
"modules": [manifest.id for manifest in registry.manifests()],
|
||||
"storage": {
|
||||
"backend": active_settings.file_storage_backend,
|
||||
"local_root": active_settings.file_storage_local_root if active_settings.file_storage_backend == "local" else None,
|
||||
"endpoint": active_settings.file_storage_s3_endpoint_url or active_settings.s3_endpoint_url,
|
||||
"bucket": active_settings.file_storage_s3_bucket or active_settings.s3_bucket,
|
||||
"region": active_settings.file_storage_s3_region or active_settings.s3_region,
|
||||
"local_root": active_settings.file_storage_local_root
|
||||
if active_settings.file_storage_backend == "local"
|
||||
else None,
|
||||
"endpoint": active_settings.file_storage_s3_endpoint_url
|
||||
or active_settings.s3_endpoint_url,
|
||||
"bucket": active_settings.file_storage_s3_bucket
|
||||
or active_settings.s3_bucket,
|
||||
"region": active_settings.file_storage_s3_region
|
||||
or active_settings.s3_region,
|
||||
},
|
||||
}
|
||||
|
||||
@app.get("/health/ready")
|
||||
def health_ready():
|
||||
runtime_agent = getattr(app.state, "govoplan_runtime_agent", None)
|
||||
if runtime_agent is not None and not runtime_agent.coordination_healthy:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
detail={
|
||||
"status": "coordination_unavailable",
|
||||
"node_id": runtime_agent.identity.node_id,
|
||||
},
|
||||
)
|
||||
if runtime_agent is not None and runtime_agent.draining:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
detail={
|
||||
"status": "draining",
|
||||
"node_id": runtime_agent.identity.node_id,
|
||||
},
|
||||
)
|
||||
return {"status": "ready"}
|
||||
|
||||
|
||||
def get_server_config() -> GovoplanServerConfig:
|
||||
return GovoplanServerConfig(
|
||||
|
||||
@@ -168,6 +168,15 @@ def create_platform_router(settings: object | None = None) -> APIRouter:
|
||||
"dependencies": list(manifest.dependencies),
|
||||
"optional_dependencies": list(manifest.optional_dependencies),
|
||||
"enabled": True,
|
||||
"architecture": (
|
||||
manifest.architecture.to_dict()
|
||||
if manifest.architecture is not None
|
||||
else None
|
||||
),
|
||||
"external_providers": [
|
||||
declaration.to_dict()
|
||||
for declaration in manifest.external_providers
|
||||
],
|
||||
"runtime_ui_capabilities": _runtime_ui_capabilities(manifest.id, settings, registry),
|
||||
"nav": [_nav_item_payload(item, manifest.id) for item in manifest.nav_items],
|
||||
"frontend": _frontend_payload(manifest),
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from govoplan_core.core.runtime_coordination import (
|
||||
RuntimeIdentity,
|
||||
heartbeat_runtime_node,
|
||||
register_runtime_node,
|
||||
runtime_identity,
|
||||
stop_runtime_node,
|
||||
)
|
||||
from govoplan_core.db.session import get_database
|
||||
|
||||
|
||||
logger = logging.getLogger("govoplan.runtime")
|
||||
|
||||
|
||||
class RuntimeNodeAgent:
|
||||
"""Register one process in the shared runtime directory and heartbeat it."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
settings: object,
|
||||
software_version: str,
|
||||
module_ids: tuple[str, ...],
|
||||
role: str | None = None,
|
||||
node_id: str | None = None,
|
||||
queues: tuple[str, ...] | None = None,
|
||||
metadata: dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
self.settings = settings
|
||||
self.identity: RuntimeIdentity = runtime_identity(
|
||||
settings,
|
||||
software_version=software_version,
|
||||
module_ids=module_ids,
|
||||
role=role,
|
||||
node_id=node_id,
|
||||
queues=queues,
|
||||
)
|
||||
self.metadata = dict(metadata or {})
|
||||
self.draining = False
|
||||
self.coordination_healthy = False
|
||||
self._task: asyncio.Task[None] | None = None
|
||||
self._stopping = False
|
||||
|
||||
@property
|
||||
def heartbeat_seconds(self) -> int:
|
||||
return max(
|
||||
2,
|
||||
int(getattr(self.settings, "runtime_heartbeat_seconds", 15)),
|
||||
)
|
||||
|
||||
async def start(self) -> None:
|
||||
await asyncio.to_thread(self._register)
|
||||
self._task = asyncio.create_task(
|
||||
self._heartbeat_loop(),
|
||||
name=f"govoplan-runtime-heartbeat:{self.identity.node_id}",
|
||||
)
|
||||
|
||||
async def stop(self) -> None:
|
||||
self._stopping = True
|
||||
task = self._task
|
||||
self._task = None
|
||||
if task is not None:
|
||||
task.cancel()
|
||||
try:
|
||||
await task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
try:
|
||||
await asyncio.to_thread(self._mark_stopped)
|
||||
except Exception: # noqa: BLE001 - shutdown must continue
|
||||
logger.exception(
|
||||
"runtime node stop marker failed node_id=%s",
|
||||
self.identity.node_id,
|
||||
)
|
||||
|
||||
def _register(self) -> None:
|
||||
with get_database().SessionLocal() as session:
|
||||
node = register_runtime_node(
|
||||
session,
|
||||
self.identity,
|
||||
metadata=self.metadata,
|
||||
)
|
||||
session.commit()
|
||||
self.draining = node.state == "draining"
|
||||
self.coordination_healthy = True
|
||||
|
||||
def _heartbeat(self) -> None:
|
||||
with get_database().SessionLocal() as session:
|
||||
node = heartbeat_runtime_node(
|
||||
session,
|
||||
self.identity,
|
||||
metadata=self.metadata,
|
||||
)
|
||||
session.commit()
|
||||
self.draining = node.state == "draining"
|
||||
self.coordination_healthy = True
|
||||
|
||||
def _mark_stopped(self) -> None:
|
||||
with get_database().SessionLocal() as session:
|
||||
stop_runtime_node(session, self.identity)
|
||||
session.commit()
|
||||
|
||||
async def _heartbeat_loop(self) -> None:
|
||||
while not self._stopping:
|
||||
await asyncio.sleep(self.heartbeat_seconds)
|
||||
await self._heartbeat_once()
|
||||
|
||||
async def _heartbeat_once(self) -> bool:
|
||||
try:
|
||||
await asyncio.to_thread(self._heartbeat)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception: # noqa: BLE001 - a later heartbeat can recover
|
||||
self.coordination_healthy = False
|
||||
logger.exception(
|
||||
"runtime heartbeat failed node_id=%s",
|
||||
self.identity.node_id,
|
||||
)
|
||||
return False
|
||||
self.coordination_healthy = True
|
||||
return True
|
||||
|
||||
|
||||
__all__ = ["RuntimeNodeAgent"]
|
||||
Reference in New Issue
Block a user