Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ac6b211827 | ||
|
|
b175658a0f | ||
|
|
f4f09314e0 | ||
|
|
399e1f8e80 | ||
|
|
6ecb94c99e | ||
|
|
f84097224b | ||
|
|
f1b7d0e4ff | ||
|
|
db66320d68 | ||
|
|
de5bf357c9 | ||
|
|
d1114ee637 | ||
|
|
1082a6524d | ||
|
|
3efdee77de | ||
|
|
d26a601dda | ||
|
|
71666db45c | ||
|
|
b09b653987 | ||
|
|
02e5d91d56 |
@@ -16,11 +16,13 @@ webui/dist/
|
||||
.policy-test-build/
|
||||
.template-preview-test-build/
|
||||
.import-test-build/
|
||||
.runtime-status-test-build/
|
||||
webui/.component-test-build/
|
||||
webui/.module-test-build/
|
||||
webui/.policy-test-build/
|
||||
webui/.template-preview-test-build/
|
||||
webui/.import-test-build/
|
||||
webui/.runtime-status-test-build/
|
||||
|
||||
# GovOPlaN shared ignore rules from govoplan-core
|
||||
# ---> Node
|
||||
|
||||
@@ -40,10 +40,27 @@ The Ops API reports:
|
||||
stale state, and drain state
|
||||
- configured versus active API and worker replica counts
|
||||
- active worker-pool names, exact queue coverage, and missing queue owners
|
||||
- provider-neutral worker state (`disabled`, `unconfigured`, `starting`,
|
||||
`healthy`, `idle`, `busy`, `degraded`, `stale`, or `unreachable`), with the
|
||||
configured backend, latest heartbeat age, and stale threshold
|
||||
- queue depth, active/reserved work, and failure count only when reported by
|
||||
the provider; unavailable values are never interpreted as zero or healthy
|
||||
- release/module-composition skew and software-version skew across active nodes
|
||||
- rendered PostgreSQL connection peak, declared server limit, and operator reserve
|
||||
- recovery operation status, mode, checkpoint count, and last update
|
||||
|
||||
The separately authorized
|
||||
`GET /api/v1/ops/infrastructure/dependencies` endpoint collects current,
|
||||
module-owned infrastructure dependencies through the Core provider contract.
|
||||
It is intended for the host deployer's destructive-change preflight, not for a
|
||||
general data export. Results contain stable configuration references, lifecycle
|
||||
states, scopes, numeric counts and required preparation actions; they never
|
||||
contain credentials, tenant identifiers, file keys or secret-bearing endpoint
|
||||
values. Ops itself contributes the current database, Redis coordination,
|
||||
ingress, and load-balancing runtime bindings; feature modules contribute their
|
||||
own persisted configuration and data. One unavailable or invalid provider makes the result incomplete, so a
|
||||
caller must fail closed rather than treating missing provider data as zero.
|
||||
|
||||
These values are intentionally diagnostic. They do not replace deployment
|
||||
configuration management, backups, monitoring, or restore drills.
|
||||
|
||||
@@ -90,6 +107,15 @@ states. S3 capacity remains provider-owned unless a configured module check
|
||||
supplies bounded usage metrics; Ops must not enumerate an object store merely to
|
||||
render a dashboard.
|
||||
|
||||
The WebUI polls this read-only projection every 15 seconds only while its page
|
||||
is visible. It permits one request at a time, stops the timer when the document
|
||||
is hidden, and performs one refresh when visibility returns. Runtime providers
|
||||
register through the Core contract; Ops itself does not import Celery, Redis,
|
||||
or module-owned job implementations. A local development profile may
|
||||
intentionally disable workers without becoming unready. A production or other
|
||||
non-development profile treats a disabled, unconfigured, stale, or unreachable
|
||||
provider as readiness-critical.
|
||||
|
||||
Promote from a single-process profile to a split-worker profile when queued
|
||||
work becomes part of normal operation:
|
||||
|
||||
@@ -195,6 +221,22 @@ make failures worse.
|
||||
PostgreSQL is the production database. SQLite remains a local-development and
|
||||
tiny disposable profile only.
|
||||
|
||||
Files avoids SQLite's second-writer deadlock by recording blob recovery intent
|
||||
inside the caller transaction. Handled rollback reconstructs durable recovery
|
||||
evidence, but a hard process loss before commit can leave an object without an
|
||||
Ops ledger row. Run a complete Files integrity scan after such a loss and
|
||||
reconcile reported orphans before resuming writes. PostgreSQL retains the
|
||||
independent pre-effect recovery-intent guarantee required for production.
|
||||
|
||||
Files hard-purge and S3 connector-write operations use independently durable
|
||||
Core recovery records and distributed fences. A purge releases database
|
||||
references before a separate, reference-checked blob-GC operation deletes
|
||||
bytes. An S3 write records only an opaque target digest plus request/content
|
||||
evidence and verifies provider metadata after its conditional effect. When Ops
|
||||
shows `outcome_unknown` or `recovery_required`, do not replay the action from
|
||||
Ops: reconcile the exact blob reference/object or S3 request/content markers
|
||||
through the owning Files workflow first.
|
||||
|
||||
Production migrations should run explicitly before startup or package
|
||||
activation. Module install/uninstall workflows must use database backup and
|
||||
restore-check hooks for PostgreSQL before migrations or destructive retirement.
|
||||
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@govoplan/ops-webui",
|
||||
"version": "0.1.15",
|
||||
"version": "0.1.21",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "webui/src/index.ts",
|
||||
@@ -17,7 +17,7 @@
|
||||
"README.md"
|
||||
],
|
||||
"peerDependencies": {
|
||||
"@govoplan/core-webui": "^0.1.15",
|
||||
"@govoplan/core-webui": "^0.1.18",
|
||||
"lucide-react": "^1.23.0",
|
||||
"react": ">=19.2.7 <20",
|
||||
"react-dom": ">=19.2.7 <20",
|
||||
|
||||
+3
-3
@@ -4,14 +4,14 @@ build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "govoplan-ops"
|
||||
version = "0.1.15"
|
||||
version = "0.1.21"
|
||||
description = "GovOPlaN operations module for health, deployment profile, and sizing visibility."
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.12"
|
||||
authors = [{ name = "GovOPlaN" }]
|
||||
dependencies = [
|
||||
"govoplan-core>=0.1.15",
|
||||
"govoplan-access>=0.1.15",
|
||||
"govoplan-core>=0.1.42",
|
||||
"govoplan-access>=0.1.18",
|
||||
]
|
||||
|
||||
[tool.setuptools.packages.find]
|
||||
|
||||
@@ -2,4 +2,4 @@
|
||||
|
||||
__all__ = ["__version__"]
|
||||
|
||||
__version__ = "0.1.15"
|
||||
__version__ = "0.1.21"
|
||||
|
||||
@@ -23,7 +23,11 @@ from govoplan_core.core.module_installer import (
|
||||
list_module_installer_runs,
|
||||
read_module_installer_run,
|
||||
)
|
||||
from govoplan_core.core.operations import OperationalCheckProviderRegistration
|
||||
from govoplan_core.core.operations import (
|
||||
OperationalCheckProviderRegistration,
|
||||
RuntimeWorkStatusContext,
|
||||
RuntimeWorkStatusProviderRegistration,
|
||||
)
|
||||
from govoplan_core.core.recovery import (
|
||||
RecoveryOperation,
|
||||
)
|
||||
@@ -43,12 +47,16 @@ from govoplan_core.db.session import get_database
|
||||
from govoplan_core.settings import settings as core_settings
|
||||
|
||||
from govoplan_ops.backend.manifest import OPS_READ_SCOPES, OPS_RUN_SCOPES
|
||||
from govoplan_ops.backend.infrastructure import (
|
||||
deployment_capability_status,
|
||||
infrastructure_dependency_inventory,
|
||||
)
|
||||
|
||||
router = APIRouter(prefix="/ops", tags=["ops"])
|
||||
_module_check_cache: dict[str, tuple[float, dict[str, Any]]] = {}
|
||||
_module_check_cache_lock = threading.Lock()
|
||||
_worker_check_cache: tuple[float, dict[str, Any]] | None = None
|
||||
_worker_check_cache_lock = threading.Lock()
|
||||
_runtime_work_cache: dict[str, tuple[float, dict[str, Any]]] = {}
|
||||
_runtime_work_cache_lock = threading.Lock()
|
||||
|
||||
|
||||
class RuntimeDrainRequest(BaseModel):
|
||||
@@ -88,6 +96,20 @@ def run_ops_checks(
|
||||
return _ops_status_payload(request, force_module_checks=True)
|
||||
|
||||
|
||||
@router.get("/infrastructure/dependencies")
|
||||
def infrastructure_dependencies(
|
||||
request: Request,
|
||||
principal: ApiPrincipal = Depends(require_any_scope(*OPS_READ_SCOPES)),
|
||||
) -> dict[str, object]:
|
||||
"""Return a fresh, non-secret inventory for host deployment preflight."""
|
||||
|
||||
del principal
|
||||
return infrastructure_dependency_inventory(
|
||||
_registry(request),
|
||||
installation_id=core_settings.installation_id,
|
||||
).to_dict()
|
||||
|
||||
|
||||
@router.get("/runtime/nodes")
|
||||
def runtime_nodes(
|
||||
request: Request,
|
||||
@@ -191,9 +213,7 @@ def _ops_status_payload(
|
||||
if isinstance(database.get("maintenance_mode"), dict)
|
||||
else {"enabled": False, "message": None}
|
||||
)
|
||||
redis_check = _redis_check()
|
||||
current_profile = _current_profile()
|
||||
worker_check = _worker_check()
|
||||
database_capacity = _database_capacity_check()
|
||||
expected_composition_hash = runtime_composition_hash(
|
||||
tuple(manifest.id for manifest in registry.manifests())
|
||||
@@ -201,12 +221,25 @@ def _ops_status_payload(
|
||||
runtime_cluster = _runtime_cluster_status(
|
||||
expected_composition_hash=expected_composition_hash
|
||||
)
|
||||
runtime_work = _runtime_work_statuses(
|
||||
registry,
|
||||
RuntimeWorkStatusContext(
|
||||
profile=current_profile,
|
||||
observed_at=datetime.now(UTC),
|
||||
stale_after_seconds=core_settings.runtime_stale_after_seconds,
|
||||
runtime_nodes=tuple(runtime_cluster.get("nodes", [])),
|
||||
),
|
||||
force=force_module_checks,
|
||||
)
|
||||
worker_check = _runtime_work_check(runtime_work, current_profile)
|
||||
redis_check = _runtime_backend_check(runtime_work)
|
||||
module_checks = _module_operational_checks(
|
||||
registry,
|
||||
force=force_module_checks,
|
||||
)
|
||||
storage_check = _storage_check()
|
||||
backup_check = _backup_restore_check()
|
||||
infrastructure = deployment_capability_status()
|
||||
recovery_metrics = runtime_cluster.get("recovery", {}).get("metrics", {})
|
||||
checks = [
|
||||
_check(
|
||||
@@ -229,6 +262,7 @@ def _ops_status_payload(
|
||||
storage_check,
|
||||
backup_check,
|
||||
_deployment_security_check(current_profile),
|
||||
_infrastructure_capability_check(infrastructure),
|
||||
*module_checks,
|
||||
]
|
||||
readiness = _readiness(checks, maintenance_mode)
|
||||
@@ -262,6 +296,12 @@ def _ops_status_payload(
|
||||
"failed_operation_count": int(recovery_metrics.get("failed") or 0),
|
||||
"outcome_unknown_count": int(recovery_metrics.get("outcome_unknown") or 0),
|
||||
"active_operation_count": int(recovery_metrics.get("active") or 0),
|
||||
"infrastructure_capability_count": len(
|
||||
infrastructure.get("capabilities", [])
|
||||
),
|
||||
"pending_post_install_task_count": len(
|
||||
infrastructure.get("post_install_tasks", [])
|
||||
),
|
||||
},
|
||||
"readiness": readiness,
|
||||
"checks": checks,
|
||||
@@ -269,9 +309,41 @@ def _ops_status_payload(
|
||||
"deployment_profiles": _deployment_profiles(current_profile),
|
||||
"sizing": _sizing_assumptions(),
|
||||
"runtime_cluster": runtime_cluster,
|
||||
"runtime_work": runtime_work,
|
||||
"infrastructure": infrastructure,
|
||||
}
|
||||
|
||||
|
||||
def _infrastructure_capability_check(
|
||||
infrastructure: Mapping[str, object],
|
||||
) -> dict[str, Any]:
|
||||
if infrastructure.get("available") is True:
|
||||
capabilities = infrastructure.get("capabilities")
|
||||
count = len(capabilities) if isinstance(capabilities, list) else 0
|
||||
return _check(
|
||||
"infrastructure_capability_receipt",
|
||||
"Infrastructure capabilities",
|
||||
"ok",
|
||||
f"A validated non-secret deployment receipt reports {count} capabilities.",
|
||||
)
|
||||
if infrastructure.get("configured") is True:
|
||||
return _check(
|
||||
"infrastructure_capability_receipt",
|
||||
"Infrastructure capabilities",
|
||||
"warning",
|
||||
str(
|
||||
infrastructure.get("error")
|
||||
or "The configured deployment capability receipt is unavailable."
|
||||
),
|
||||
)
|
||||
return _check(
|
||||
"infrastructure_capability_receipt",
|
||||
"Infrastructure capabilities",
|
||||
"ok",
|
||||
"No deployment capability receipt is mounted in this runtime profile.",
|
||||
)
|
||||
|
||||
|
||||
def _runtime_cluster_status(
|
||||
*,
|
||||
expected_composition_hash: str | None = None,
|
||||
@@ -713,142 +785,178 @@ def _maintenance_check(maintenance_mode: dict[str, Any]) -> dict[str, Any]:
|
||||
)
|
||||
|
||||
|
||||
def _redis_check() -> dict[str, Any]:
|
||||
if not core_settings.celery_enabled:
|
||||
return _check(
|
||||
"redis_broker",
|
||||
"Redis broker",
|
||||
"inactive",
|
||||
"Redis is not required while Celery is disabled.",
|
||||
)
|
||||
try:
|
||||
from redis import Redis
|
||||
def _runtime_work_statuses(
|
||||
registry: PlatformRegistry,
|
||||
context: RuntimeWorkStatusContext,
|
||||
*,
|
||||
force: bool = False,
|
||||
) -> list[dict[str, Any]]:
|
||||
registrations: list[RuntimeWorkStatusProviderRegistration] = []
|
||||
for manifest in registry.manifests():
|
||||
registrations.extend(manifest.runtime_work_status_providers)
|
||||
|
||||
client = Redis.from_url(
|
||||
core_settings.redis_url, socket_connect_timeout=0.75, socket_timeout=0.75
|
||||
)
|
||||
client.ping()
|
||||
return _check(
|
||||
"redis_broker",
|
||||
"Redis broker",
|
||||
"ok",
|
||||
f"Redis broker reachable at {_redact_url(core_settings.redis_url)}.",
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 - diagnostic endpoint should report the concrete failure.
|
||||
return _check(
|
||||
"redis_broker",
|
||||
"Redis broker",
|
||||
"error",
|
||||
f"Redis broker check failed: {exc}",
|
||||
readiness_critical=True,
|
||||
)
|
||||
results: list[dict[str, Any]] = []
|
||||
seen: set[str] = set()
|
||||
now = time.monotonic()
|
||||
for registration in registrations:
|
||||
key = f"{registration.module_id}:{registration.provider_id}"
|
||||
if registration.provider_id in seen:
|
||||
results.append(_unavailable_runtime_work(registration, "Duplicate provider id."))
|
||||
continue
|
||||
seen.add(registration.provider_id)
|
||||
with _runtime_work_cache_lock:
|
||||
cached = _runtime_work_cache.get(key)
|
||||
if (
|
||||
not force
|
||||
and cached is not None
|
||||
and now - cached[0] < max(1, registration.cache_seconds)
|
||||
):
|
||||
results.append(dict(cached[1]))
|
||||
continue
|
||||
try:
|
||||
status_result = registration.provider(context)
|
||||
if status_result.provider_id != registration.provider_id:
|
||||
raise ValueError("provider id mismatch")
|
||||
result = status_result.as_dict()
|
||||
except Exception: # noqa: BLE001 - provider failures remain isolated and sanitized.
|
||||
result = _unavailable_runtime_work(
|
||||
registration,
|
||||
"The runtime-work provider failed without usable evidence.",
|
||||
)
|
||||
with _runtime_work_cache_lock:
|
||||
_runtime_work_cache[key] = (now, result)
|
||||
results.append(dict(result))
|
||||
return results
|
||||
|
||||
|
||||
def _worker_check() -> dict[str, Any]:
|
||||
if not core_settings.celery_enabled:
|
||||
def _unavailable_runtime_work(
|
||||
registration: RuntimeWorkStatusProviderRegistration,
|
||||
detail: str,
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"provider_id": registration.provider_id,
|
||||
"label": registration.provider_id,
|
||||
"backend": "unavailable",
|
||||
"enabled": None,
|
||||
"configured": None,
|
||||
"state": "unreachable",
|
||||
"detail": detail,
|
||||
"observed_at": datetime.now(UTC).isoformat(),
|
||||
"active_workers": None,
|
||||
"last_heartbeat_at": None,
|
||||
"queue_depths": {},
|
||||
"active_work": None,
|
||||
"reserved_work": None,
|
||||
"failures": None,
|
||||
"stale_after_seconds": None,
|
||||
"guidance": "Restore or configure the owning status provider.",
|
||||
}
|
||||
|
||||
|
||||
def _runtime_work_check(
|
||||
statuses: list[dict[str, Any]],
|
||||
profile: str,
|
||||
) -> dict[str, Any]:
|
||||
if not statuses:
|
||||
return _check(
|
||||
"worker_split",
|
||||
"Background workers",
|
||||
"warning",
|
||||
"Celery is disabled; long-running work executes only through synchronous or dev paths.",
|
||||
metrics={"workers": 0, "active_tasks": 0, "missing_queues": []},
|
||||
"No runtime-work status provider is installed; worker and queue health is unavailable.",
|
||||
readiness_critical=profile not in {"local-dev", "development"},
|
||||
metrics={"workers": None, "active_tasks": None, "reserved_tasks": None},
|
||||
)
|
||||
global _worker_check_cache
|
||||
now = time.monotonic()
|
||||
with _worker_check_cache_lock:
|
||||
if _worker_check_cache is not None and now - _worker_check_cache[0] < 15:
|
||||
return dict(_worker_check_cache[1])
|
||||
try:
|
||||
from govoplan_core.celery_app import celery
|
||||
|
||||
inspector = celery.control.inspect(timeout=0.75)
|
||||
replies = inspector.ping() or {}
|
||||
active_queues_by_worker = inspector.active_queues() or {}
|
||||
active_by_worker = inspector.active() or {}
|
||||
except Exception as exc: # noqa: BLE001 - diagnostic endpoint should report the concrete failure.
|
||||
result = _check(
|
||||
"worker_split",
|
||||
"Background workers",
|
||||
"error",
|
||||
f"Worker heartbeat check failed: {exc}",
|
||||
readiness_critical=True,
|
||||
metrics={"workers": 0, "active_tasks": 0},
|
||||
)
|
||||
with _worker_check_cache_lock:
|
||||
_worker_check_cache = (now, result)
|
||||
return result
|
||||
if replies:
|
||||
worker_names = ", ".join(sorted(replies))
|
||||
active_queues = sorted(
|
||||
{
|
||||
str(queue.get("name"))
|
||||
for queues in active_queues_by_worker.values()
|
||||
if isinstance(queues, list)
|
||||
for queue in queues
|
||||
if isinstance(queue, Mapping) and queue.get("name")
|
||||
}
|
||||
)
|
||||
expected_queues = _celery_queues()
|
||||
missing_queues = sorted(set(expected_queues) - set(active_queues))
|
||||
active_tasks = sum(
|
||||
len(tasks) for tasks in active_by_worker.values() if isinstance(tasks, list)
|
||||
)
|
||||
queue_depths = _queue_depths(expected_queues)
|
||||
state = "warning" if missing_queues else "ok"
|
||||
detail = f"{len(replies)} worker(s) replied: {worker_names}."
|
||||
if missing_queues:
|
||||
detail += (
|
||||
" No worker consumes configured queues: "
|
||||
+ ", ".join(missing_queues)
|
||||
+ "."
|
||||
development_profile = profile in {"local-dev", "development"}
|
||||
state_rank = {
|
||||
"unreachable": 8,
|
||||
"stale": 7,
|
||||
"degraded": 6,
|
||||
"unconfigured": 5,
|
||||
"starting": 4,
|
||||
"busy": 3,
|
||||
"healthy": 2,
|
||||
"idle": 1,
|
||||
"disabled": 0 if development_profile else 6,
|
||||
}
|
||||
worst = max(statuses, key=lambda item: state_rank.get(str(item.get("state")), 8))
|
||||
runtime_state = str(worst.get("state") or "unreachable")
|
||||
disabled_in_dev = runtime_state == "disabled" and development_profile
|
||||
check_state = (
|
||||
"inactive"
|
||||
if disabled_in_dev
|
||||
else "ok"
|
||||
if runtime_state in {"healthy", "idle", "busy"}
|
||||
else "warning"
|
||||
if runtime_state in {"disabled", "starting", "degraded"}
|
||||
else "error"
|
||||
)
|
||||
queue_depths: dict[str, int | None] = {}
|
||||
for item in statuses:
|
||||
depths = item.get("queue_depths")
|
||||
if isinstance(depths, Mapping):
|
||||
queue_depths.update(
|
||||
{
|
||||
str(queue): int(value) if isinstance(value, int) else None
|
||||
for queue, value in depths.items()
|
||||
}
|
||||
)
|
||||
result = _check(
|
||||
"worker_split",
|
||||
"Background workers",
|
||||
state,
|
||||
detail,
|
||||
readiness_critical=bool(missing_queues),
|
||||
metrics={
|
||||
"workers": len(replies),
|
||||
"active_tasks": active_tasks,
|
||||
"expected_queues": expected_queues,
|
||||
"active_queues": active_queues,
|
||||
"missing_queues": missing_queues,
|
||||
"queue_depths": queue_depths,
|
||||
},
|
||||
)
|
||||
with _worker_check_cache_lock:
|
||||
_worker_check_cache = (now, result)
|
||||
return result
|
||||
result = _check(
|
||||
return _check(
|
||||
"worker_split",
|
||||
"Background workers",
|
||||
"error",
|
||||
"Celery is enabled, but no workers replied to heartbeat.",
|
||||
readiness_critical=True,
|
||||
metrics={"workers": 0, "active_tasks": 0},
|
||||
check_state,
|
||||
str(worst.get("detail") or "Runtime-work status is unavailable."),
|
||||
readiness_critical=(
|
||||
not disabled_in_dev
|
||||
and runtime_state
|
||||
in {"disabled", "unconfigured", "degraded", "stale", "unreachable"}
|
||||
),
|
||||
metrics={
|
||||
"state": runtime_state,
|
||||
"workers": _sum_known(statuses, "active_workers"),
|
||||
"active_tasks": _sum_known(statuses, "active_work"),
|
||||
"reserved_tasks": _sum_known(statuses, "reserved_work"),
|
||||
"failures": _sum_known(statuses, "failures"),
|
||||
"queue_depths": queue_depths,
|
||||
},
|
||||
)
|
||||
with _worker_check_cache_lock:
|
||||
_worker_check_cache = (now, result)
|
||||
return result
|
||||
|
||||
|
||||
def _queue_depths(queues: list[str]) -> dict[str, int]:
|
||||
try:
|
||||
from redis import Redis
|
||||
|
||||
client = Redis.from_url(
|
||||
core_settings.redis_url,
|
||||
socket_connect_timeout=0.75,
|
||||
socket_timeout=0.75,
|
||||
def _runtime_backend_check(statuses: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
if not statuses or all(item.get("enabled") is False for item in statuses):
|
||||
return _check(
|
||||
"redis_broker",
|
||||
"Work backend",
|
||||
"inactive",
|
||||
"No enabled runtime-work backend requires a connectivity assertion.",
|
||||
)
|
||||
pipeline = client.pipeline(transaction=False)
|
||||
for queue in queues:
|
||||
pipeline.llen(queue)
|
||||
values = pipeline.execute()
|
||||
except Exception: # noqa: BLE001 - worker heartbeat remains the readiness source.
|
||||
return {}
|
||||
return {queue: int(value) for queue, value in zip(queues, values, strict=True)}
|
||||
if any(item.get("state") == "unreachable" for item in statuses):
|
||||
return _check(
|
||||
"redis_broker",
|
||||
"Work backend",
|
||||
"error",
|
||||
"An enabled runtime-work backend is unreachable.",
|
||||
readiness_critical=True,
|
||||
)
|
||||
if any(item.get("configured") is not True for item in statuses if item.get("enabled") is True):
|
||||
return _check(
|
||||
"redis_broker",
|
||||
"Work backend",
|
||||
"warning",
|
||||
"An enabled runtime-work backend is not fully configured.",
|
||||
readiness_critical=True,
|
||||
)
|
||||
return _check(
|
||||
"redis_broker",
|
||||
"Work backend",
|
||||
"ok",
|
||||
"Enabled runtime-work providers returned bounded status evidence.",
|
||||
)
|
||||
|
||||
|
||||
def _sum_known(statuses: list[dict[str, Any]], field: str) -> int | None:
|
||||
values = [item.get(field) for item in statuses]
|
||||
known = [int(value) for value in values if isinstance(value, int)]
|
||||
return sum(known) if known else None
|
||||
|
||||
|
||||
def _storage_check() -> dict[str, Any]:
|
||||
@@ -1306,9 +1414,9 @@ def _readiness(
|
||||
|
||||
def _current_profile() -> str:
|
||||
app_env = str(core_settings.app_env or "").lower()
|
||||
if app_env == "dev" and core_settings.celery_enabled:
|
||||
if app_env in {"dev", "test"} and core_settings.celery_enabled:
|
||||
return "production-like-dev"
|
||||
if app_env == "dev":
|
||||
if app_env in {"dev", "test"}:
|
||||
return "local-dev"
|
||||
if core_settings.celery_enabled:
|
||||
return "split-worker"
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
"""German translations for public structured documentation metadata."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
|
||||
GERMAN_STRUCTURED_TRANSLATIONS: dict[str, dict[str, Any]] = {'ops.health-governance-and-sizing': {'consequence_classes': {'run_probes': 'Führen Sie begrenzte '
|
||||
'moduleigene '
|
||||
'Gesundheitssonden aus '
|
||||
'und aktualisieren Sie '
|
||||
'die sanierte '
|
||||
'Betriebsprojektion'}},
|
||||
'ops.runtime-coordination-and-recovery': {'consequence_classes': {'cancel_node_drain': 'Rückkehr '
|
||||
'eines '
|
||||
'noch '
|
||||
'ablaufenden '
|
||||
'Laufzeitknotens '
|
||||
'zu Active '
|
||||
'Scheduling',
|
||||
'drain_node': 'Beenden Sie die '
|
||||
'Weiterleitung '
|
||||
'neuer Arbeiten '
|
||||
'an die '
|
||||
'ausgewählte '
|
||||
'Laufzeitinkarnation, '
|
||||
'während die '
|
||||
'Arbeiten während '
|
||||
'des Fluges '
|
||||
'abgeschlossen '
|
||||
'sind',
|
||||
'inspect_recovery': 'Lesen Sie '
|
||||
'den '
|
||||
'sanierten '
|
||||
'dauerhaften '
|
||||
'Wiederherstellungszustand, '
|
||||
'ohne den '
|
||||
'Besitzeffekt '
|
||||
'zu '
|
||||
'wiederholen '
|
||||
'oder zu '
|
||||
'reparieren'},
|
||||
'limitations': ['Drain wird im '
|
||||
'Laufzeit-Herzschlagintervall '
|
||||
'beobachtet und beendet die aktive '
|
||||
'Arbeit nicht zwangsweise.',
|
||||
'Ops erstellt oder stellt keine Backups '
|
||||
'her und erhält niemals private '
|
||||
'Artefakte oder '
|
||||
'Schlüssel-Depotreferenzen.',
|
||||
'SQLite ist ein rein '
|
||||
'entwicklungsbezogenes '
|
||||
'Wiederherstellungsprofil; Dateien, die '
|
||||
'vor einem unbehandelten Prozessverlust '
|
||||
'erstellt wurden, erfordern '
|
||||
'möglicherweise eine Erkennung durch '
|
||||
'Integritätsscan, da die Absicht der '
|
||||
'Anrufertransaktion nicht festgelegt '
|
||||
'wurde.',
|
||||
'Eine verifizierte Quittung beweist die '
|
||||
'aufgezeichnete Übung; es macht keinen '
|
||||
'unsicheren '
|
||||
'Post-Migrationscode-Rollback '
|
||||
'reversibel.'],
|
||||
'steps': ['Vergleichen Sie aktive nicht-stale Knoten '
|
||||
'mit den konfigurierten API- und '
|
||||
'Worker-Replica-Erwartungen.',
|
||||
'Fordern Sie die Entleerung an und warten '
|
||||
'Sie, bis der Knoten die Entleerung meldet, '
|
||||
'bevor Sie ihn ersetzen.',
|
||||
'Überprüfen Sie alle '
|
||||
'Wiederherstellungs-erforderlichen, '
|
||||
'ergebnisunbekannten oder manuellen Eingriffe '
|
||||
'und folgen Sie dem aufgezeichneten '
|
||||
'Wiederherstellungsmodus.',
|
||||
'Unterscheiden Sie bei Dateien zwischen Blob '
|
||||
'Upload/Reparatur, genehmigtem Hard Purge, '
|
||||
'Blob Garbage Collection und '
|
||||
'S3-Connector-Rückschreibung, bevor Sie die '
|
||||
'Eigentümerdatenbank/das Objekt oder den '
|
||||
'Anbieternachweis überprüfen.',
|
||||
'Bestätigen Sie, dass der Backup-Beweis '
|
||||
'verifiziert und aktuell ist, bevor Sie eine '
|
||||
'Release-Änderung der Migration genehmigen.',
|
||||
'Überprüfen Sie Ersatzzusammensetzung, '
|
||||
'Bereitschaft, Warteschlangenverbraucher und '
|
||||
'Wiederherstellungsnachweise, bevor Sie den '
|
||||
'Vorgang abschließen.']}}
|
||||
@@ -0,0 +1,133 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from govoplan_core.core.infrastructure_capabilities import (
|
||||
INFRASTRUCTURE_DEPENDENCY_PROVIDER_CAPABILITY_PREFIX,
|
||||
InfrastructureDependency,
|
||||
InfrastructureDependencyInventory,
|
||||
InfrastructureDependencyProvider,
|
||||
collect_infrastructure_dependency_inventory,
|
||||
deployment_capability_status as _deployment_capability_status,
|
||||
load_infrastructure_capability_receipt,
|
||||
)
|
||||
from govoplan_core.settings import settings as core_settings
|
||||
|
||||
|
||||
OPS_INFRASTRUCTURE_DEPENDENCY_CAPABILITY = (
|
||||
f"{INFRASTRUCTURE_DEPENDENCY_PROVIDER_CAPABILITY_PREFIX}ops"
|
||||
)
|
||||
|
||||
|
||||
class OpsInfrastructureDependencyProvider(InfrastructureDependencyProvider):
|
||||
"""Report Core runtime bindings that optional feature providers do not own."""
|
||||
|
||||
module_id = "ops"
|
||||
capability_ids = (
|
||||
"database.postgresql",
|
||||
"coordination.redis",
|
||||
"network.ingress",
|
||||
"runtime.load_balancing",
|
||||
)
|
||||
|
||||
def infrastructure_dependencies(self) -> tuple[InfrastructureDependency, ...]:
|
||||
dependencies = [
|
||||
InfrastructureDependency(
|
||||
capability_id="database.postgresql",
|
||||
module_id=self.module_id,
|
||||
dependency_type="application_state_binding",
|
||||
dependency_ref="runtime:database.postgresql",
|
||||
state="runtime_binding",
|
||||
scope="system",
|
||||
summary=(
|
||||
"GovOPlaN persists application and control-plane state in the active PostgreSQL database."
|
||||
),
|
||||
metrics={},
|
||||
required_action=(
|
||||
"Create and restore-verify a coordinated database backup before rebinding PostgreSQL."
|
||||
),
|
||||
),
|
||||
InfrastructureDependency(
|
||||
capability_id="network.ingress",
|
||||
module_id=self.module_id,
|
||||
dependency_type="public_runtime_binding",
|
||||
dependency_ref="runtime:network.ingress",
|
||||
state="runtime_binding",
|
||||
scope="system",
|
||||
summary="Ops and application APIs are reached through the active ingress boundary.",
|
||||
metrics={},
|
||||
required_action=(
|
||||
"Verify trusted proxy headers, TLS, health, and public reachability on the replacement ingress."
|
||||
),
|
||||
),
|
||||
InfrastructureDependency(
|
||||
capability_id="runtime.load_balancing",
|
||||
module_id=self.module_id,
|
||||
dependency_type="runtime_routing_binding",
|
||||
dependency_ref="runtime:load-balancing",
|
||||
state="runtime_binding",
|
||||
scope="system",
|
||||
summary="API and WebUI replicas are reached through the runtime load balancer.",
|
||||
metrics={},
|
||||
required_action=(
|
||||
"Verify health-aware API and WebUI routing before replacing the load balancer."
|
||||
),
|
||||
),
|
||||
]
|
||||
if core_settings.redis_url.strip():
|
||||
dependencies.append(
|
||||
InfrastructureDependency(
|
||||
capability_id="coordination.redis",
|
||||
module_id=self.module_id,
|
||||
dependency_type="coordination_runtime_binding",
|
||||
dependency_ref="runtime:coordination.redis",
|
||||
state="runtime_binding",
|
||||
scope="system",
|
||||
summary=(
|
||||
"The active runtime uses Redis for queues or shared coordination."
|
||||
),
|
||||
metrics={"celery_enabled": int(core_settings.celery_enabled)},
|
||||
required_action=(
|
||||
"Drain queued and in-flight work, verify idempotent recovery, and provision replacement coordination before rebinding or disabling Redis."
|
||||
),
|
||||
)
|
||||
)
|
||||
return tuple(dependencies)
|
||||
|
||||
|
||||
def deployment_capability_status(
|
||||
path: Path | None = None,
|
||||
) -> dict[str, object]:
|
||||
"""Project Core's validated non-secret deployment receipt into Ops."""
|
||||
|
||||
return _deployment_capability_status(path)
|
||||
|
||||
|
||||
def infrastructure_dependency_inventory(
|
||||
registry: object,
|
||||
*,
|
||||
installation_id: str,
|
||||
) -> InfrastructureDependencyInventory:
|
||||
"""Collect module-owned persisted dependencies for an authorized host plan."""
|
||||
|
||||
receipt = load_infrastructure_capability_receipt()
|
||||
if receipt is None:
|
||||
raise ValueError(
|
||||
"Infrastructure dependency inventory requires a mounted deployment receipt."
|
||||
)
|
||||
if receipt.installation_id != installation_id:
|
||||
raise ValueError(
|
||||
"Mounted deployment receipt belongs to a different installation."
|
||||
)
|
||||
return collect_infrastructure_dependency_inventory(
|
||||
registry,
|
||||
installation_id=installation_id,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"deployment_capability_status",
|
||||
"infrastructure_dependency_inventory",
|
||||
"OPS_INFRASTRUCTURE_DEPENDENCY_CAPABILITY",
|
||||
"OpsInfrastructureDependencyProvider",
|
||||
]
|
||||
@@ -1,11 +1,19 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from govoplan_core.core.modules import with_documentation_structured_translations
|
||||
from govoplan_ops.backend.german_structured_documentation import GERMAN_STRUCTURED_TRANSLATIONS
|
||||
from govoplan_ops.backend.infrastructure import (
|
||||
OPS_INFRASTRUCTURE_DEPENDENCY_CAPABILITY,
|
||||
OpsInfrastructureDependencyProvider,
|
||||
)
|
||||
|
||||
from govoplan_core.core.access import (
|
||||
CAPABILITY_AUTH_PERMISSION_EVALUATOR,
|
||||
CAPABILITY_AUTH_PRINCIPAL_RESOLVER,
|
||||
)
|
||||
from govoplan_core.core.modules import (
|
||||
DocumentationCondition,
|
||||
DocumentationLink,
|
||||
DocumentationTopic,
|
||||
FrontendModule,
|
||||
FrontendRoute,
|
||||
@@ -20,6 +28,8 @@ from govoplan_core.core.provider_governance import (
|
||||
ModuleArchitectureDocumentation,
|
||||
ModuleMaturityEvidence,
|
||||
)
|
||||
from govoplan_core.core.operations import RuntimeWorkStatusProviderRegistration
|
||||
from govoplan_core.core.runtime_work import celery_runtime_work_status
|
||||
from govoplan_core.core.views import ViewSurface
|
||||
|
||||
OPS_READ_SCOPE = "ops:operations:read"
|
||||
@@ -90,7 +100,7 @@ def _route_factory(context: ModuleContext):
|
||||
manifest = ModuleManifest(
|
||||
id="ops",
|
||||
name="Ops",
|
||||
version="0.1.15",
|
||||
version="0.1.21",
|
||||
required_capabilities=(
|
||||
CAPABILITY_AUTH_PRINCIPAL_RESOLVER,
|
||||
CAPABILITY_AUTH_PERMISSION_EVALUATOR,
|
||||
@@ -124,21 +134,67 @@ manifest = ModuleManifest(
|
||||
level="system",
|
||||
),
|
||||
),
|
||||
runtime_work_status_providers=(
|
||||
RuntimeWorkStatusProviderRegistration(
|
||||
module_id="ops",
|
||||
provider_id="core.celery",
|
||||
provider=celery_runtime_work_status,
|
||||
cache_seconds=15,
|
||||
),
|
||||
),
|
||||
capability_factories={
|
||||
OPS_INFRASTRUCTURE_DEPENDENCY_CAPABILITY: (
|
||||
lambda context: OpsInfrastructureDependencyProvider()
|
||||
),
|
||||
},
|
||||
documentation=(
|
||||
DocumentationTopic(
|
||||
id="ops.health-governance-and-sizing",
|
||||
title="Inspect platform health and deployment posture",
|
||||
summary="Ops combines module-owned health checks with deployment profile, governance inventory, worker assumptions, and sizing guidance.",
|
||||
body="Read-only status distinguishes configured capabilities from healthy integrations. Authorized operators can run bounded probes; a probe must not perform unbounded business work or silently repair data. Use readiness and worker results when diagnosing a node, and use the deployment profile and sizing assumptions when planning horizontal capacity.",
|
||||
body="Read-only status distinguishes configured capabilities from healthy integrations. Worker and queue providers use a Core runtime-status contract, so Ops never imports a provider backend. The surface distinguishes intentionally disabled, unconfigured, starting, healthy with unsupported queue depth, measured idle, busy, degraded, stale, and unreachable states. It shows enabled/configured state, backend, workers, heartbeat age and stale threshold, queue depth, active/reserved work, and failures only when each value is actually reported; unavailable values are never rendered as zero or healthy. Local development treats intentionally disabled workers as expected, while production profiles require an enabled, configured, reachable provider before queue-backed work is accepted. Polling is bounded to one request, pauses while the page is hidden, and refreshes on return. When the deployment mounts a signed or locally generated non-secret infrastructure capability receipt, Ops shows whether PostgreSQL, Redis, SMTP, file storage, load balancing, and ingress are configured, externally supplied, available but unconfigured, or unavailable. Secret values never cross this boundary; only stable environment or credential-envelope references may be disclosed. Pending post-install tasks remain visible with a stable resume key. The separately authorized infrastructure-dependency endpoint asks enabled module providers for their current persisted configuration and data dependencies. Each provider returns only stable references, states, scopes, numeric metrics, and required migration actions; one failed provider marks the entire fresh inventory incomplete so the host deployer blocks a capability change. Authorized operators can run bounded probes; a probe must not perform unbounded business work or silently repair data. Use readiness and worker results when diagnosing a node, and use the deployment profile and sizing assumptions when planning horizontal capacity.",
|
||||
documentation_types=("admin", "user"),
|
||||
audience=("operator", "system_admin"),
|
||||
related_modules=("audit", "docs", "notifications"),
|
||||
links=(
|
||||
DocumentationLink(
|
||||
label="Infrastructure dependency inventory API",
|
||||
href="/api/v1/ops/infrastructure/dependencies",
|
||||
kind="api",
|
||||
),
|
||||
DocumentationLink(
|
||||
label="Operations profile handbook",
|
||||
href="govoplan-ops/docs/SCALABILITY_PROFILES.md",
|
||||
kind="repository",
|
||||
),
|
||||
),
|
||||
translations={
|
||||
"de": {
|
||||
"title": "Plattformzustand und Bereitstellungsprofil prüfen",
|
||||
"summary": "Ops führt modulbezogene Zustandsprüfungen mit Bereitstellungsprofil, Governance-Inventar, Worker-Annahmen und Dimensionierungshinweisen zusammen.",
|
||||
"body": (
|
||||
"Der schreibgeschützte Status unterscheidet konfigurierte Fähigkeiten von funktionsfähigen Integrationen. "
|
||||
"Worker- und Warteschlangenanbieter verwenden einen Core-Vertrag für den Laufzeitstatus, sodass Ops niemals ein Anbieter-Backend importiert. "
|
||||
"Die Oberfläche unterscheidet bewusst deaktivierte, nicht konfigurierte, startende, gesunde, untätige, ausgelastete, beeinträchtigte, veraltete und nicht erreichbare Zustände; eine nicht unterstützte Warteschlangentiefe wird ausdrücklich ausgewiesen. "
|
||||
"Aktivierung, Konfiguration, Backend, Worker, Alter und Grenzwert des Heartbeats, Warteschlangentiefe, aktive oder reservierte Arbeit sowie Fehler werden nur angezeigt, wenn der Anbieter den jeweiligen Wert tatsächlich meldet; fehlende Werte erscheinen niemals als null oder gesund. "
|
||||
"In der lokalen Entwicklung sind bewusst deaktivierte Worker zulässig, während Produktionsprofile einen aktivierten, konfigurierten und erreichbaren Anbieter verlangen, bevor warteschlangengestützte Arbeit angenommen wird. "
|
||||
"Die Abfrage bleibt auf eine Anfrage je Intervall begrenzt, pausiert bei ausgeblendeter Seite und wird bei der Rückkehr fortgesetzt. "
|
||||
"Ist ein signierter oder lokal erzeugter Infrastrukturbeleg ohne Geheimwerte eingebunden, zeigt Ops für PostgreSQL, Redis, SMTP, Dateispeicher, Lastverteilung und Ingress, ob die Fähigkeit konfiguriert, extern bereitgestellt, verfügbar aber nicht konfiguriert oder nicht verfügbar ist. "
|
||||
"Geheimwerte überschreiten diese Grenze nie; offengelegt werden dürfen nur stabile Umgebungs- oder Credential-Envelope-Referenzen. "
|
||||
"Ausstehende Aufgaben nach einer Installation bleiben mit einem stabilen Fortsetzungsschlüssel sichtbar. Der separat autorisierte Infrastruktur-Abhängigkeitsendpunkt fragt aktivierte Modulprovider nach ihren aktuellen gespeicherten Konfigurations- und Datenabhängigkeiten. "
|
||||
"Jeder Provider liefert nur stabile Referenzen, Zustände, Geltungsbereiche, numerische Kennzahlen und erforderliche Migrationsmaßnahmen; schlägt ein Provider fehl, ist das gesamte frische Inventar unvollständig und der Host-Deployer blockiert die Fähigkeitsänderung. "
|
||||
"Autorisierte Betriebsverantwortliche dürfen begrenzte Prüfungen ausführen; eine Prüfung darf weder unbegrenzte Facharbeit auslösen noch Daten stillschweigend reparieren. "
|
||||
"Nutzen Sie Bereitschafts- und Worker-Ergebnisse zur Diagnose eines Knotens sowie Bereitstellungsprofil und Dimensionierungsannahmen zur Planung horizontaler Kapazität."
|
||||
),
|
||||
}
|
||||
},
|
||||
metadata={
|
||||
"kind": "reference",
|
||||
"help_contexts": [
|
||||
"ops.page",
|
||||
"ops.page.summary",
|
||||
"ops.page.health",
|
||||
"ops.page.runtime",
|
||||
"ops.page.governance",
|
||||
"ops.page.deployment",
|
||||
"ops.page.sizing",
|
||||
@@ -154,7 +210,7 @@ manifest = ModuleManifest(
|
||||
id="ops.runtime-coordination-and-recovery",
|
||||
title="Drain runtime nodes and inspect recovery evidence",
|
||||
summary="Ops projects shared runtime heartbeats, replica gaps, drain controls, and recovery states that require operator attention.",
|
||||
body="Use the runtime table to identify stale or composition-skewed API and worker replicas. Drain before replacement so API readiness closes and workers stop taking new queue work; cancellation is available while the node is still draining. The recovery table reports durable Core recovery operations. A rejected operation is a verified provider rejection and needs no recovery; outcome-unknown and recovery-required operations still require reconciliation through the owning module. Core module-lifecycle entries block every later install or live graph change: use the installer run id to verify package, backup, migration, and health evidence before rollback or forward repair. Mail SMTP and IMAP APPEND entries use stable attempt identifiers and digest-only evidence: reconcile the Mail command from provider evidence, never by replaying the original effect from Ops. Dataflow database-only runs are atomic, while published-output runs use forward recovery: reconcile the recorded output digest and sink idempotency key before allowing another publication. Backup status separately projects only the sanitized deployment verification receipt: a verified status identifies a coordinated recovery point and isolated restore drill, while absent, expired, or invalid evidence blocks a release-changing migration.",
|
||||
body="Use the runtime table to identify stale or composition-skewed API and worker replicas. Drain before replacement so API readiness closes and workers stop taking new queue work; cancellation is available while the node is still draining. The recovery table reports durable Core recovery operations. A rejected operation is a verified provider rejection and needs no recovery; outcome-unknown and recovery-required operations still require reconciliation through the owning module. Core module-lifecycle entries block every later install or live graph change: use the installer run id to verify package, backup, migration, and health evidence before rollback or forward repair. Mail SMTP and IMAP APPEND entries use stable attempt identifiers and digest-only evidence: reconcile the Mail command from provider evidence, never by replaying the original effect from Ops. Files blob writes, hard purge, reference-checked garbage collection, and conditional S3 connector writes record Core recovery evidence. For a Files connector outcome, inspect the provider request/content markers and revision before allowing another write to the fenced path; for blob GC, recheck FileVersion references and exact object absence. Development SQLite can show only handled-rollback reconstruction for caller-transaction blob uploads; after a hard SQLite process loss, run the owning Files integrity scan because an orphan may have no Ops ledger row. Dataflow database-only runs are atomic, while published-output runs use forward recovery: reconcile the recorded output digest and sink idempotency key before allowing another publication. Backup status separately projects only the sanitized deployment verification receipt: a verified status identifies a coordinated recovery point and isolated restore drill, while absent, expired, or invalid evidence blocks a release-changing migration.",
|
||||
documentation_types=("admin", "user"),
|
||||
audience=("operator", "system_admin"),
|
||||
conditions=(
|
||||
@@ -163,7 +219,24 @@ manifest = ModuleManifest(
|
||||
any_scopes=OPS_READ_SCOPES,
|
||||
),
|
||||
),
|
||||
related_modules=("audit", "notifications"),
|
||||
related_modules=("audit", "files", "notifications"),
|
||||
translations={
|
||||
"de": {
|
||||
"title": "Laufzeitknoten leeren und Wiederherstellungsnachweise prüfen",
|
||||
"summary": "Ops projiziert gemeinsame Laufzeit-Heartbeats, Replikatlücken, Leerungssteuerung und Wiederherstellungszustände, die betriebliche Aufmerksamkeit erfordern.",
|
||||
"body": (
|
||||
"Verwenden Sie die Laufzeittabelle, um veraltete API- und Worker-Replikate oder Replikate mit abweichender Modulzusammensetzung zu erkennen. "
|
||||
"Leeren Sie einen Knoten vor dem Austausch, damit seine API-Bereitschaft geschlossen wird und Worker keine neue Warteschlangenarbeit annehmen; solange der Knoten noch geleert wird, kann der Vorgang abgebrochen werden. "
|
||||
"Die Wiederherstellungstabelle zeigt dauerhafte Core-Wiederherstellungsvorgänge. Eine abgelehnte Operation ist eine bestätigte Ablehnung des Anbieters und benötigt keine Wiederherstellung; Vorgänge mit unbekanntem Ergebnis oder erforderlicher Wiederherstellung müssen weiterhin im besitzenden Modul abgeglichen werden. "
|
||||
"Core-Einträge zum Modullebenszyklus sperren jede spätere Installation oder Änderung des laufenden Modulgraphen: Prüfen Sie anhand der Installer-Lauf-ID Paket-, Sicherungs-, Migrations- und Zustandsnachweise, bevor Sie zurückrollen oder vorwärts reparieren. "
|
||||
"SMTP- und IMAP-APPEND-Einträge von Mail verwenden stabile Versuchskennungen und ausschließlich Digest-Nachweise; gleichen Sie den Mail-Befehl mit Anbieternachweisen ab und wiederholen Sie niemals die ursprüngliche Wirkung aus Ops. "
|
||||
"Files erfasst Core-Wiederherstellungsnachweise für Blob-Schreibvorgänge, genehmigte harte Löschung, referenzgeprüfte Speicherbereinigung und bedingte S3-Connector-Schreibvorgänge. Prüfen Sie bei einem Files-Connector-Ergebnis Anbieteranfrage, Inhaltsmerkmale und Revision, bevor ein weiterer Schreibvorgang auf den gesperrten Pfad zugelassen wird; prüfen Sie bei der Blob-Bereinigung erneut FileVersion-Referenzen und die genaue Abwesenheit des Objekts. "
|
||||
"SQLite für die Entwicklung kann bei Blob-Uploads innerhalb einer aufrufenden Transaktion nur die Rekonstruktion behandelter Rollbacks zeigen; nach einem harten Prozessverlust muss die Files-Integritätsprüfung ausgeführt werden, weil ein verwaistes Objekt ohne Ops-Ledger-Eintrag existieren kann. "
|
||||
"Reine Datenbankläufe von Dataflow sind atomar, während Läufe mit veröffentlichten Ausgaben vorwärts repariert werden: Gleichen Sie den erfassten Ausgabedigest und den Idempotenzschlüssel des Ziels ab, bevor eine weitere Veröffentlichung erlaubt wird. "
|
||||
"Der Sicherungsstatus projiziert getrennt nur den bereinigten Bereitstellungsbeleg: Ein verifizierter Status weist einen koordinierten Wiederherstellungspunkt und eine isolierte Wiederherstellungsprobe nach; fehlende, abgelaufene oder ungültige Nachweise sperren eine migrationsbedingte Release-Änderung."
|
||||
),
|
||||
}
|
||||
},
|
||||
metadata={
|
||||
"kind": "workflow",
|
||||
"route": "/ops",
|
||||
@@ -172,12 +245,14 @@ manifest = ModuleManifest(
|
||||
"Compare active non-stale nodes with the configured API and worker replica expectations.",
|
||||
"Request drain and wait for the node to report draining before replacing it.",
|
||||
"Inspect every recovery-required, outcome-unknown, or manual-intervention operation and follow its recorded recovery mode.",
|
||||
"For Files, distinguish blob upload/repair, approved hard purge, blob garbage collection, and S3 connector write-back before checking the owning database/object or provider evidence.",
|
||||
"Confirm that backup evidence is verified and current before authorizing a release-changing migration.",
|
||||
"Verify replacement composition, readiness, queue consumers, and recovery evidence before closing the operation.",
|
||||
],
|
||||
"limitations": [
|
||||
"Drain is observed on the runtime heartbeat interval and does not forcibly terminate active work.",
|
||||
"Ops does not create or restore backups and never receives private artifact or key-custody references.",
|
||||
"SQLite is a development-only recovery profile; Files objects created before an unhandled process loss may require integrity-scan discovery because the caller-transaction intent was not committed.",
|
||||
"A verified receipt proves the recorded drill; it does not make an unsafe post-migration code rollback reversible.",
|
||||
],
|
||||
"help_contexts": [
|
||||
@@ -325,5 +400,10 @@ manifest = ModuleManifest(
|
||||
)
|
||||
|
||||
|
||||
manifest = with_documentation_structured_translations(
|
||||
manifest, locale="de", translations=GERMAN_STRUCTURED_TRANSLATIONS
|
||||
)
|
||||
|
||||
|
||||
def get_manifest() -> ModuleManifest:
|
||||
return manifest
|
||||
|
||||
@@ -0,0 +1,151 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
from types import SimpleNamespace
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from govoplan_core.core.infrastructure_capabilities import InfrastructureDependency
|
||||
from govoplan_ops.backend.infrastructure import (
|
||||
OpsInfrastructureDependencyProvider,
|
||||
deployment_capability_status,
|
||||
infrastructure_dependency_inventory,
|
||||
)
|
||||
|
||||
|
||||
class _Provider:
|
||||
module_id = "mail"
|
||||
capability_ids = ("mail.smtp",)
|
||||
|
||||
def infrastructure_dependencies(self) -> tuple[InfrastructureDependency, ...]:
|
||||
return (
|
||||
InfrastructureDependency(
|
||||
capability_id="mail.smtp",
|
||||
module_id="mail",
|
||||
dependency_type="smtp_endpoint",
|
||||
dependency_ref="mail:server-1",
|
||||
state="active",
|
||||
scope="system",
|
||||
summary="Persisted SMTP endpoint.",
|
||||
metrics={"credential_binding_count": 1},
|
||||
required_action="Rebind it before changing SMTP.",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class _Registry:
|
||||
def capability_names(self) -> tuple[str, ...]:
|
||||
return ("infrastructure.dependency_inventory.mail",)
|
||||
|
||||
def capability(self, name: str) -> object | None:
|
||||
return _Provider() if name.endswith(".mail") else None
|
||||
|
||||
|
||||
class InfrastructureCapabilityTests(unittest.TestCase):
|
||||
def test_collects_authorized_module_dependency_inventory(self) -> None:
|
||||
with patch(
|
||||
"govoplan_ops.backend.infrastructure.load_infrastructure_capability_receipt",
|
||||
return_value=SimpleNamespace(installation_id="govoplan-test"),
|
||||
):
|
||||
result = infrastructure_dependency_inventory(
|
||||
_Registry(),
|
||||
installation_id="govoplan-test",
|
||||
).to_dict()
|
||||
|
||||
self.assertTrue(result["complete"])
|
||||
self.assertEqual("mail.smtp", result["dependencies"][0]["capability_id"])
|
||||
self.assertEqual(1, result["dependencies"][0]["metrics"]["credential_binding_count"])
|
||||
|
||||
def test_ops_provider_reports_runtime_bindings_without_endpoint_secrets(self) -> None:
|
||||
dependencies = OpsInfrastructureDependencyProvider().infrastructure_dependencies()
|
||||
by_capability = {item.capability_id: item for item in dependencies}
|
||||
|
||||
self.assertIn("database.postgresql", by_capability)
|
||||
self.assertIn("network.ingress", by_capability)
|
||||
self.assertEqual(
|
||||
"runtime:database.postgresql",
|
||||
by_capability["database.postgresql"].dependency_ref,
|
||||
)
|
||||
self.assertNotIn("postgresql://", json.dumps([item.to_dict() for item in dependencies]))
|
||||
|
||||
def test_reads_bounded_non_secret_capability_receipt(self) -> None:
|
||||
with tempfile.TemporaryDirectory(prefix="govoplan-ops-capabilities-") as root:
|
||||
path = Path(root) / "capabilities.json"
|
||||
path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"schema_version": 1,
|
||||
"installation_id": "govoplan-test",
|
||||
"profile": "evaluation",
|
||||
"capabilities": [
|
||||
{
|
||||
"id": "mail.smtp",
|
||||
"label": "SMTP delivery",
|
||||
"state": "available_unconfigured",
|
||||
"source": "operator-supplied",
|
||||
"detail": "Mail needs a profile.",
|
||||
"endpoint": {},
|
||||
"secret_refs": ["env:SMTP_CREDENTIAL_REF"],
|
||||
"dependent_modules": ["mail"],
|
||||
}
|
||||
],
|
||||
"post_install_tasks": [
|
||||
{
|
||||
"id": "mail.smtp-profile",
|
||||
"resume_key": "govoplan-test:mail.smtp-profile:v1",
|
||||
"capability_id": "mail.smtp",
|
||||
"state": "pending",
|
||||
"owner_module": "mail",
|
||||
"summary": "Configure Mail.",
|
||||
"required_inputs": ["credential envelope reference"],
|
||||
"secret_boundary": "credential-envelope-reference-only",
|
||||
}
|
||||
],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
result = deployment_capability_status(path)
|
||||
|
||||
self.assertTrue(result["available"])
|
||||
self.assertEqual("mail.smtp", result["capabilities"][0]["id"])
|
||||
self.assertEqual("mail.smtp-profile", result["post_install_tasks"][0]["id"])
|
||||
|
||||
def test_rejects_inline_secret_instead_of_reference(self) -> None:
|
||||
with tempfile.TemporaryDirectory(prefix="govoplan-ops-capabilities-") as root:
|
||||
path = Path(root) / "capabilities.json"
|
||||
path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"schema_version": 1,
|
||||
"installation_id": "govoplan-test",
|
||||
"profile": "evaluation",
|
||||
"capabilities": [
|
||||
{
|
||||
"id": "mail.smtp",
|
||||
"label": "SMTP delivery",
|
||||
"state": "configured",
|
||||
"source": "operator-supplied",
|
||||
"detail": "Configured.",
|
||||
"endpoint": {},
|
||||
"secret_refs": ["plaintext-secret"],
|
||||
"dependent_modules": [],
|
||||
}
|
||||
],
|
||||
"post_install_tasks": [],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
result = deployment_capability_status(path)
|
||||
|
||||
self.assertFalse(result["available"])
|
||||
self.assertIn("environment references", str(result["error"]))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -54,6 +54,14 @@ class OpsInterfaceDocumentationContractTests(unittest.TestCase):
|
||||
self.assertIn("cancel_node_drain", recovery.metadata["consequence_classes"])
|
||||
self.assertIn("inspect_recovery", recovery.metadata["consequence_classes"])
|
||||
|
||||
def test_public_documentation_has_complete_german_baseline(self) -> None:
|
||||
for topic in get_manifest().documentation:
|
||||
german = topic.translations.get("de", {})
|
||||
self.assertTrue(
|
||||
all(german.get(field) for field in ("title", "summary", "body")),
|
||||
topic.id,
|
||||
)
|
||||
|
||||
def test_webui_uses_shared_operational_patterns(self) -> None:
|
||||
page = (REPO_ROOT / "webui/src/features/ops/OpsPage.tsx").read_text(encoding="utf-8")
|
||||
widget = (REPO_ROOT / "webui/src/features/ops/OpsHealthWidget.tsx").read_text(encoding="utf-8")
|
||||
|
||||
@@ -7,6 +7,9 @@ from pathlib import Path
|
||||
from govoplan_core.core.operations import (
|
||||
OperationalCheck,
|
||||
OperationalCheckProviderRegistration,
|
||||
RuntimeWorkStatus,
|
||||
RuntimeWorkStatusContext,
|
||||
RuntimeWorkStatusProviderRegistration,
|
||||
)
|
||||
from govoplan_ops.backend.api.v1 import routes
|
||||
|
||||
@@ -14,6 +17,7 @@ from govoplan_ops.backend.api.v1 import routes
|
||||
@dataclass
|
||||
class _Manifest:
|
||||
operational_check_providers: tuple[OperationalCheckProviderRegistration, ...]
|
||||
runtime_work_status_providers: tuple[RuntimeWorkStatusProviderRegistration, ...] = ()
|
||||
|
||||
|
||||
class _Registry:
|
||||
@@ -68,6 +72,98 @@ def test_module_operational_check_failure_is_isolated() -> None:
|
||||
assert "secret detail" not in result["detail"]
|
||||
|
||||
|
||||
def test_runtime_work_provider_cache_force_and_unknown_metrics() -> None:
|
||||
routes._runtime_work_cache.clear()
|
||||
calls = 0
|
||||
|
||||
def provider(context: RuntimeWorkStatusContext) -> RuntimeWorkStatus:
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
return RuntimeWorkStatus(
|
||||
provider_id="example.queue",
|
||||
label="Example queue",
|
||||
backend="Example",
|
||||
enabled=True,
|
||||
configured=True,
|
||||
state="healthy",
|
||||
detail="Workers answered; queue depth unsupported.",
|
||||
observed_at=context.observed_at,
|
||||
active_workers=1,
|
||||
queue_depths={"example": None},
|
||||
)
|
||||
|
||||
registry = _Registry()
|
||||
registry._manifest.runtime_work_status_providers = ( # type: ignore[misc]
|
||||
RuntimeWorkStatusProviderRegistration(
|
||||
module_id="example",
|
||||
provider_id="example.queue",
|
||||
provider=provider,
|
||||
),
|
||||
)
|
||||
context = RuntimeWorkStatusContext(
|
||||
profile="split-worker",
|
||||
observed_at=datetime.now(UTC),
|
||||
stale_after_seconds=60,
|
||||
)
|
||||
|
||||
first = routes._runtime_work_statuses(registry, context)
|
||||
second = routes._runtime_work_statuses(registry, context)
|
||||
forced = routes._runtime_work_statuses(registry, context, force=True)
|
||||
|
||||
assert first[0]["queue_depths"] == {"example": None}
|
||||
assert second == first
|
||||
assert forced[0]["state"] == "healthy"
|
||||
assert calls == 2
|
||||
|
||||
|
||||
def test_runtime_work_provider_failure_is_sanitized() -> None:
|
||||
routes._runtime_work_cache.clear()
|
||||
|
||||
def provider(context: RuntimeWorkStatusContext) -> RuntimeWorkStatus:
|
||||
del context
|
||||
raise RuntimeError("redis://user:secret@example.test")
|
||||
|
||||
registry = _Registry()
|
||||
registry._manifest.runtime_work_status_providers = ( # type: ignore[misc]
|
||||
RuntimeWorkStatusProviderRegistration(
|
||||
module_id="example",
|
||||
provider_id="example.failed",
|
||||
provider=provider,
|
||||
),
|
||||
)
|
||||
result = routes._runtime_work_statuses(
|
||||
registry,
|
||||
RuntimeWorkStatusContext(
|
||||
profile="split-worker",
|
||||
observed_at=datetime.now(UTC),
|
||||
stale_after_seconds=60,
|
||||
),
|
||||
force=True,
|
||||
)[0]
|
||||
|
||||
assert result["state"] == "unreachable"
|
||||
assert "secret" not in result["detail"]
|
||||
|
||||
|
||||
def test_disabled_workers_are_expected_only_in_development() -> None:
|
||||
disabled = {
|
||||
"provider_id": "example.queue",
|
||||
"state": "disabled",
|
||||
"detail": "Intentionally disabled.",
|
||||
"enabled": False,
|
||||
"configured": True,
|
||||
"queue_depths": {},
|
||||
}
|
||||
|
||||
development = routes._runtime_work_check([disabled], "local-dev")
|
||||
production = routes._runtime_work_check([disabled], "single-process")
|
||||
|
||||
assert development["state"] == "inactive"
|
||||
assert development["readiness_critical"] is False
|
||||
assert production["state"] == "warning"
|
||||
assert production["readiness_critical"] is True
|
||||
|
||||
|
||||
def test_shared_runtime_cluster_missing_replicas_blocks_readiness() -> None:
|
||||
check = routes._runtime_cluster_check(
|
||||
{
|
||||
|
||||
+5
-2
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@govoplan/ops-webui",
|
||||
"version": "0.1.15",
|
||||
"version": "0.1.21",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "src/index.ts",
|
||||
@@ -12,8 +12,11 @@
|
||||
"import": "./src/index.ts"
|
||||
}
|
||||
},
|
||||
"scripts": {
|
||||
"test:runtime-status": "rm -rf .runtime-status-test-build && mkdir -p .runtime-status-test-build && printf '{\"type\":\"commonjs\"}\\n' > .runtime-status-test-build/package.json && ../../govoplan-core/webui/node_modules/.bin/tsc -p tsconfig.runtime-status-tests.json && node .runtime-status-test-build/tests/runtime-status.test.js"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@govoplan/core-webui": "^0.1.15",
|
||||
"@govoplan/core-webui": "^0.1.18",
|
||||
"lucide-react": "^1.23.0",
|
||||
"react": ">=19.2.7 <20",
|
||||
"react-dom": ">=19.2.7 <20",
|
||||
|
||||
+60
-3
@@ -17,6 +17,28 @@ export type OpsDeploymentProfile = {
|
||||
fit: string;
|
||||
};
|
||||
|
||||
export type OpsInfrastructureCapability = {
|
||||
id: string;
|
||||
label: string;
|
||||
state: "configured" | "available_unconfigured" | "externally_supplied" | "unavailable";
|
||||
source: string;
|
||||
detail: string;
|
||||
endpoint: Record<string, string | number | boolean | null>;
|
||||
secret_refs: string[];
|
||||
dependent_modules: string[];
|
||||
};
|
||||
|
||||
export type OpsPostInstallTask = {
|
||||
id: string;
|
||||
resume_key: string;
|
||||
capability_id: string;
|
||||
state: string;
|
||||
owner_module: string;
|
||||
summary: string;
|
||||
required_inputs: string[];
|
||||
secret_boundary: string;
|
||||
};
|
||||
|
||||
export type OpsSizingAssumption = {
|
||||
area: string;
|
||||
baseline: string;
|
||||
@@ -147,6 +169,25 @@ export type OpsRuntimeCluster = {
|
||||
};
|
||||
};
|
||||
|
||||
export type OpsRuntimeWorkStatus = {
|
||||
provider_id: string;
|
||||
label: string;
|
||||
backend: string;
|
||||
enabled: boolean | null;
|
||||
configured: boolean | null;
|
||||
state: "disabled" | "unconfigured" | "starting" | "healthy" | "idle" | "busy" | "degraded" | "stale" | "unreachable" | string;
|
||||
detail: string;
|
||||
observed_at: string;
|
||||
active_workers?: number | null;
|
||||
last_heartbeat_at?: string | null;
|
||||
queue_depths: Record<string, number | null>;
|
||||
active_work?: number | null;
|
||||
reserved_work?: number | null;
|
||||
failures?: number | null;
|
||||
stale_after_seconds?: number | null;
|
||||
guidance: string;
|
||||
};
|
||||
|
||||
export type OpsStatus = {
|
||||
summary: {
|
||||
app_env: string;
|
||||
@@ -173,12 +214,15 @@ export type OpsStatus = {
|
||||
};
|
||||
backup_state?: string;
|
||||
worker_metrics: {
|
||||
workers?: number;
|
||||
active_tasks?: number;
|
||||
state?: string;
|
||||
workers?: number | null;
|
||||
active_tasks?: number | null;
|
||||
reserved_tasks?: number | null;
|
||||
failures?: number | null;
|
||||
expected_queues?: string[];
|
||||
active_queues?: string[];
|
||||
missing_queues?: string[];
|
||||
queue_depths?: Record<string, number>;
|
||||
queue_depths?: Record<string, number | null>;
|
||||
};
|
||||
operational_probe_count: number;
|
||||
runtime_node_count: number;
|
||||
@@ -186,6 +230,8 @@ export type OpsStatus = {
|
||||
failed_operation_count?: number;
|
||||
outcome_unknown_count?: number;
|
||||
active_operation_count?: number;
|
||||
infrastructure_capability_count?: number;
|
||||
pending_post_install_task_count?: number;
|
||||
};
|
||||
readiness: {
|
||||
ready: boolean;
|
||||
@@ -220,6 +266,17 @@ export type OpsStatus = {
|
||||
deployment_profiles: OpsDeploymentProfile[];
|
||||
sizing: OpsSizingAssumption[];
|
||||
runtime_cluster: OpsRuntimeCluster;
|
||||
runtime_work: OpsRuntimeWorkStatus[];
|
||||
infrastructure: {
|
||||
configured: boolean;
|
||||
available: boolean;
|
||||
schema_version?: number | null;
|
||||
installation_id?: string | null;
|
||||
profile?: string | null;
|
||||
capabilities: OpsInfrastructureCapability[];
|
||||
post_install_tasks: OpsPostInstallTask[];
|
||||
error?: string | null;
|
||||
};
|
||||
};
|
||||
|
||||
export function fetchOpsStatus(settings: ApiSettings): Promise<OpsStatus> {
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { DescriptionList } from "@govoplan/core-webui";
|
||||
import { MetricGrid } from "@govoplan/core-webui";
|
||||
import { useEffect, useState } from "react";
|
||||
import {
|
||||
DismissibleAlert,
|
||||
@@ -10,6 +12,7 @@ import {
|
||||
} from "@govoplan/core-webui";
|
||||
import { fetchOpsStatus, type OpsStatus } from "../../api/ops";
|
||||
import { OPS_DOCUMENTATION } from "./interfacePatterns";
|
||||
import { knownMetric, knownQueueDepthTotal, runtimeWorkTone } from "./runtimeStatus";
|
||||
|
||||
export default function OpsHealthWidget({ settings, refreshKey }: { settings: ApiSettings; refreshKey: number }) {
|
||||
const [status, setStatus] = useState<OpsStatus | null>(null);
|
||||
@@ -33,8 +36,7 @@ export default function OpsHealthWidget({ settings, refreshKey }: { settings: Ap
|
||||
const ready = status?.readiness.ready ?? false;
|
||||
const workerMetrics = status?.summary.worker_metrics;
|
||||
const queueDepths = workerMetrics?.queue_depths ?? {};
|
||||
const queuedTasks = Object.values(queueDepths).reduce((sum, value) => sum + value, 0);
|
||||
const missingQueues = workerMetrics?.missing_queues ?? [];
|
||||
const queuedTasks = knownQueueDepthTotal(queueDepths);
|
||||
const storageUsage = status?.summary.storage_metrics?.capacity_used_percent;
|
||||
|
||||
return (
|
||||
@@ -43,27 +45,27 @@ export default function OpsHealthWidget({ settings, refreshKey }: { settings: Ap
|
||||
<div className="button-row compact-actions">
|
||||
<DocumentationHelpLink reference={OPS_DOCUMENTATION} />
|
||||
</div>
|
||||
<div className="metric-grid inside dashboard-widget-metrics">
|
||||
<MetricGrid columns={3} spacing="none">
|
||||
<MetricCard label="Readiness" value={ready ? "ready" : "blocked"} tone={ready ? "good" : "danger"} detail={status?.readiness.profile ?? "-"} />
|
||||
<MetricCard label="Workers" value={status?.summary.celery_enabled ? workerMetrics?.workers ?? 0 : "off"} tone={status?.summary.celery_enabled && !missingQueues.length ? "good" : "warning"} detail={missingQueues.length ? `${missingQueues.length} queue(s) without a consumer` : status?.summary.celery_enabled ? "All configured queues covered" : "Single-process mode"} />
|
||||
<MetricCard label="Active tasks" value={workerMetrics?.active_tasks ?? 0} tone="info" detail={status?.summary.celery_enabled ? "Reported by live workers" : "Workers disabled"} />
|
||||
<MetricCard label="Queued tasks" value={queuedTasks} tone={queuedTasks ? "warning" : "neutral"} detail={Object.keys(queueDepths).length ? `${Object.keys(queueDepths).length} measured queue(s)` : "Queue depth unavailable"} />
|
||||
<MetricCard label="Workers" value={knownMetric(workerMetrics?.workers)} tone={runtimeWorkTone(workerMetrics?.state ?? "unreachable")} detail={workerMetrics?.state ?? "unavailable"} />
|
||||
<MetricCard label="Active tasks" value={knownMetric(workerMetrics?.active_tasks)} tone="info" detail={workerMetrics?.active_tasks == null ? "Metric unavailable" : "Reported by the runtime provider"} />
|
||||
<MetricCard label="Queued tasks" value={queuedTasks ?? "unavailable"} tone={queuedTasks === null ? "neutral" : queuedTasks ? "warning" : "good"} detail={queuedTasks === null ? "Queue depth unavailable" : `${Object.values(queueDepths).filter((value) => typeof value === "number").length} measured queue(s)`} />
|
||||
<MetricCard label="Storage" value={storageUsage === undefined ? status?.summary.file_storage_backend ?? "-" : `${storageUsage}%`} tone={storageUsage !== undefined && storageUsage >= 90 ? "danger" : storageUsage !== undefined && storageUsage >= 75 ? "warning" : "neutral"} detail="Managed Files backend capacity" />
|
||||
<MetricCard label="Failed" value={status?.summary.failed_operation_count ?? 0} tone={status?.summary.failed_operation_count ? "danger" : "good"} detail="Recovery-ledger operations" />
|
||||
<MetricCard label="Unknown" value={status?.summary.outcome_unknown_count ?? 0} tone={status?.summary.outcome_unknown_count ? "danger" : "good"} detail="Outcome-unknown operations" />
|
||||
<MetricCard label="Backup" value={status?.summary.backup_state ?? "unknown"} tone={status?.summary.backup_state === "ok" ? "good" : "warning"} detail="Backup and restore evidence" />
|
||||
<MetricCard label="Probes" value={status?.summary.operational_probe_count ?? 0} tone="neutral" detail="Module-owned operational checks" />
|
||||
<MetricCard label="Warnings" value={warningCount + errorCount} tone={errorCount ? "danger" : warningCount ? "warning" : "good"} detail="Current health checks" />
|
||||
</div>
|
||||
</MetricGrid>
|
||||
{status?.readiness.blockers.length ?
|
||||
<dl className="detail-list dashboard-compact-list below-grid">
|
||||
<DescriptionList variant="inline" termWidth="compact" className="below-grid">
|
||||
{status.readiness.blockers.slice(0, 3).map((blocker) =>
|
||||
<div key={blocker.id}>
|
||||
<dt><StatusBadge status={blocker.state === "error" ? "error" : "warning"} label={blocker.state} /></dt>
|
||||
<dd><strong>{blocker.label}</strong><span className="muted"> · {blocker.detail}</span></dd>
|
||||
</div>
|
||||
)}
|
||||
</dl> :
|
||||
</DescriptionList> :
|
||||
<p className="muted below-grid">No readiness blockers reported.</p>
|
||||
}
|
||||
</LoadingFrame>);
|
||||
|
||||
@@ -1,17 +1,18 @@
|
||||
import { useEffect, useState } from "react";
|
||||
import { DescriptionList } from "@govoplan/core-webui";
|
||||
import { MetricGrid } from "@govoplan/core-webui";
|
||||
import { useCallback, useEffect, useRef, useState } from "react";
|
||||
import { PauseCircle, PlayCircle, RefreshCw } from "lucide-react";
|
||||
import {
|
||||
import { ContentGrid,
|
||||
ActionBlockerHint,
|
||||
Button,
|
||||
Card,
|
||||
ConfirmDialog,
|
||||
DataGrid,
|
||||
DismissibleAlert,
|
||||
DocumentationHelpLink,
|
||||
LoadingFrame,
|
||||
MetricCard,
|
||||
PageScrollViewport,
|
||||
PageTitle,
|
||||
PageActionBar,
|
||||
PageLayout,
|
||||
StatusBadge,
|
||||
TableActionGroup,
|
||||
adminErrorMessage,
|
||||
@@ -29,8 +30,10 @@ import {
|
||||
type OpsCheck,
|
||||
type OpsDeploymentProfile,
|
||||
type OpsGovernanceModule,
|
||||
type OpsInfrastructureCapability,
|
||||
type OpsRecoveryOperation,
|
||||
type OpsRuntimeNode,
|
||||
type OpsRuntimeWorkStatus,
|
||||
type OpsSizingAssumption,
|
||||
type OpsStatus
|
||||
} from "../../api/ops";
|
||||
@@ -39,6 +42,14 @@ import {
|
||||
OPS_I18N,
|
||||
OPS_RECOVERY_DOCUMENTATION,
|
||||
} from "./interfacePatterns";
|
||||
import {
|
||||
heartbeatAgeLabel,
|
||||
heartbeatAgeSeconds,
|
||||
knownMetric,
|
||||
knownQueueDepthTotal,
|
||||
runtimeWorkTone,
|
||||
shouldPollRuntimeStatus
|
||||
} from "./runtimeStatus";
|
||||
|
||||
export default function OpsPage({ settings, auth }: {settings: ApiSettings;auth: AuthInfo;}) {
|
||||
const [status, setStatus] = useState<OpsStatus | null>(null);
|
||||
@@ -47,18 +58,27 @@ export default function OpsPage({ settings, auth }: {settings: ApiSettings;auth:
|
||||
const [error, setError] = useState("");
|
||||
const [drainTarget, setDrainTarget] = useState<OpsRuntimeNode | null>(null);
|
||||
const [nodeActionId, setNodeActionId] = useState("");
|
||||
const loadInFlight = useRef<Promise<void> | null>(null);
|
||||
|
||||
async function load() {
|
||||
setLoading(true);
|
||||
setError("");
|
||||
try {
|
||||
setStatus(await fetchOpsStatus(settings));
|
||||
} catch (err) {
|
||||
setError(adminErrorMessage(err));
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
}
|
||||
const load = useCallback((background = false): Promise<void> => {
|
||||
if (loadInFlight.current) return loadInFlight.current;
|
||||
const request = (async () => {
|
||||
if (!background) setLoading(true);
|
||||
setError("");
|
||||
try {
|
||||
setStatus(await fetchOpsStatus(settings));
|
||||
} catch (err) {
|
||||
setError(adminErrorMessage(err));
|
||||
} finally {
|
||||
if (!background) setLoading(false);
|
||||
}
|
||||
})();
|
||||
loadInFlight.current = request;
|
||||
void request.finally(() => {
|
||||
if (loadInFlight.current === request) loadInFlight.current = null;
|
||||
});
|
||||
return request;
|
||||
}, [settings.accessToken, settings.apiBaseUrl, settings.apiKey]);
|
||||
|
||||
async function runChecks() {
|
||||
setRunningProbes(true);
|
||||
@@ -102,7 +122,37 @@ export default function OpsPage({ settings, auth }: {settings: ApiSettings;auth:
|
||||
}
|
||||
}
|
||||
|
||||
useEffect(() => {void load();}, [settings.apiBaseUrl, settings.apiKey, settings.accessToken]);
|
||||
useEffect(() => {
|
||||
let intervalId: number | null = null;
|
||||
const stopPolling = () => {
|
||||
if (intervalId !== null) window.clearInterval(intervalId);
|
||||
intervalId = null;
|
||||
};
|
||||
const startPolling = () => {
|
||||
stopPolling();
|
||||
if (document.visibilityState === "hidden") return;
|
||||
intervalId = window.setInterval(() => {
|
||||
if (shouldPollRuntimeStatus(document.visibilityState === "hidden", Boolean(loadInFlight.current))) {
|
||||
void load(true);
|
||||
}
|
||||
}, 15_000);
|
||||
};
|
||||
const handleVisibility = () => {
|
||||
if (document.visibilityState === "hidden") {
|
||||
stopPolling();
|
||||
return;
|
||||
}
|
||||
if (shouldPollRuntimeStatus(false, Boolean(loadInFlight.current))) void load(true);
|
||||
startPolling();
|
||||
};
|
||||
void load();
|
||||
startPolling();
|
||||
document.addEventListener("visibilitychange", handleVisibility);
|
||||
return () => {
|
||||
stopPolling();
|
||||
document.removeEventListener("visibilitychange", handleVisibility);
|
||||
};
|
||||
}, [load]);
|
||||
|
||||
const checks = status?.checks ?? [];
|
||||
const warningCount = checks.filter((item) => item.state === "warning").length;
|
||||
@@ -117,25 +167,23 @@ export default function OpsPage({ settings, auth }: {settings: ApiSettings;auth:
|
||||
? OPS_I18N.runPermissionRequired
|
||||
: undefined;
|
||||
const queueDepths = status?.summary.worker_metrics.queue_depths ?? {};
|
||||
const queuedTasks = Object.values(queueDepths).reduce((total, value) => total + value, 0);
|
||||
const queuedTasks = knownQueueDepthTotal(queueDepths);
|
||||
const storageUsage = status?.summary.storage_metrics?.capacity_used_percent;
|
||||
|
||||
return (
|
||||
<PageScrollViewport>
|
||||
<div className="content-pad workspace-data-page">
|
||||
<div className="page-heading split workspace-heading">
|
||||
<div>
|
||||
<PageTitle loading={loading}>i18n:govoplan-ops.ops.907a54c2</PageTitle>
|
||||
<p>i18n:govoplan-ops.runtime_health_deployment_profile_worker_split_a.55340156</p>
|
||||
</div>
|
||||
<div className="button-row compact-actions">
|
||||
<DocumentationHelpLink reference={OPS_DOCUMENTATION} />
|
||||
<Button variant="primary" onClick={() => void runChecks()} disabled={Boolean(runProbesDisabledReason)} disabledReason={runProbesDisabledReason}>i18n:govoplan-ops.surface.run_probes</Button>
|
||||
<Button onClick={() => void load()} disabled={loading} disabledReason={loading ? OPS_I18N.loading : undefined}><RefreshCw size={16} /> i18n:govoplan-ops.reload.cce71553</Button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{error && <DismissibleAlert tone="danger" resetKey={error} floating>{error}</DismissibleAlert>}
|
||||
<PageLayout
|
||||
archetype="overview"
|
||||
title="i18n:govoplan-ops.ops.907a54c2"
|
||||
description="i18n:govoplan-ops.runtime_health_deployment_profile_worker_split_a.55340156"
|
||||
error={error}
|
||||
actions={<PageActionBar
|
||||
variant="overview"
|
||||
refreshable
|
||||
reloadAction={{ onReload: () => void load(), loading, disabledReason: loading ? OPS_I18N.loading : undefined }}
|
||||
helpAction={<DocumentationHelpLink reference={OPS_DOCUMENTATION} />}
|
||||
primaryActions={<Button variant="primary" onClick={() => void runChecks()} disabled={Boolean(runProbesDisabledReason)} disabledReason={runProbesDisabledReason}>i18n:govoplan-ops.surface.run_probes</Button>}
|
||||
/>}
|
||||
>
|
||||
{status && !ready && (
|
||||
<ActionBlockerHint
|
||||
reason={{
|
||||
@@ -150,14 +198,14 @@ export default function OpsPage({ settings, auth }: {settings: ApiSettings;auth:
|
||||
)}
|
||||
|
||||
<LoadingFrame loading={loading} label="i18n:govoplan-ops.loading_operations_status.6890fe6e">
|
||||
<div className="metric-grid">
|
||||
<MetricGrid>
|
||||
<MetricCard label="i18n:govoplan-ops.profile.ff4fc027" value={status?.summary.active_profile ?? "-"} tone="info" detail={status?.summary.database_url ?? "i18n:govoplan-ops.no_database_url.51a2db0c"} />
|
||||
<MetricCard label="i18n:govoplan-ops.readiness.1db9d6fb" value={ready ? "ready" : "not ready"} tone={ready ? "good" : "danger"} detail={status?.readiness.blockers.length ? `${status.readiness.blockers.length} blocker(s)` : "i18n:govoplan-ops.no_readiness_blockers.0df259bd"} />
|
||||
<MetricCard label="i18n:govoplan-ops.modules.04e9462c" value={status?.summary.module_count ?? 0} tone="neutral" detail="i18n:govoplan-ops.enabled_in_the_runtime_registry.d2c6142d" />
|
||||
<MetricCard label="i18n:govoplan-ops.permissions.842c35eb" value={status?.governance.summary.permission_count ?? 0} tone="neutral" detail="i18n:govoplan-ops.declared_governance_permissions.d08d3bf1" />
|
||||
<MetricCard label="i18n:govoplan-ops.policies.e7800f56" value={status?.governance.summary.policy_count ?? 0} tone="neutral" detail="i18n:govoplan-ops.registered_policy_capabilities.112a2b64" />
|
||||
<MetricCard label="i18n:govoplan-ops.workers.b6ef3acd" value={status?.summary.celery_enabled ? status.summary.worker_metrics.workers ?? 0 : "off"} tone={status?.summary.celery_enabled && !(status.summary.worker_metrics.missing_queues?.length) ? "good" : "warning"} detail={workerMetricDetail(status)} />
|
||||
<MetricCard label="Queued tasks" value={queuedTasks} tone={queuedTasks ? "warning" : "good"} detail={Object.keys(queueDepths).length ? `${Object.keys(queueDepths).length} measured queue(s)` : "Queue depth unavailable"} />
|
||||
<MetricCard label="i18n:govoplan-ops.workers.b6ef3acd" value={knownMetric(status?.summary.worker_metrics.workers)} tone={runtimeWorkTone(status?.summary.worker_metrics.state ?? "unreachable")} detail={workerMetricDetail(status)} />
|
||||
<MetricCard label="Queued tasks" value={queuedTasks ?? "unavailable"} tone={queuedTasks === null ? "neutral" : queuedTasks ? "warning" : "good"} detail={queuedTasks === null ? "Queue depth unavailable" : `${Object.values(queueDepths).filter((value) => typeof value === "number").length} measured queue(s)`} />
|
||||
<MetricCard label="Storage" value={storageUsage === undefined ? status?.summary.file_storage_backend ?? "-" : `${storageUsage}%`} tone={storageUsage !== undefined && storageUsage >= 90 ? "danger" : storageUsage !== undefined && storageUsage >= 75 ? "warning" : "neutral"} detail={storageMetricDetail(status)} />
|
||||
<MetricCard label="Backup evidence" value={status?.summary.backup_state ?? "unknown"} tone={status?.summary.backup_state === "ok" ? "good" : "warning"} detail="Latest coordinated backup and restore-drill evidence" />
|
||||
<MetricCard label="Failed operations" value={status?.summary.failed_operation_count ?? 0} tone={status?.summary.failed_operation_count ? "danger" : "good"} detail="Terminal failures or manual intervention" />
|
||||
@@ -168,9 +216,10 @@ export default function OpsPage({ settings, auth }: {settings: ApiSettings;auth:
|
||||
<MetricCard label="Database capacity" value={databaseCapacityValue(status)} tone={databaseCapacityTone(status)} detail="Peak pooled connections / available connections" />
|
||||
<MetricCard label="Recovery" value={status?.summary.recovery_required_count ?? 0} tone={status?.summary.recovery_required_count ? "danger" : "good"} detail="Operations requiring recovery attention" />
|
||||
<MetricCard label="Provider bindings" value={status?.governance.summary.configured_external_provider_count ?? 0} tone={status?.governance.summary.provider_attention_count ? "warning" : "good"} detail={`${status?.governance.summary.provider_attention_count ?? 0} requiring attention`} />
|
||||
</div>
|
||||
<MetricCard label="i18n:govoplan-ops.infrastructure_capabilities" value={status?.summary.infrastructure_capability_count ?? 0} tone={status?.infrastructure.available ? "good" : "neutral"} detail={i18nMessage("i18n:govoplan-ops.pending_post_install_tasks", { value0: status?.summary.pending_post_install_task_count ?? 0 })} />
|
||||
</MetricGrid>
|
||||
|
||||
<div className="dashboard-grid">
|
||||
<ContentGrid columns={2} collapseAt="workspace" className="">
|
||||
<Card title="i18n:govoplan-ops.health_checks.201c869f">
|
||||
<CheckList checks={checks} />
|
||||
</Card>
|
||||
@@ -185,6 +234,10 @@ export default function OpsPage({ settings, auth }: {settings: ApiSettings;auth:
|
||||
/>
|
||||
</Card>
|
||||
|
||||
<Card title="Worker and queue readiness">
|
||||
<RuntimeWorkTable items={status?.runtime_work ?? []} />
|
||||
</Card>
|
||||
|
||||
<Card title="Recovery evidence">
|
||||
<RecoveryTable operations={status?.runtime_cluster.recovery.operations ?? []} />
|
||||
</Card>
|
||||
@@ -197,10 +250,20 @@ export default function OpsPage({ settings, auth }: {settings: ApiSettings;auth:
|
||||
<ProfileList profiles={status?.deployment_profiles ?? []} />
|
||||
</Card>
|
||||
|
||||
<Card title="i18n:govoplan-ops.infrastructure_capabilities">
|
||||
<InfrastructureCapabilityTable items={status?.infrastructure.capabilities ?? []} />
|
||||
{(status?.infrastructure.post_install_tasks.length ?? 0) > 0 && <DescriptionList variant="inline">
|
||||
{status?.infrastructure.post_install_tasks.map((task) => <div key={task.resume_key}>
|
||||
<dt><StatusBadge status="warning" label={task.state} /></dt>
|
||||
<dd><strong>{task.summary}</strong><span className="muted"> · {task.owner_module} · {task.required_inputs.join(", ")}</span></dd>
|
||||
</div>)}
|
||||
</DescriptionList>}
|
||||
</Card>
|
||||
|
||||
<Card title="i18n:govoplan-ops.sizing_assumptions.6ade9a90">
|
||||
<SizingTable items={status?.sizing ?? []} />
|
||||
</Card>
|
||||
</div>
|
||||
</ContentGrid>
|
||||
</LoadingFrame>
|
||||
<ConfirmDialog
|
||||
open={Boolean(drainTarget)}
|
||||
@@ -211,11 +274,74 @@ export default function OpsPage({ settings, auth }: {settings: ApiSettings;auth:
|
||||
onCancel={() => setDrainTarget(null)}
|
||||
onConfirm={() => void confirmDrain()}
|
||||
/>
|
||||
</div>
|
||||
</PageScrollViewport>);
|
||||
</PageLayout>);
|
||||
|
||||
}
|
||||
|
||||
function RuntimeWorkTable({ items }: { items: OpsRuntimeWorkStatus[] }) {
|
||||
const columns: DataGridColumn<OpsRuntimeWorkStatus>[] = [
|
||||
{
|
||||
id: "provider",
|
||||
header: "Backend",
|
||||
width: "minmax(200px, 1fr)",
|
||||
minWidth: 180,
|
||||
resizable: true,
|
||||
sortable: true,
|
||||
filterable: true,
|
||||
value: (item) => `${item.label} ${item.backend}`,
|
||||
render: (item) => <div><strong>{item.label}</strong><span className="muted block">{item.backend} · {item.provider_id}</span></div>
|
||||
},
|
||||
{
|
||||
id: "state",
|
||||
header: "State",
|
||||
width: 150,
|
||||
sortable: true,
|
||||
filterable: true,
|
||||
value: (item) => item.state,
|
||||
render: (item) => <StatusBadge status={stateTone(item.state)} label={item.state} />
|
||||
},
|
||||
{
|
||||
id: "activity",
|
||||
header: "Activity",
|
||||
width: "minmax(180px, .7fr)",
|
||||
minWidth: 170,
|
||||
value: (item) => `${item.active_workers ?? ""} ${item.active_work ?? ""} ${item.reserved_work ?? ""}`,
|
||||
render: (item) => `${knownMetric(item.active_workers)} workers · ${knownMetric(item.active_work)} active · ${knownMetric(item.reserved_work)} reserved`
|
||||
},
|
||||
{
|
||||
id: "queues",
|
||||
header: "Queue depth",
|
||||
width: "minmax(220px, 1fr)",
|
||||
minWidth: 190,
|
||||
resizable: true,
|
||||
value: (item) => Object.entries(item.queue_depths).map(([queue, depth]) => `${queue}:${depth ?? "unavailable"}`).join(" "),
|
||||
render: (item) => runtimeQueueSummary(item.queue_depths)
|
||||
},
|
||||
{
|
||||
id: "heartbeat",
|
||||
header: "Last heartbeat",
|
||||
width: "minmax(180px, .7fr)",
|
||||
minWidth: 170,
|
||||
sortable: true,
|
||||
value: (item) => item.last_heartbeat_at ?? "",
|
||||
render: (item) => {
|
||||
const age = heartbeatAgeSeconds(Date.now(), item.last_heartbeat_at);
|
||||
return <div>{heartbeatAgeLabel(age)}<span className="muted block">stale after {item.stale_after_seconds ?? "unavailable"}s</span></div>;
|
||||
}
|
||||
},
|
||||
{
|
||||
id: "guidance",
|
||||
header: "Guidance",
|
||||
width: "minmax(260px, 1.3fr)",
|
||||
minWidth: 220,
|
||||
resizable: true,
|
||||
value: (item) => `${item.detail} ${item.guidance}`,
|
||||
render: (item) => <div>{item.detail}<span className="muted block">{item.guidance}</span></div>
|
||||
}
|
||||
];
|
||||
return <DataGrid id="ops-runtime-work-status" rows={items} columns={columns} getRowKey={(item) => item.provider_id} emptyText="Worker and queue status unavailable." />;
|
||||
}
|
||||
|
||||
function RuntimeNodeTable({
|
||||
nodes,
|
||||
canManage,
|
||||
@@ -258,7 +384,7 @@ function RuntimeNodeTable({
|
||||
resizable: true,
|
||||
sortable: true,
|
||||
value: (node) => node.last_heartbeat_at,
|
||||
render: (node) => new Date(node.last_heartbeat_at).toLocaleString()
|
||||
render: (node) => <div>{heartbeatAgeLabel(heartbeatAgeSeconds(Date.now(), node.last_heartbeat_at))}<span className="muted block">{new Date(node.last_heartbeat_at).toLocaleString()}</span></div>
|
||||
},
|
||||
{
|
||||
id: "queues",
|
||||
@@ -470,14 +596,14 @@ function GovernanceTable({ modules }: { modules: OpsGovernanceModule[] }) {
|
||||
function CheckList({ checks }: {checks: OpsCheck[];}) {
|
||||
if (!checks.length) return <p className="muted">i18n:govoplan-ops.no_health_checks_reported.03c067c4</p>;
|
||||
return (
|
||||
<dl className="detail-list">
|
||||
<DescriptionList variant="inline">
|
||||
{checks.map((check) =>
|
||||
<div key={check.id}>
|
||||
<dt><StatusBadge status={stateTone(check.state)} label={check.state} /></dt>
|
||||
<dd><strong>{check.label}</strong><span className="muted"> · {check.detail}</span></dd>
|
||||
</div>
|
||||
)}
|
||||
</dl>);
|
||||
</DescriptionList>);
|
||||
|
||||
}
|
||||
|
||||
@@ -507,10 +633,38 @@ function SizingTable({ items }: {items: OpsSizingAssumption[];}) {
|
||||
|
||||
}
|
||||
|
||||
function InfrastructureCapabilityTable({ items }: {items: OpsInfrastructureCapability[];}) {
|
||||
if (!items.length) return <p className="muted">i18n:govoplan-ops.no_infrastructure_capabilities</p>;
|
||||
const columns: DataGridColumn<OpsInfrastructureCapability>[] = [
|
||||
{ id: "capability", header: "i18n:govoplan-ops.capability", width: "minmax(220px, 1fr)", minWidth: 190, resizable: true, sortable: true, filterable: true, value: (item) => `${item.label} ${item.id}`, render: (item) => <div><strong>{item.label}</strong><span className="muted block">{item.id}</span></div> },
|
||||
{ id: "state", header: "i18n:govoplan-ops.status.bae7d5be", width: 190, sortable: true, filterable: true, value: (item) => item.state, render: (item) => <StatusBadge status={capabilityTone(item.state)} label={item.state.replaceAll("_", " ")} /> },
|
||||
{ id: "source", header: "i18n:govoplan-ops.source", width: "minmax(180px, .7fr)", minWidth: 160, resizable: true, filterable: true, value: (item) => item.source },
|
||||
{ id: "endpoint", header: "i18n:govoplan-ops.endpoint", width: "minmax(220px, 1fr)", minWidth: 180, resizable: true, filterable: true, value: (item) => endpointLabel(item.endpoint) },
|
||||
{ id: "consumers", header: "i18n:govoplan-ops.consumers", width: "minmax(220px, 1fr)", minWidth: 180, resizable: true, filterable: true, value: (item) => item.dependent_modules.join(" "), render: (item) => item.dependent_modules.join(", ") || "-" }
|
||||
];
|
||||
return <DataGrid id="ops-infrastructure-capabilities" rows={items} columns={columns} getRowKey={(item) => item.id} />;
|
||||
}
|
||||
|
||||
function endpointLabel(endpoint: OpsInfrastructureCapability["endpoint"]): string {
|
||||
if (typeof endpoint.host === "string") {
|
||||
const scheme = typeof endpoint.scheme === "string" ? `${endpoint.scheme}://` : "";
|
||||
const port = typeof endpoint.port === "number" ? `:${endpoint.port}` : "";
|
||||
return `${scheme}${endpoint.host}${port}`;
|
||||
}
|
||||
return typeof endpoint.reference === "string" ? endpoint.reference : "-";
|
||||
}
|
||||
|
||||
function capabilityTone(state: OpsInfrastructureCapability["state"]): string {
|
||||
if (state === "configured" || state === "externally_supplied") return "success";
|
||||
if (state === "available_unconfigured") return "warning";
|
||||
return "inactive";
|
||||
}
|
||||
|
||||
function stateTone(state: string): string {
|
||||
if (state === "ok") return "success";
|
||||
if (state === "warning") return "warning";
|
||||
if (state === "error") return "error";
|
||||
if (["ok", "healthy", "idle"].includes(state)) return "success";
|
||||
if (state === "busy") return "info";
|
||||
if (["warning", "starting", "degraded", "unconfigured"].includes(state)) return "warning";
|
||||
if (["error", "stale", "unreachable"].includes(state)) return "error";
|
||||
return "inactive";
|
||||
}
|
||||
|
||||
@@ -549,11 +703,17 @@ function databaseCapacityTone(status: OpsStatus | null): "good" | "warning" | "d
|
||||
}
|
||||
|
||||
function workerMetricDetail(status: OpsStatus | null): string {
|
||||
if (!status?.summary.celery_enabled) return "i18n:govoplan-ops.celery_worker_setting.323d7737";
|
||||
if (!status) return "Worker status unavailable";
|
||||
const metrics = status.summary.worker_metrics;
|
||||
if (metrics.missing_queues?.length) return `Missing queues: ${metrics.missing_queues.join(", ")}`;
|
||||
const queued = Object.values(metrics.queue_depths ?? {}).reduce((total, value) => total + value, 0);
|
||||
return `${metrics.active_tasks ?? 0} active · ${queued} queued`;
|
||||
if (metrics.state === "disabled") return "Workers intentionally disabled";
|
||||
const queued = knownQueueDepthTotal(metrics.queue_depths ?? {});
|
||||
return `${knownMetric(metrics.active_tasks)} active · ${knownMetric(metrics.reserved_tasks)} reserved · ${queued ?? "unavailable"} queued`;
|
||||
}
|
||||
|
||||
function runtimeQueueSummary(depths: Record<string, number | null>): string {
|
||||
const entries = Object.entries(depths);
|
||||
if (!entries.length) return "unavailable";
|
||||
return entries.map(([queue, depth]) => `${queue}: ${depth ?? "unavailable"}`).join(" · ");
|
||||
}
|
||||
|
||||
function storageMetricDetail(status: OpsStatus | null): string {
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
export function knownMetric(value: number | null | undefined): number | "unavailable" {
|
||||
return typeof value === "number" && Number.isFinite(value) ? value : "unavailable";
|
||||
}
|
||||
|
||||
export function knownQueueDepthTotal(depths: Record<string, number | null | undefined>): number | null {
|
||||
const values = Object.values(depths).filter((value): value is number => typeof value === "number" && Number.isFinite(value));
|
||||
return values.length ? values.reduce((total, value) => total + value, 0) : null;
|
||||
}
|
||||
|
||||
export function runtimeWorkTone(state: string): "good" | "info" | "warning" | "danger" | "neutral" {
|
||||
if (state === "healthy" || state === "idle") return "good";
|
||||
if (state === "busy") return "info";
|
||||
if (state === "starting" || state === "degraded" || state === "unconfigured") return "warning";
|
||||
if (state === "stale" || state === "unreachable") return "danger";
|
||||
return "neutral";
|
||||
}
|
||||
|
||||
export function heartbeatAgeSeconds(now: number, value?: string | null): number | null {
|
||||
if (!value) return null;
|
||||
const timestamp = Date.parse(value);
|
||||
if (!Number.isFinite(timestamp)) return null;
|
||||
return Math.max(0, Math.floor((now - timestamp) / 1000));
|
||||
}
|
||||
|
||||
export function heartbeatAgeLabel(ageSeconds: number | null): string {
|
||||
if (ageSeconds === null) return "unavailable";
|
||||
if (ageSeconds < 60) return `${ageSeconds}s ago`;
|
||||
if (ageSeconds < 3600) return `${Math.floor(ageSeconds / 60)}m ago`;
|
||||
return `${Math.floor(ageSeconds / 3600)}h ago`;
|
||||
}
|
||||
|
||||
export function shouldPollRuntimeStatus(hidden: boolean, requestInFlight: boolean): boolean {
|
||||
return !hidden && !requestInFlight;
|
||||
}
|
||||
@@ -11,6 +11,13 @@ export const generatedTranslations: PlatformTranslations = {
|
||||
"i18n:govoplan-ops.surface.governance": "Governance inventory",
|
||||
"i18n:govoplan-ops.surface.deployment": "Deployment profiles",
|
||||
"i18n:govoplan-ops.surface.sizing": "Sizing assumptions",
|
||||
"i18n:govoplan-ops.infrastructure_capabilities": "Infrastructure capabilities",
|
||||
"i18n:govoplan-ops.pending_post_install_tasks": "{value0} pending post-install task(s)",
|
||||
"i18n:govoplan-ops.no_infrastructure_capabilities": "No deployment capability receipt is available.",
|
||||
"i18n:govoplan-ops.capability": "Capability",
|
||||
"i18n:govoplan-ops.source": "Source",
|
||||
"i18n:govoplan-ops.endpoint": "Endpoint",
|
||||
"i18n:govoplan-ops.consumers": "Consumers",
|
||||
"i18n:govoplan-ops.surface.run_probes": "Run operational probes",
|
||||
"i18n:govoplan-ops.surface.drain_node": "Drain runtime node",
|
||||
"i18n:govoplan-ops.reason.loading": "Operations status is loading.",
|
||||
@@ -80,6 +87,13 @@ export const generatedTranslations: PlatformTranslations = {
|
||||
"i18n:govoplan-ops.surface.governance": "Governance-Inventar",
|
||||
"i18n:govoplan-ops.surface.deployment": "Bereitstellungsprofile",
|
||||
"i18n:govoplan-ops.surface.sizing": "Dimensionierungsannahmen",
|
||||
"i18n:govoplan-ops.infrastructure_capabilities": "Infrastruktur-Fähigkeiten",
|
||||
"i18n:govoplan-ops.pending_post_install_tasks": "{value0} ausstehende Nachinstallationsaufgabe(n)",
|
||||
"i18n:govoplan-ops.no_infrastructure_capabilities": "Es ist kein Bereitstellungsnachweis für Infrastruktur-Fähigkeiten verfügbar.",
|
||||
"i18n:govoplan-ops.capability": "Fähigkeit",
|
||||
"i18n:govoplan-ops.source": "Quelle",
|
||||
"i18n:govoplan-ops.endpoint": "Endpunkt",
|
||||
"i18n:govoplan-ops.consumers": "Verwendende Module",
|
||||
"i18n:govoplan-ops.surface.run_probes": "Betriebsprüfungen ausführen",
|
||||
"i18n:govoplan-ops.surface.drain_node": "Laufzeitknoten leeren",
|
||||
"i18n:govoplan-ops.reason.loading": "Der Betriebsstatus wird geladen.",
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
import assert from "node:assert/strict";
|
||||
import {
|
||||
heartbeatAgeLabel,
|
||||
heartbeatAgeSeconds,
|
||||
knownMetric,
|
||||
knownQueueDepthTotal,
|
||||
runtimeWorkTone,
|
||||
shouldPollRuntimeStatus
|
||||
} from "../src/features/ops/runtimeStatus";
|
||||
|
||||
assert.equal(knownMetric(null), "unavailable");
|
||||
assert.equal(knownMetric(0), 0);
|
||||
assert.equal(knownQueueDepthTotal({ mail: null }), null);
|
||||
assert.equal(knownQueueDepthTotal({ mail: 0, calendar: 2 }), 2);
|
||||
assert.equal(runtimeWorkTone("stale"), "danger");
|
||||
assert.equal(runtimeWorkTone("disabled"), "neutral");
|
||||
assert.equal(heartbeatAgeSeconds(Date.parse("2026-08-19T12:01:00Z"), "2026-08-19T12:00:00Z"), 60);
|
||||
assert.equal(heartbeatAgeLabel(60), "1m ago");
|
||||
assert.equal(heartbeatAgeLabel(null), "unavailable");
|
||||
assert.equal(shouldPollRuntimeStatus(true, false), false);
|
||||
assert.equal(shouldPollRuntimeStatus(false, true), false);
|
||||
assert.equal(shouldPollRuntimeStatus(false, false), true);
|
||||
|
||||
console.log("Ops runtime status and polling model tests passed.");
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"module": "CommonJS",
|
||||
"moduleResolution": "Node",
|
||||
"target": "ES2022",
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true,
|
||||
"typeRoots": ["../../govoplan-core/webui/node_modules/@types"],
|
||||
"types": ["node"],
|
||||
"outDir": ".runtime-status-test-build"
|
||||
},
|
||||
"include": [
|
||||
"src/features/ops/runtimeStatus.ts",
|
||||
"tests/runtime-status.test.ts"
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user