feat: add institutional governance and recovery contracts

This commit is contained in:
2026-08-01 17:46:54 +02:00
parent b65b48832b
commit 7192d32e65
61 changed files with 12539 additions and 168 deletions
+2
View File
@@ -6,6 +6,7 @@ from datetime import datetime
from typing import Literal, Protocol, cast, runtime_checkable
from govoplan_core.core.modules import AccessDecision
from govoplan_core.core.institutional import GovernedContextEnvelope
ACCESS_MODULE_ID = "access"
@@ -377,6 +378,7 @@ class AuditEvent:
occurred_at: datetime | None = None
correlation_id: str | None = None
causation_id: str | None = None
institutional_context: GovernedContextEnvelope | None = None
details: Mapping[str, object] = field(default_factory=dict)
+2
View File
@@ -8,6 +8,7 @@ from typing import Literal, Protocol, runtime_checkable
from govoplan_core.core.access import (
CAPABILITY_AUTH_AUTOMATION_PRINCIPAL_PROVIDER,
)
from govoplan_core.core.institutional import GovernedContextEnvelope
AutomationInvocationKind = Literal[
"manual",
@@ -110,6 +111,7 @@ class ActionExecutionRequest:
input: Mapping[str, object]
idempotency_key: str
invocation: AutomationInvocation
institutional_context: GovernedContextEnvelope | None = None
actor_ref: str | None = None
preview_ref: str | None = None
metadata: Mapping[str, object] = field(default_factory=dict)
@@ -7,7 +7,8 @@ from datetime import UTC, datetime
from pathlib import Path
import json
import os
from typing import Any, Literal, Protocol, runtime_checkable
import re
from typing import Any, Literal, Protocol, cast, runtime_checkable
from govoplan_core.core.module_package_catalog import (
_canonical_catalog_bytes,
@@ -21,6 +22,12 @@ from govoplan_core.core.module_package_catalog import (
_load_private_key,
_parse_trusted_keys,
)
from govoplan_core.core.external_references import (
IntegrationMaturity,
SOURCE_AUTHORITY_MODES,
SourceAuthorityMode,
integration_maturity_rank,
)
from govoplan_core.security.http_fetch import fetch_http_text
@@ -28,6 +35,166 @@ CONFIGURATION_PROVIDER_CAPABILITY = "configuration.provider"
DiagnosticSeverity = Literal["blocker", "warning", "info"]
PlanAction = Literal["create", "update", "bind", "skip", "blocked", "noop"]
ConfigurationPackageClass = Literal[
"reference",
"product",
"sector",
"deployment",
"integration",
]
ConfigurationPackageEvidenceKind = Literal[
"target_test",
"migration",
"upgrade",
"recovery",
"security",
"operations",
"accessibility",
"privacy",
"documentation",
]
CONFIGURATION_PACKAGE_CLASSES: tuple[ConfigurationPackageClass, ...] = (
"reference",
"product",
"sector",
"deployment",
"integration",
)
CONFIGURATION_PACKAGE_EVIDENCE_KINDS: tuple[
ConfigurationPackageEvidenceKind, ...
] = (
"target_test",
"migration",
"upgrade",
"recovery",
"security",
"operations",
"accessibility",
"privacy",
"documentation",
)
_SHA256_RE = re.compile(r"^sha256:[0-9a-f]{64}$")
@dataclass(frozen=True, slots=True)
class ConfigurationPackageParent:
package_id: str
version: str
relation: Literal["derived_from", "specializes", "extends"] = "derived_from"
def __post_init__(self) -> None:
if not self.package_id.strip() or not self.version.strip():
raise ValueError("Configuration package parent id and version are required.")
if self.relation not in {"derived_from", "specializes", "extends"}:
raise ValueError(
f"Unsupported configuration package parent relation: {self.relation!r}."
)
@classmethod
def from_mapping(cls, value: Mapping[str, Any]) -> "ConfigurationPackageParent":
relation = _optional_str(value, "relation") or "derived_from"
if relation not in {"derived_from", "specializes", "extends"}:
raise ValueError(f"Unsupported configuration package parent relation: {relation!r}.")
return cls(
package_id=_required_str(value, "package_id"),
version=_required_str(value, "version"),
relation=relation,
)
def to_dict(self) -> dict[str, str]:
return {
"package_id": self.package_id,
"version": self.version,
"relation": self.relation,
}
@dataclass(frozen=True, slots=True)
class ConfigurationPackageEvidence:
kind: ConfigurationPackageEvidenceKind
reference: str
summary: str
checksum: str | None = None
def __post_init__(self) -> None:
if self.kind not in CONFIGURATION_PACKAGE_EVIDENCE_KINDS:
raise ValueError(
f"Unsupported configuration package evidence kind: {self.kind!r}."
)
if not self.reference.strip() or not self.summary.strip():
raise ValueError(
"Configuration package evidence reference and summary are required."
)
if self.checksum is not None and not _SHA256_RE.fullmatch(self.checksum):
raise ValueError(
"Configuration package evidence checksum must use sha256:<64 lowercase hex>."
)
@classmethod
def from_mapping(cls, value: Mapping[str, Any]) -> "ConfigurationPackageEvidence":
kind = _required_str(value, "kind")
if kind not in CONFIGURATION_PACKAGE_EVIDENCE_KINDS:
raise ValueError(f"Unsupported configuration package evidence kind: {kind!r}.")
return cls(
kind=kind,
reference=_required_str(value, "reference"),
summary=_required_str(value, "summary"),
checksum=_optional_str(value, "checksum"),
)
def to_dict(self) -> dict[str, object]:
return {
"kind": self.kind,
"reference": self.reference,
"summary": self.summary,
"checksum": self.checksum,
}
@dataclass(frozen=True, slots=True)
class ConfigurationProviderExpectation:
provider_id: str
authority_mode: SourceAuthorityMode
minimum_maturity: IntegrationMaturity
binding_ref: str | None = None
health_expectation: str = "healthy"
freshness_expectation: str | None = None
recovery_expectation: str | None = None
def __post_init__(self) -> None:
if not self.provider_id.strip():
raise ValueError("Configuration provider expectation id is required.")
if self.authority_mode not in SOURCE_AUTHORITY_MODES:
raise ValueError(
f"Unsupported provider authority mode: {self.authority_mode!r}."
)
integration_maturity_rank(self.minimum_maturity)
if not self.health_expectation.strip():
raise ValueError("Provider health expectation is required.")
@classmethod
def from_mapping(cls, value: Mapping[str, Any]) -> "ConfigurationProviderExpectation":
return cls(
provider_id=_required_str(value, "provider_id"),
authority_mode=_required_str(value, "authority_mode"),
minimum_maturity=_required_str(value, "minimum_maturity"),
binding_ref=_optional_str(value, "binding_ref"),
health_expectation=_optional_str(value, "health_expectation") or "healthy",
freshness_expectation=_optional_str(value, "freshness_expectation"),
recovery_expectation=_optional_str(value, "recovery_expectation"),
)
def to_dict(self) -> dict[str, object]:
return {
"provider_id": self.provider_id,
"authority_mode": self.authority_mode,
"minimum_maturity": self.minimum_maturity,
"binding_ref": self.binding_ref,
"health_expectation": self.health_expectation,
"freshness_expectation": self.freshness_expectation,
"recovery_expectation": self.recovery_expectation,
}
@dataclass(frozen=True, slots=True)
@@ -75,6 +242,7 @@ class ConfigurationPackageManifest:
package_id: str
name: str
version: str
package_class: ConfigurationPackageClass = "product"
description: str | None = None
publisher: str | None = None
category: str | None = None
@@ -88,6 +256,29 @@ class ConfigurationPackageManifest:
artifact_ref: str | None = None
artifact_sha256: str | None = None
signature: Mapping[str, Any] | None = None
parents: tuple[ConfigurationPackageParent, ...] = ()
evidence: tuple[ConfigurationPackageEvidence, ...] = ()
provider_expectations: tuple[ConfigurationProviderExpectation, ...] = ()
def __post_init__(self) -> None:
if self.package_class not in CONFIGURATION_PACKAGE_CLASSES:
raise ValueError(
f"Unsupported configuration package class: {self.package_class!r}."
)
parent_keys = {(item.package_id, item.version) for item in self.parents}
if len(parent_keys) != len(self.parents):
raise ValueError("Configuration package parents must be unique.")
evidence_keys = {(item.kind, item.reference) for item in self.evidence}
if len(evidence_keys) != len(self.evidence):
raise ValueError("Configuration package evidence must be unique.")
provider_ids = [item.provider_id for item in self.provider_expectations]
if len(provider_ids) != len(set(provider_ids)):
raise ValueError("Configuration package provider expectations must be unique.")
for expectation in self.provider_expectations:
integration_maturity_rank(expectation.minimum_maturity)
issues = configuration_package_claim_issues(self)
if issues:
raise ValueError("Invalid configuration package claim: " + "; ".join(issues))
@classmethod
def from_mapping(cls, value: Mapping[str, Any]) -> "ConfigurationPackageManifest":
@@ -95,6 +286,7 @@ class ConfigurationPackageManifest:
package_id=_required_str(value, "package_id"),
name=_required_str(value, "name"),
version=_required_str(value, "version"),
package_class=_optional_str(value, "package_class") or "product",
description=_optional_str(value, "description"),
publisher=_optional_str(value, "publisher"),
category=_optional_str(value, "category"),
@@ -108,6 +300,21 @@ class ConfigurationPackageManifest:
artifact_ref=_optional_str(value, "artifact_ref"),
artifact_sha256=_optional_str(value, "artifact_sha256"),
signature=value.get("signature") if isinstance(value.get("signature"), Mapping) else None,
parents=tuple(
ConfigurationPackageParent.from_mapping(item)
for item in _object_list(value.get("parents"), field_name="parents")
),
evidence=tuple(
ConfigurationPackageEvidence.from_mapping(item)
for item in _object_list(value.get("evidence"), field_name="evidence")
),
provider_expectations=tuple(
ConfigurationProviderExpectation.from_mapping(item)
for item in _object_list(
value.get("provider_expectations"),
field_name="provider_expectations",
)
),
)
def to_dict(self) -> dict[str, object]:
@@ -115,12 +322,18 @@ class ConfigurationPackageManifest:
"package_id": self.package_id,
"name": self.name,
"version": self.version,
"package_class": self.package_class,
"required_modules": [item.to_dict() for item in self.required_modules],
"required_capabilities": list(self.required_capabilities),
"optional_modules": [item.to_dict() for item in self.optional_modules],
"fragments": [item.to_dict() for item in self.fragments],
"data_requirements": [dict(item) for item in self.data_requirements],
"tags": list(self.tags),
"parents": [item.to_dict() for item in self.parents],
"evidence": [item.to_dict() for item in self.evidence],
"provider_expectations": [
item.to_dict() for item in self.provider_expectations
],
}
for key, value in (
("description", self.description),
@@ -221,6 +434,12 @@ class ConfigurationPreflightContext:
supplied_data: Mapping[str, Any] = field(default_factory=dict)
installed_modules: Mapping[str, str] = field(default_factory=dict)
capabilities: frozenset[str] = frozenset()
external_provider_declarations: Mapping[str, Mapping[str, Any]] = field(
default_factory=dict
)
external_provider_states: Mapping[str, Mapping[str, Any]] = field(
default_factory=dict
)
dry_run: bool = True
@@ -286,6 +505,7 @@ def dry_run_configuration_package(
diagnostics.extend(_module_requirement_diagnostics(manifest, context))
diagnostics.extend(_capability_requirement_diagnostics(manifest, context))
diagnostics.extend(_provider_expectation_diagnostics(manifest, context))
for item in manifest.data_requirements:
requirement = ConfigurationRequiredData.from_mapping(item)
required_data.append(requirement)
@@ -369,6 +589,8 @@ def apply_configuration_package(
supplied_data=supplied_data if supplied_data is not None else context.supplied_data,
installed_modules=context.installed_modules,
capabilities=context.capabilities,
external_provider_declarations=context.external_provider_declarations,
external_provider_states=context.external_provider_states,
dry_run=False,
)
preflight = dry_run_configuration_package(manifest, providers, apply_context)
@@ -665,6 +887,154 @@ def record_configuration_package_catalog_acceptance(validation: dict[str, object
state_path.write_text(json.dumps(state, indent=2, sort_keys=True) + "\n", encoding="utf-8")
def configuration_package_claim_issues(
manifest: ConfigurationPackageManifest,
) -> tuple[str, ...]:
evidence_kinds = {item.kind for item in manifest.evidence}
required_evidence: dict[str, frozenset[str]] = {
"reference": frozenset(
{
"target_test",
"recovery",
"security",
"operations",
"accessibility",
"privacy",
"documentation",
}
),
"product": frozenset(),
"sector": frozenset({"documentation"}),
"deployment": frozenset(
{"target_test", "recovery", "security", "operations"}
),
"integration": frozenset(
{"target_test", "recovery", "operations", "documentation"}
),
}
issues: list[str] = []
missing = sorted(required_evidence[manifest.package_class] - evidence_kinds)
if missing:
issues.append(
f"{manifest.package_class} package is missing evidence: "
+ ", ".join(missing)
)
if manifest.package_class in {"reference", "deployment", "integration"}:
unbound = sorted(
item.kind
for item in manifest.evidence
if item.kind != "documentation" and item.checksum is None
)
if unbound:
issues.append(
f"{manifest.package_class} package has evidence without checksums: "
+ ", ".join(unbound)
)
if manifest.package_class == "sector" and not manifest.parents:
issues.append("sector packages must declare a parent package/version")
if manifest.package_class == "integration" and not manifest.provider_expectations:
issues.append("integration packages must declare external provider expectations")
return tuple(issues)
def validate_configuration_package_derivation(
child: ConfigurationPackageManifest,
parent: ConfigurationPackageManifest,
) -> tuple[ConfigurationDiagnostic, ...]:
diagnostics: list[ConfigurationDiagnostic] = []
if not any(
item.package_id == parent.package_id and item.version == parent.version
for item in child.parents
):
diagnostics.append(
ConfigurationDiagnostic(
severity="blocker",
code="package_parent_provenance_missing",
message=(
f"Package {child.package_id!r} does not declare parent "
f"{parent.package_id}@{parent.version}."
),
object_ref=parent.package_id,
)
)
child_modules = {item.module_id: item for item in child.required_modules}
for requirement in parent.required_modules:
candidate = child_modules.get(requirement.module_id)
if candidate is None or (
requirement.version is not None
and candidate.version != requirement.version
):
diagnostics.append(
ConfigurationDiagnostic(
severity="blocker",
code="package_parent_module_constraint_loosened",
message=(
f"Derived package loosens parent module requirement "
f"{requirement.module_id!r}."
),
module_id=requirement.module_id,
)
)
for capability in set(parent.required_capabilities) - set(
child.required_capabilities
):
diagnostics.append(
ConfigurationDiagnostic(
severity="blocker",
code="package_parent_capability_constraint_loosened",
message=(
f"Derived package removes required capability {capability!r}."
),
object_ref=capability,
)
)
child_providers = {
item.provider_id: item for item in child.provider_expectations
}
for expectation in parent.provider_expectations:
candidate = child_providers.get(expectation.provider_id)
if candidate is None:
diagnostics.append(
ConfigurationDiagnostic(
severity="blocker",
code="package_parent_provider_constraint_removed",
message=(
f"Derived package removes provider expectation "
f"{expectation.provider_id!r}."
),
object_ref=expectation.provider_id,
)
)
continue
if candidate.authority_mode != expectation.authority_mode:
diagnostics.append(
ConfigurationDiagnostic(
severity="blocker",
code="package_parent_authority_mode_changed",
message=(
f"Derived package changes authority mode for provider "
f"{expectation.provider_id!r}."
),
object_ref=expectation.provider_id,
)
)
if integration_maturity_rank(
candidate.minimum_maturity
) < integration_maturity_rank(expectation.minimum_maturity):
diagnostics.append(
ConfigurationDiagnostic(
severity="blocker",
code="package_parent_provider_maturity_loosened",
message=(
f"Derived package lowers provider maturity for "
f"{expectation.provider_id!r}."
),
object_ref=expectation.provider_id,
)
)
return tuple(diagnostics)
def _configuration_package_manifest(package: ConfigurationPackageManifest | Mapping[str, Any]) -> ConfigurationPackageManifest:
if isinstance(package, ConfigurationPackageManifest):
return package
@@ -716,6 +1086,194 @@ def _capability_requirement_diagnostics(manifest: ConfigurationPackageManifest,
return diagnostics
def _provider_expectation_diagnostics(
manifest: ConfigurationPackageManifest,
context: ConfigurationPreflightContext,
) -> list[ConfigurationDiagnostic]:
diagnostics: list[ConfigurationDiagnostic] = []
for expectation in manifest.provider_expectations:
declaration = context.external_provider_declarations.get(
expectation.provider_id
)
if declaration is None:
diagnostics.append(
ConfigurationDiagnostic(
severity="blocker",
code="external_provider_missing",
message=(
f"Required external provider {expectation.provider_id!r} "
"is not installed or declared."
),
object_ref=expectation.provider_id,
resolution=(
"Install and enable a module exposing the declared provider."
),
)
)
continue
supported_modes = {
str(item)
for item in declaration.get("authority_modes", ())
if str(item).strip()
}
if expectation.authority_mode not in supported_modes:
diagnostics.append(
ConfigurationDiagnostic(
severity="blocker",
code="external_provider_authority_mode_unsupported",
message=(
f"Provider {expectation.provider_id!r} does not support "
f"authority mode {expectation.authority_mode!r}."
),
object_ref=expectation.provider_id,
)
)
actual_maturity = str(declaration.get("maturity") or "discover")
try:
maturity_sufficient = integration_maturity_rank(
cast(IntegrationMaturity, actual_maturity)
) >= integration_maturity_rank(expectation.minimum_maturity)
except ValueError:
maturity_sufficient = False
if not maturity_sufficient:
diagnostics.append(
ConfigurationDiagnostic(
severity="blocker",
code="external_provider_maturity_insufficient",
message=(
f"Provider {expectation.provider_id!r} has maturity "
f"{actual_maturity!r}; {expectation.minimum_maturity!r} is required."
),
object_ref=expectation.provider_id,
)
)
provider_state = context.external_provider_states.get(
expectation.provider_id
)
if provider_state is None:
diagnostics.append(
ConfigurationDiagnostic(
severity="warning",
code="external_provider_health_unverified",
message=(
f"Provider {expectation.provider_id!r} has no current "
"health/freshness observation."
),
object_ref=expectation.provider_id,
)
)
continue
state = provider_state
if expectation.binding_ref is not None:
bindings = provider_state.get("bindings")
matching_binding = next(
(
item
for item in bindings
if isinstance(item, Mapping)
and str(item.get("binding_ref") or "")
== expectation.binding_ref
),
None,
) if isinstance(bindings, Sequence) and not isinstance(
bindings, (str, bytes)
) else None
if matching_binding is None and str(
provider_state.get("binding_ref") or ""
) == expectation.binding_ref:
matching_binding = provider_state
if matching_binding is None:
diagnostics.append(
ConfigurationDiagnostic(
severity="blocker",
code="external_provider_binding_mismatch",
message=(
f"Provider {expectation.provider_id!r} is not observed through "
f"required binding {expectation.binding_ref!r}."
),
object_ref=expectation.provider_id,
)
)
continue
state = matching_binding
health = str(state.get("health") or state.get("health_state") or "unknown")
accepted_health = (
{"ok", "healthy"}
if expectation.health_expectation == "healthy"
else {expectation.health_expectation}
)
if health not in accepted_health:
diagnostics.append(
ConfigurationDiagnostic(
severity="blocker",
code="external_provider_unhealthy",
message=(
f"Provider {expectation.provider_id!r} health is {health!r}."
),
object_ref=expectation.provider_id,
resolution="Restore provider health or use a documented degraded path.",
)
)
observed_authority_mode = str(state.get("authority_mode") or "")
if (
observed_authority_mode
and observed_authority_mode != expectation.authority_mode
):
diagnostics.append(
ConfigurationDiagnostic(
severity="blocker",
code="external_provider_binding_authority_mismatch",
message=(
f"Provider {expectation.provider_id!r} is configured as "
f"{observed_authority_mode!r}; {expectation.authority_mode!r} "
"is required."
),
object_ref=expectation.provider_id,
)
)
if expectation.freshness_expectation is not None:
freshness = str(
state.get("freshness")
or state.get("freshness_state")
or "unknown"
)
if freshness != expectation.freshness_expectation:
diagnostics.append(
ConfigurationDiagnostic(
severity="blocker",
code="external_provider_freshness_expectation_failed",
message=(
f"Provider {expectation.provider_id!r} freshness is "
f"{freshness!r}; {expectation.freshness_expectation!r} is required."
),
object_ref=expectation.provider_id,
)
)
if expectation.recovery_expectation is not None:
behavior = declaration.get("behavior")
declared_recovery = (
behavior.get(expectation.recovery_expectation)
if isinstance(behavior, Mapping)
else None
)
observed_recovery = state.get("recovery") or state.get(
"recovery_state"
)
if not declared_recovery and observed_recovery != expectation.recovery_expectation:
diagnostics.append(
ConfigurationDiagnostic(
severity="blocker",
code="external_provider_recovery_expectation_failed",
message=(
f"Provider {expectation.provider_id!r} does not satisfy "
f"recovery expectation {expectation.recovery_expectation!r}."
),
object_ref=expectation.provider_id,
)
)
return diagnostics
def _dedupe_diagnostics(items: Sequence[ConfigurationDiagnostic]) -> list[ConfigurationDiagnostic]:
seen: set[tuple[object, ...]] = set()
result: list[ConfigurationDiagnostic] = []
+179
View File
@@ -5,6 +5,11 @@ from dataclasses import dataclass, field
from datetime import datetime
from typing import Literal, Protocol, runtime_checkable
from govoplan_core.core.external_references import (
SOURCE_AUTHORITY_MODES,
SourceAuthorityMode,
)
CAPABILITY_DATASOURCE_CATALOGUE = "datasources.catalogue"
CAPABILITY_DATASOURCE_LIFECYCLE = "datasources.lifecycle"
@@ -53,6 +58,137 @@ class DatasourceField:
nullable: bool = True
@dataclass(frozen=True, slots=True)
class DatasourceGovernance:
"""Provider-neutral governance facts attached to a datasource revision."""
owner_ref: str | None = None
steward_ref: str | None = None
responsible_organization_ref: str | None = None
responsible_function_ref: str | None = None
authoritative_source_ref: str | None = None
authority_mode: SourceAuthorityMode = "linked_reference"
legal_basis_refs: tuple[str, ...] = ()
purposes: tuple[str, ...] = ()
semantic_definition: str | None = None
schema_owner_ref: str | None = None
official_keys: tuple[str, ...] = ()
classification: str = "internal"
privacy_profile_ref: str | None = None
retention_policy_ref: str | None = None
hold_refs: tuple[str, ...] = ()
publication_state: str = "draft"
transfer_agreement_ref: str | None = None
freshness_policy: Mapping[str, object] = field(default_factory=dict)
quality_policy: Mapping[str, object] = field(default_factory=dict)
known_limits: tuple[str, ...] = ()
correction_procedure_ref: str | None = None
affected_refs: tuple[str, ...] = ()
dependency_refs: tuple[str, ...] = ()
def __post_init__(self) -> None:
if self.authority_mode not in SOURCE_AUTHORITY_MODES:
raise DatasourceValidationError(
f"Unsupported datasource authority mode: {self.authority_mode!r}."
)
if not self.classification.strip():
raise DatasourceValidationError("Datasource classification is required.")
if not self.publication_state.strip():
raise DatasourceValidationError("Datasource publication state is required.")
for field_name in (
"legal_basis_refs",
"purposes",
"official_keys",
"hold_refs",
"known_limits",
"affected_refs",
"dependency_refs",
):
values = getattr(self, field_name)
if any(not value.strip() for value in values):
raise DatasourceValidationError(
f"Datasource governance {field_name} cannot contain empty values."
)
if len(values) != len(set(values)):
raise DatasourceValidationError(
f"Datasource governance {field_name} cannot contain duplicates."
)
@classmethod
def from_mapping(cls, value: Mapping[str, object] | None) -> "DatasourceGovernance":
source = value or {}
return cls(
owner_ref=_optional_governance_text(source.get("owner_ref")),
steward_ref=_optional_governance_text(source.get("steward_ref")),
responsible_organization_ref=_optional_governance_text(
source.get("responsible_organization_ref")
),
responsible_function_ref=_optional_governance_text(
source.get("responsible_function_ref")
),
authoritative_source_ref=_optional_governance_text(
source.get("authoritative_source_ref")
),
authority_mode=str(
source.get("authority_mode") or "linked_reference"
), # type: ignore[arg-type]
legal_basis_refs=_governance_texts(source.get("legal_basis_refs")),
purposes=_governance_texts(source.get("purposes")),
semantic_definition=_optional_governance_text(
source.get("semantic_definition")
),
schema_owner_ref=_optional_governance_text(source.get("schema_owner_ref")),
official_keys=_governance_texts(source.get("official_keys")),
classification=str(source.get("classification") or "internal"),
privacy_profile_ref=_optional_governance_text(
source.get("privacy_profile_ref")
),
retention_policy_ref=_optional_governance_text(
source.get("retention_policy_ref")
),
hold_refs=_governance_texts(source.get("hold_refs")),
publication_state=str(source.get("publication_state") or "draft"),
transfer_agreement_ref=_optional_governance_text(
source.get("transfer_agreement_ref")
),
freshness_policy=_governance_mapping(source.get("freshness_policy")),
quality_policy=_governance_mapping(source.get("quality_policy")),
known_limits=_governance_texts(source.get("known_limits")),
correction_procedure_ref=_optional_governance_text(
source.get("correction_procedure_ref")
),
affected_refs=_governance_texts(source.get("affected_refs")),
dependency_refs=_governance_texts(source.get("dependency_refs")),
)
def to_dict(self) -> dict[str, object]:
return {
"owner_ref": self.owner_ref,
"steward_ref": self.steward_ref,
"responsible_organization_ref": self.responsible_organization_ref,
"responsible_function_ref": self.responsible_function_ref,
"authoritative_source_ref": self.authoritative_source_ref,
"authority_mode": self.authority_mode,
"legal_basis_refs": list(self.legal_basis_refs),
"purposes": list(self.purposes),
"semantic_definition": self.semantic_definition,
"schema_owner_ref": self.schema_owner_ref,
"official_keys": list(self.official_keys),
"classification": self.classification,
"privacy_profile_ref": self.privacy_profile_ref,
"retention_policy_ref": self.retention_policy_ref,
"hold_refs": list(self.hold_refs),
"publication_state": self.publication_state,
"transfer_agreement_ref": self.transfer_agreement_ref,
"freshness_policy": dict(self.freshness_policy),
"quality_policy": dict(self.quality_policy),
"known_limits": list(self.known_limits),
"correction_procedure_ref": self.correction_procedure_ref,
"affected_refs": list(self.affected_refs),
"dependency_refs": list(self.dependency_refs),
}
@dataclass(frozen=True, slots=True)
class DatasourceDescriptor:
ref: str
@@ -75,6 +211,7 @@ class DatasourceDescriptor:
capabilities: tuple[str, ...] = ("read",)
provenance: Mapping[str, object] = field(default_factory=dict)
metadata: Mapping[str, object] = field(default_factory=dict)
governance: DatasourceGovernance = field(default_factory=DatasourceGovernance)
@dataclass(frozen=True, slots=True)
@@ -93,6 +230,7 @@ class DatasourceMaterialization:
created_at: datetime | None = None
provenance: Mapping[str, object] = field(default_factory=dict)
metadata: Mapping[str, object] = field(default_factory=dict)
governance: DatasourceGovernance = field(default_factory=DatasourceGovernance)
@dataclass(frozen=True, slots=True)
@@ -115,6 +253,7 @@ class DatasourceStage:
promoted_materialization_ref: str | None = None
provenance: Mapping[str, object] = field(default_factory=dict)
metadata: Mapping[str, object] = field(default_factory=dict)
governance: DatasourceGovernance = field(default_factory=DatasourceGovernance)
@dataclass(frozen=True, slots=True)
@@ -151,6 +290,7 @@ class DatasourceStageInput:
provider_ref: str | None = None
provenance: Mapping[str, object] = field(default_factory=dict)
metadata: Mapping[str, object] = field(default_factory=dict)
governance: DatasourceGovernance | None = None
@dataclass(frozen=True, slots=True)
@@ -169,6 +309,7 @@ class DatasourcePublicationRequest:
source_timestamp: datetime | None = None
provenance: Mapping[str, object] = field(default_factory=dict)
metadata: Mapping[str, object] = field(default_factory=dict)
governance: DatasourceGovernance | None = None
@dataclass(frozen=True, slots=True)
@@ -226,6 +367,13 @@ class DatasourceCatalogueProvider(Protocol):
*,
query: str = "",
limit: int = 100,
authority_mode: str | None = None,
classification: str | None = None,
publication_state: str | None = None,
owner_ref: str | None = None,
responsible_organization_ref: str | None = None,
affected_ref: str | None = None,
dependency_ref: str | None = None,
) -> Sequence[DatasourceDescriptor]:
...
@@ -298,6 +446,17 @@ class DatasourceLifecycleProvider(Protocol):
source_name: str,
mode: DatasourceMode,
description: str | None = None,
governance: DatasourceGovernance | None = None,
) -> DatasourceDescriptor:
...
def update_datasource_governance(
self,
session: object,
principal: object,
*,
datasource_ref: str,
governance: DatasourceGovernance,
) -> DatasourceDescriptor:
...
@@ -406,6 +565,25 @@ def _capability(registry: object | None, name: str) -> object | None:
return registry.capability(name)
def _optional_governance_text(value: object) -> str | None:
if value is None:
return None
cleaned = str(value).strip()
return cleaned or None
def _governance_texts(value: object) -> tuple[str, ...]:
if not isinstance(value, Sequence) or isinstance(value, (str, bytes)):
return ()
return tuple(str(item).strip() for item in value)
def _governance_mapping(value: object) -> Mapping[str, object]:
if not isinstance(value, Mapping):
return {}
return {str(key): item for key, item in value.items()}
__all__ = [
"CAPABILITY_DATASOURCE_CATALOGUE",
"CAPABILITY_DATASOURCE_LIFECYCLE",
@@ -416,6 +594,7 @@ __all__ = [
"DatasourceDescriptor",
"DatasourceError",
"DatasourceField",
"DatasourceGovernance",
"DatasourceKind",
"DatasourceLifecycleProvider",
"DatasourceMaterialization",
+9
View File
@@ -13,6 +13,8 @@ import uuid
from sqlalchemy import event as sqlalchemy_event
from sqlalchemy.orm import Session
from govoplan_core.core.institutional import GovernedContextEnvelope
_TRACE_ID_RE = re.compile(r"^[A-Za-z0-9_.:-]{1,128}$")
_CONSUMER_ID_RE = re.compile(r"^[a-z][a-z0-9_.:-]{0,127}$")
@@ -87,6 +89,7 @@ class PlatformEvent:
subject: EventObjectRef | None = None
resource: EventObjectRef | None = None
classification: EventClassification = "internal"
institutional_context: GovernedContextEnvelope | None = None
def to_dict(self) -> dict[str, Any]:
return {
@@ -102,6 +105,11 @@ class PlatformEvent:
"subject": self.subject.to_dict() if self.subject else None,
"resource": self.resource.to_dict() if self.resource else None,
"classification": self.classification,
"institutional_context": (
self.institutional_context.to_dict()
if self.institutional_context is not None
else None
),
}
@@ -254,6 +262,7 @@ def ensure_event_trace(event: PlatformEvent) -> PlatformEvent:
subject=event.subject,
resource=event.resource,
classification=event.classification,
institutional_context=event.institutional_context,
)
@@ -17,6 +17,14 @@ IntegrationMaturity = Literal[
"migrate",
"replace",
]
SourceAuthorityMode = Literal[
"native_authoritative",
"external_authoritative",
"external_mirror",
"governed_sync",
"governance_overlay",
"linked_reference",
]
INTEGRATION_MATURITY_ORDER: tuple[IntegrationMaturity, ...] = (
"discover",
@@ -28,6 +36,14 @@ INTEGRATION_MATURITY_ORDER: tuple[IntegrationMaturity, ...] = (
"migrate",
"replace",
)
SOURCE_AUTHORITY_MODES: tuple[SourceAuthorityMode, ...] = (
"native_authoritative",
"external_authoritative",
"external_mirror",
"governed_sync",
"governance_overlay",
"linked_reference",
)
class ExternalReferenceValidationError(ValueError):
@@ -42,6 +58,7 @@ class ExternalObjectReference:
object_type: str
object_id: str
maturity: IntegrationMaturity = "link"
authority_mode: SourceAuthorityMode = "linked_reference"
connector_id: str | None = None
canonical_url: str | None = None
version: str | None = None
@@ -65,6 +82,24 @@ class ExternalObjectReference:
raise ExternalReferenceValidationError(
f"Unsupported integration maturity: {self.maturity!r}."
)
if self.authority_mode not in SOURCE_AUTHORITY_MODES:
raise ExternalReferenceValidationError(
f"Unsupported source-authority mode: {self.authority_mode!r}."
)
if (
self.authority_mode == "external_mirror"
and not self.supports("read")
):
raise ExternalReferenceValidationError(
"External-mirror references require read maturity or higher."
)
if (
self.authority_mode == "governed_sync"
and not self.supports("synchronize")
):
raise ExternalReferenceValidationError(
"Governed-sync references require synchronize maturity or higher."
)
if self.connector_id is not None:
connector_id = self.connector_id.strip()
if not connector_id:
@@ -94,6 +129,7 @@ class ExternalObjectReference:
"object_type": self.object_type,
"object_id": self.object_id,
"maturity": self.maturity,
"authority_mode": self.authority_mode,
"connector_id": self.connector_id,
"canonical_url": self.canonical_url,
"version": self.version,
@@ -133,5 +169,7 @@ __all__ = [
"ExternalReferenceValidationError",
"INTEGRATION_MATURITY_ORDER",
"IntegrationMaturity",
"SOURCE_AUTHORITY_MODES",
"SourceAuthorityMode",
"integration_maturity_rank",
]
+424 -55
View File
@@ -90,7 +90,9 @@ class _ConfigIssueCollector:
def add(self, level: ConfigIssueLevel, key: str, message: str, action: str) -> None:
if self.strict and level == "warning":
level = "error"
self.issues.append(ConfigIssue(level=level, key=key, message=message, action=action))
self.issues.append(
ConfigIssue(level=level, key=key, message=message, action=action)
)
_LOCAL_PROFILES = {"dev", "local", "local-dev", "test"}
@@ -120,9 +122,15 @@ def generate_master_key() -> str:
return Fernet.generate_key().decode("ascii")
def env_template(*, profile: str = "self-hosted", generate_secrets: bool = False) -> str:
def env_template(
*, profile: str = "self-hosted", generate_secrets: bool = False
) -> str:
clean_profile = normalize_install_profile(profile)
master_key = generate_master_key() if generate_secrets else "<generate-with-govoplan-config-env-template-generate-secrets>"
master_key = (
generate_master_key()
if generate_secrets
else "<generate-with-govoplan-config-env-template-generate-secrets>"
)
if clean_profile == "production-like":
return _production_like_env_template(master_key)
return _self_hosted_env_template(master_key)
@@ -144,14 +152,19 @@ def validate_runtime_configuration(
_validate_async_and_auth_settings(env, runtime, collector)
_validate_cors_settings(env, runtime, collector)
_validate_file_storage_settings(env, runtime, collector)
_validate_shared_state_settings(env, collector)
_validate_outbound_connector_policy(env, runtime, collector)
_validate_module_catalog_trust(env, runtime, collector)
return ConfigValidationResult(profile=runtime.name, issues=tuple(collector.issues))
def _runtime_profile(env: Mapping[str, str], *, profile: str | None) -> _RuntimeProfile:
clean_profile = normalize_install_profile(profile or env.get("GOVOPLAN_INSTALL_PROFILE") or env.get("APP_ENV"))
production = clean_profile in _PRODUCTION_PROFILES or env.get("APP_ENV", "").strip().lower() in {"prod", "production"}
clean_profile = normalize_install_profile(
profile or env.get("GOVOPLAN_INSTALL_PROFILE") or env.get("APP_ENV")
)
production = clean_profile in _PRODUCTION_PROFILES or env.get(
"APP_ENV", ""
).strip().lower() in {"prod", "production"}
production_like = production or clean_profile in _PRODUCTION_LIKE_PROFILES
return _RuntimeProfile(
name=clean_profile,
@@ -161,86 +174,222 @@ def _runtime_profile(env: Mapping[str, str], *, profile: str | None) -> _Runtime
)
def _validate_app_env(env: Mapping[str, str], runtime: _RuntimeProfile, collector: _ConfigIssueCollector) -> None:
def _validate_app_env(
env: Mapping[str, str], runtime: _RuntimeProfile, collector: _ConfigIssueCollector
) -> None:
app_env = _clean(env.get("APP_ENV"))
if not app_env and runtime.production_like:
collector.add("error", "APP_ENV", "APP_ENV is missing for a production-like install.", "Set APP_ENV=staging, APP_ENV=production, or another explicit deployment profile.")
collector.add(
"error",
"APP_ENV",
"APP_ENV is missing for a production-like install.",
"Set APP_ENV=staging, APP_ENV=production, or another explicit deployment profile.",
)
elif app_env.lower() in {"dev", "test", "local"} and runtime.production_like:
collector.add("error", "APP_ENV", f"APP_ENV={app_env!r} is not valid for profile {runtime.name!r}.", "Use APP_ENV=staging for production-like testing or APP_ENV=production for a real deployment.")
collector.add(
"error",
"APP_ENV",
f"APP_ENV={app_env!r} is not valid for profile {runtime.name!r}.",
"Use APP_ENV=staging for production-like testing or APP_ENV=production for a real deployment.",
)
def _validate_database_settings(env: Mapping[str, str], runtime: _RuntimeProfile, collector: _ConfigIssueCollector) -> None:
def _validate_database_settings(
env: Mapping[str, str], runtime: _RuntimeProfile, collector: _ConfigIssueCollector
) -> None:
database_url = _clean(env.get("DATABASE_URL"))
if not database_url:
collector.add("error", "DATABASE_URL", "DATABASE_URL is missing.", "Set DATABASE_URL to the PostgreSQL SQLAlchemy URL used by the API and modules.")
collector.add(
"error",
"DATABASE_URL",
"DATABASE_URL is missing.",
"Set DATABASE_URL to the PostgreSQL SQLAlchemy URL used by the API and modules.",
)
return
backend = _database_backend(database_url)
if backend is None:
collector.add("error", "DATABASE_URL", "DATABASE_URL is not a valid SQLAlchemy URL.", "Use a value like postgresql+psycopg://user:password@host:5432/database.")
collector.add(
"error",
"DATABASE_URL",
"DATABASE_URL is not a valid SQLAlchemy URL.",
"Use a value like postgresql+psycopg://user:password@host:5432/database.",
)
return
if backend == "sqlite" and runtime.production_like:
collector.add("error", "DATABASE_URL", "SQLite is only supported for disposable local development.", "Use PostgreSQL for production-like and self-hosted installs.")
collector.add(
"error",
"DATABASE_URL",
"SQLite is only supported for disposable local development.",
"Use PostgreSQL for production-like and self-hosted installs.",
)
elif backend != "postgresql" and runtime.production:
collector.add("warning", "DATABASE_URL", f"Database backend {backend!r} is not the preferred production target.", "Use PostgreSQL unless this deployment has an explicit support decision.")
collector.add(
"warning",
"DATABASE_URL",
f"Database backend {backend!r} is not the preferred production target.",
"Use PostgreSQL unless this deployment has an explicit support decision.",
)
if backend == "postgresql" and not _clean(env.get("GOVOPLAN_DATABASE_URL_PGTOOLS")):
collector.add("warning", "GOVOPLAN_DATABASE_URL_PGTOOLS", "PostgreSQL backup/restore tools URL is missing.", "Set GOVOPLAN_DATABASE_URL_PGTOOLS to the same database without the SQLAlchemy driver marker, for example postgresql://user:password@host:5432/database.")
collector.add(
"warning",
"GOVOPLAN_DATABASE_URL_PGTOOLS",
"PostgreSQL backup/restore tools URL is missing.",
"Set GOVOPLAN_DATABASE_URL_PGTOOLS to the same database without the SQLAlchemy driver marker, for example postgresql://user:password@host:5432/database.",
)
def _validate_master_key(env: Mapping[str, str], runtime: _RuntimeProfile, collector: _ConfigIssueCollector) -> None:
def _validate_master_key(
env: Mapping[str, str], runtime: _RuntimeProfile, collector: _ConfigIssueCollector
) -> None:
master_key = _clean(env.get("MASTER_KEY_B64"))
if not master_key and not runtime.local:
collector.add("error", "MASTER_KEY_B64", "MASTER_KEY_B64 is required outside local dev/test.", "Generate a Fernet key with `govoplan-config env-template --generate-secrets` or `python -c 'from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())'` and store it in deployment secrets.")
collector.add(
"error",
"MASTER_KEY_B64",
"MASTER_KEY_B64 is required outside local dev/test.",
"Generate a Fernet key with `govoplan-config env-template --generate-secrets` or `python -c 'from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())'` and store it in deployment secrets.",
)
return
if not master_key:
return
error = _master_key_error(master_key)
if error:
collector.add("error", "MASTER_KEY_B64", error, "Replace MASTER_KEY_B64 with a Fernet key or base64-encoded 32-byte key.")
collector.add(
"error",
"MASTER_KEY_B64",
error,
"Replace MASTER_KEY_B64 with a Fernet key or base64-encoded 32-byte key.",
)
elif "change-me" in master_key.lower() or "generate" in master_key.lower():
collector.add("error", "MASTER_KEY_B64", "MASTER_KEY_B64 still looks like a placeholder.", "Generate a real deployment key and store it outside git.")
collector.add(
"error",
"MASTER_KEY_B64",
"MASTER_KEY_B64 still looks like a placeholder.",
"Generate a real deployment key and store it outside git.",
)
def _validate_enabled_modules(env: Mapping[str, str], runtime: _RuntimeProfile, collector: _ConfigIssueCollector) -> None:
def _validate_enabled_modules(
env: Mapping[str, str], runtime: _RuntimeProfile, collector: _ConfigIssueCollector
) -> None:
enabled_modules = _csv(env.get("ENABLED_MODULES"))
if not enabled_modules and runtime.production_like:
collector.add("error", "ENABLED_MODULES", "ENABLED_MODULES is missing.", "Set ENABLED_MODULES explicitly so startup module composition is intentional.")
collector.add(
"error",
"ENABLED_MODULES",
"ENABLED_MODULES is missing.",
"Set ENABLED_MODULES explicitly so startup module composition is intentional.",
)
elif "access" not in enabled_modules and runtime.production_like:
collector.add("error", "ENABLED_MODULES", "The access module is not enabled.", "Include `access` unless this deployment has a replacement auth/principal provider.")
collector.add(
"error",
"ENABLED_MODULES",
"The access module is not enabled.",
"Include `access` unless this deployment has a replacement auth/principal provider.",
)
elif enabled_modules and "admin" not in enabled_modules and runtime.production_like:
collector.add("warning", "ENABLED_MODULES", "The admin module is not enabled.", "Keep `admin` enabled for operator UI unless this is a deliberately headless install.")
collector.add(
"warning",
"ENABLED_MODULES",
"The admin module is not enabled.",
"Keep `admin` enabled for operator UI unless this is a deliberately headless install.",
)
def _validate_async_and_auth_settings(env: Mapping[str, str], runtime: _RuntimeProfile, collector: _ConfigIssueCollector) -> None:
def _validate_async_and_auth_settings(
env: Mapping[str, str], runtime: _RuntimeProfile, collector: _ConfigIssueCollector
) -> None:
if _truthy(env.get("CELERY_ENABLED")) and not _clean(env.get("REDIS_URL")):
collector.add("error", "REDIS_URL", "CELERY_ENABLED=true but REDIS_URL is missing.", "Set REDIS_URL to the Redis broker/result backend used by workers.")
collector.add(
"error",
"REDIS_URL",
"CELERY_ENABLED=true but REDIS_URL is missing.",
"Set REDIS_URL to the Redis broker/result backend used by workers.",
)
if runtime.production and _truthy(env.get("DEV_BOOTSTRAP_ENABLED")):
collector.add("error", "DEV_BOOTSTRAP_ENABLED", "Development bootstrap is enabled in production.", "Set DEV_BOOTSTRAP_ENABLED=false and create first administrators through the controlled bootstrap path.")
collector.add(
"error",
"DEV_BOOTSTRAP_ENABLED",
"Development bootstrap is enabled in production.",
"Set DEV_BOOTSTRAP_ENABLED=false and create first administrators through the controlled bootstrap path.",
)
if runtime.production and not _truthy(env.get("AUTH_COOKIE_SECURE")):
collector.add("error", "AUTH_COOKIE_SECURE", "Secure auth cookies are disabled for production.", "Set AUTH_COOKIE_SECURE=true behind HTTPS.")
collector.add(
"error",
"AUTH_COOKIE_SECURE",
"Secure auth cookies are disabled for production.",
"Set AUTH_COOKIE_SECURE=true behind HTTPS.",
)
def _validate_cors_settings(env: Mapping[str, str], runtime: _RuntimeProfile, collector: _ConfigIssueCollector) -> None:
def _validate_cors_settings(
env: Mapping[str, str], runtime: _RuntimeProfile, collector: _ConfigIssueCollector
) -> None:
cors_origins = _csv(env.get("CORS_ORIGINS"))
if runtime.production_like and not cors_origins:
collector.add("error", "CORS_ORIGINS", "CORS_ORIGINS is missing.", "Set CORS_ORIGINS to the exact WebUI origin or origins.")
collector.add(
"error",
"CORS_ORIGINS",
"CORS_ORIGINS is missing.",
"Set CORS_ORIGINS to the exact WebUI origin or origins.",
)
elif "*" in cors_origins and runtime.production_like:
collector.add("error", "CORS_ORIGINS", "Wildcard CORS is not allowed for production-like installs.", "Replace `*` with exact HTTPS/WebUI origins.")
collector.add(
"error",
"CORS_ORIGINS",
"Wildcard CORS is not allowed for production-like installs.",
"Replace `*` with exact HTTPS/WebUI origins.",
)
elif runtime.production and set(cors_origins) <= _DEFAULT_LOCAL_CORS:
collector.add("warning", "CORS_ORIGINS", "CORS_ORIGINS still contains only local development origins.", "Set CORS_ORIGINS to the deployed WebUI origin.")
collector.add(
"warning",
"CORS_ORIGINS",
"CORS_ORIGINS still contains only local development origins.",
"Set CORS_ORIGINS to the deployed WebUI origin.",
)
trusted_hosts = _csv(env.get("GOVOPLAN_TRUSTED_HOSTS"))
if runtime.production_like and not trusted_hosts:
collector.add("error", "GOVOPLAN_TRUSTED_HOSTS", "Trusted HTTP hosts are not configured.", "Set GOVOPLAN_TRUSTED_HOSTS to the exact API host names accepted by this deployment.")
collector.add(
"error",
"GOVOPLAN_TRUSTED_HOSTS",
"Trusted HTTP hosts are not configured.",
"Set GOVOPLAN_TRUSTED_HOSTS to the exact API host names accepted by this deployment.",
)
elif "*" in trusted_hosts and runtime.production_like:
collector.add("error", "GOVOPLAN_TRUSTED_HOSTS", "Wildcard trusted hosts are not allowed for production-like installs.", "Replace `*` with exact host names or narrowly scoped `*.example.org` entries.")
collector.add(
"error",
"GOVOPLAN_TRUSTED_HOSTS",
"Wildcard trusted hosts are not allowed for production-like installs.",
"Replace `*` with exact host names or narrowly scoped `*.example.org` entries.",
)
forwarded_allow_ips = _csv(env.get("FORWARDED_ALLOW_IPS"))
if runtime.production_like and "*" in forwarded_allow_ips:
collector.add("error", "FORWARDED_ALLOW_IPS", "Proxy headers must not be trusted from every address.", "Set FORWARDED_ALLOW_IPS to the reverse proxy address or network passed to Uvicorn.")
collector.add(
"error",
"FORWARDED_ALLOW_IPS",
"Proxy headers must not be trusted from every address.",
"Set FORWARDED_ALLOW_IPS to the reverse proxy address or network passed to Uvicorn.",
)
def _validate_file_storage_settings(env: Mapping[str, str], runtime: _RuntimeProfile, collector: _ConfigIssueCollector) -> None:
def _validate_file_storage_settings(
env: Mapping[str, str], runtime: _RuntimeProfile, collector: _ConfigIssueCollector
) -> None:
storage_backend = _clean(env.get("FILE_STORAGE_BACKEND")) or "local"
deployment_managed_raw = _clean(env.get("FILE_STORAGE_S3_DEPLOYMENT_MANAGED")).lower()
if deployment_managed_raw and deployment_managed_raw not in {"true", "false", "1", "0", "yes", "no", "on", "off"}:
deployment_managed_raw = _clean(
env.get("FILE_STORAGE_S3_DEPLOYMENT_MANAGED")
).lower()
endpoint_trusted_raw = _clean(env.get("FILE_STORAGE_S3_ENDPOINT_TRUSTED")).lower()
if deployment_managed_raw and deployment_managed_raw not in {
"true",
"false",
"1",
"0",
"yes",
"no",
"on",
"off",
}:
collector.add(
"error",
"FILE_STORAGE_S3_DEPLOYMENT_MANAGED",
@@ -248,13 +397,30 @@ def _validate_file_storage_settings(env: Mapping[str, str], runtime: _RuntimePro
"Set FILE_STORAGE_S3_DEPLOYMENT_MANAGED=false, or let the supported installer manage Garage.",
)
deployment_managed = _truthy(deployment_managed_raw)
if endpoint_trusted_raw and endpoint_trusted_raw not in {
"true",
"false",
"1",
"0",
"yes",
"no",
"on",
"off",
}:
collector.add(
"error",
"FILE_STORAGE_S3_ENDPOINT_TRUSTED",
"External S3 endpoint trust must be an explicit boolean.",
"Set it only for a deployment-controlled HTTPS storage origin.",
)
endpoint_trusted = _truthy(endpoint_trusted_raw)
if storage_backend == "local":
if deployment_managed:
if deployment_managed or endpoint_trusted:
collector.add(
"error",
"FILE_STORAGE_S3_DEPLOYMENT_MANAGED",
"Managed S3 trust cannot be enabled for local file storage.",
"Set FILE_STORAGE_S3_DEPLOYMENT_MANAGED=false.",
"FILE_STORAGE_S3_ENDPOINT_TRUSTED",
"S3 endpoint trust cannot be enabled for local file storage.",
"Disable both S3 trust settings while FILE_STORAGE_BACKEND=local.",
)
_validate_local_file_storage(env, runtime, collector)
elif storage_backend == "s3":
@@ -262,16 +428,34 @@ def _validate_file_storage_settings(env: Mapping[str, str], runtime: _RuntimePro
env,
collector,
deployment_managed=deployment_managed,
endpoint_trusted=endpoint_trusted,
)
else:
collector.add("error", "FILE_STORAGE_BACKEND", f"Unsupported FILE_STORAGE_BACKEND={storage_backend!r}.", "Use `local` or `s3`.")
collector.add(
"error",
"FILE_STORAGE_BACKEND",
f"Unsupported FILE_STORAGE_BACKEND={storage_backend!r}.",
"Use `local` or `s3`.",
)
def _validate_local_file_storage(env: Mapping[str, str], runtime: _RuntimeProfile, collector: _ConfigIssueCollector) -> None:
def _validate_local_file_storage(
env: Mapping[str, str], runtime: _RuntimeProfile, collector: _ConfigIssueCollector
) -> None:
if not _clean(env.get("FILE_STORAGE_LOCAL_ROOT")) and runtime.production_like:
collector.add("error", "FILE_STORAGE_LOCAL_ROOT", "Local file storage root is missing.", "Set FILE_STORAGE_LOCAL_ROOT to a durable, backed-up path.")
collector.add(
"error",
"FILE_STORAGE_LOCAL_ROOT",
"Local file storage root is missing.",
"Set FILE_STORAGE_LOCAL_ROOT to a durable, backed-up path.",
)
elif runtime.production:
collector.add("warning", "FILE_STORAGE_BACKEND", "Production is configured for local file storage.", "Confirm the path is durable and backed up, or use object storage once the deployment needs independent file scaling.")
collector.add(
"warning",
"FILE_STORAGE_BACKEND",
"Production is configured for local file storage.",
"Confirm the path is durable and backed up, or use object storage once the deployment needs independent file scaling.",
)
def _validate_s3_file_storage(
@@ -279,10 +463,22 @@ def _validate_s3_file_storage(
collector: _ConfigIssueCollector,
*,
deployment_managed: bool,
endpoint_trusted: bool,
) -> None:
for key in ("FILE_STORAGE_S3_ENDPOINT_URL", "FILE_STORAGE_S3_REGION", "FILE_STORAGE_S3_ACCESS_KEY_ID", "FILE_STORAGE_S3_SECRET_ACCESS_KEY", "FILE_STORAGE_S3_BUCKET"):
for key in (
"FILE_STORAGE_S3_ENDPOINT_URL",
"FILE_STORAGE_S3_REGION",
"FILE_STORAGE_S3_ACCESS_KEY_ID",
"FILE_STORAGE_S3_SECRET_ACCESS_KEY",
"FILE_STORAGE_S3_BUCKET",
):
if not _clean(env.get(key)):
collector.add("error", key, f"{key} is required when FILE_STORAGE_BACKEND=s3.", "Configure all FILE_STORAGE_S3_* settings through deployment secrets.")
collector.add(
"error",
key,
f"{key} is required when FILE_STORAGE_BACKEND=s3.",
"Configure all FILE_STORAGE_S3_* settings through deployment secrets.",
)
if (
deployment_managed
and _clean(env.get("FILE_STORAGE_S3_ENDPOINT_URL")) != "http://garage:3900"
@@ -293,6 +489,126 @@ def _validate_s3_file_storage(
"Installer-managed S3 trust is restricted to http://garage:3900.",
"Use the exact managed Garage endpoint or disable FILE_STORAGE_S3_DEPLOYMENT_MANAGED.",
)
if deployment_managed and endpoint_trusted:
collector.add(
"error",
"FILE_STORAGE_S3_ENDPOINT_TRUSTED",
"Managed Garage trust and external endpoint trust are mutually exclusive.",
"Use installer-managed Garage trust or one explicit external endpoint.",
)
if not deployment_managed and not endpoint_trusted:
collector.add(
"error",
"FILE_STORAGE_S3_ENDPOINT_TRUSTED",
"External S3 storage requires an explicit deployment trust decision.",
"Set FILE_STORAGE_S3_ENDPOINT_TRUSTED=true only for a deployment-controlled HTTPS origin.",
)
endpoint = _clean(env.get("FILE_STORAGE_S3_ENDPOINT_URL"))
if endpoint_trusted and not endpoint.lower().startswith("https://"):
collector.add(
"error",
"FILE_STORAGE_S3_ENDPOINT_URL",
"Deployment-trusted external S3 storage must use HTTPS.",
"Use an HTTPS storage origin with certificate verification.",
)
def _validate_shared_state_settings(
env: Mapping[str, str],
collector: _ConfigIssueCollector,
) -> None:
state_profile = (_clean(env.get("GOVOPLAN_STATE_PROFILE")) or "local").lower()
if state_profile not in {"local", "host-shared", "shared"}:
collector.add(
"error",
"GOVOPLAN_STATE_PROFILE",
f"Unsupported state profile {state_profile!r}.",
"Use `local` for one process per role, `host-shared` for one Compose host, or `shared` for a multi-host stateless tier.",
)
return
try:
api_replicas = int(_clean(env.get("GOVOPLAN_EXPECTED_API_REPLICAS")) or "1")
worker_replicas = int(
_clean(env.get("GOVOPLAN_EXPECTED_WORKER_REPLICAS")) or "0"
)
except ValueError:
collector.add(
"error",
"GOVOPLAN_EXPECTED_API_REPLICAS",
"Expected replica counts must be integers.",
"Set GOVOPLAN_EXPECTED_API_REPLICAS and GOVOPLAN_EXPECTED_WORKER_REPLICAS to non-negative integers.",
)
return
if api_replicas < 1 or worker_replicas < 0:
collector.add(
"error",
"GOVOPLAN_EXPECTED_API_REPLICAS",
"Expected replica counts are outside their supported range.",
"Configure at least one API replica and zero or more worker replicas.",
)
if state_profile == "local":
if api_replicas > 1 or worker_replicas > 1:
collector.add(
"error",
"GOVOPLAN_STATE_PROFILE",
"A local-state profile cannot safely run replicated API or worker nodes.",
"Use `host-shared` with one shared host volume, or `shared` with PostgreSQL, Redis, and S3-compatible object storage.",
)
return
installation_id = _clean(env.get("GOVOPLAN_INSTALLATION_ID"))
if not installation_id or (
state_profile == "shared" and installation_id == "govoplan-local"
):
collector.add(
"error",
"GOVOPLAN_INSTALLATION_ID",
"Shared-state deployments require a stable installation identifier.",
"Set one immutable deployment-wide GOVOPLAN_INSTALLATION_ID on every node.",
)
if _database_backend(_clean(env.get("DATABASE_URL"))) != "postgresql":
collector.add(
"error",
"DATABASE_URL",
"Shared-state deployments require PostgreSQL.",
"Point every API, scheduler, and worker node at the same logical PostgreSQL service.",
)
if not _clean(env.get("REDIS_URL")):
collector.add(
"error",
"REDIS_URL",
"Shared-state deployments require a common Redis service.",
"Configure the same Redis endpoint for all API and worker nodes.",
)
if (
state_profile == "shared"
and (_clean(env.get("FILE_STORAGE_BACKEND")) or "local").lower() != "s3"
):
collector.add(
"error",
"FILE_STORAGE_BACKEND",
"Shared-state deployments cannot use node-local object storage.",
"Set FILE_STORAGE_BACKEND=s3 and configure one shared S3-compatible bucket.",
)
try:
heartbeat = int(_clean(env.get("GOVOPLAN_RUNTIME_HEARTBEAT_SECONDS")) or "15")
stale_after = int(
_clean(env.get("GOVOPLAN_RUNTIME_STALE_AFTER_SECONDS")) or "60"
)
except ValueError:
collector.add(
"error",
"GOVOPLAN_RUNTIME_HEARTBEAT_SECONDS",
"Runtime heartbeat and stale intervals must be integers.",
"Use a heartbeat interval shorter than one third of the stale interval.",
)
else:
if heartbeat < 2 or stale_after < max(10, heartbeat * 3):
collector.add(
"error",
"GOVOPLAN_RUNTIME_STALE_AFTER_SECONDS",
"Runtime stale detection leaves insufficient room for missed heartbeats.",
"Set stale-after to at least three heartbeat intervals and at least ten seconds.",
)
def _validate_outbound_connector_policy(
@@ -300,8 +616,19 @@ def _validate_outbound_connector_policy(
runtime: _RuntimeProfile,
collector: _ConfigIssueCollector,
) -> None:
private_networks = _clean(env.get("GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS")).lower()
if runtime.production_like and private_networks not in {"true", "false", "1", "0", "yes", "no", "on", "off"}:
private_networks = _clean(
env.get("GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS")
).lower()
if runtime.production_like and private_networks not in {
"true",
"false",
"1",
"0",
"yes",
"no",
"on",
"off",
}:
collector.add(
"error",
"GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS",
@@ -321,13 +648,21 @@ def _validate_outbound_connector_policy(
except ValueError:
parsed = 0
if parsed <= 0:
collector.add("error", key, f"{key} must be a positive byte count.", "Use a positive integer byte limit.")
collector.add(
"error",
key,
f"{key} must be a positive byte count.",
"Use a positive integer byte limit.",
)
secret_env_names = [
item.strip()
for item in env.get("GOVOPLAN_CONNECTOR_SECRET_ENV_ALLOWLIST", "").split(",")
if item.strip()
]
if any(re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", item) is None for item in secret_env_names):
if any(
re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", item) is None
for item in secret_env_names
):
collector.add(
"error",
"GOVOPLAN_CONNECTOR_SECRET_ENV_ALLOWLIST",
@@ -348,14 +683,28 @@ def _validate_outbound_connector_policy(
)
def _validate_module_catalog_trust(env: Mapping[str, str], runtime: _RuntimeProfile, collector: _ConfigIssueCollector) -> None:
catalog_source = _clean(env.get("GOVOPLAN_MODULE_PACKAGE_CATALOG_URL")) or _clean(env.get("GOVOPLAN_MODULE_PACKAGE_CATALOG"))
def _validate_module_catalog_trust(
env: Mapping[str, str], runtime: _RuntimeProfile, collector: _ConfigIssueCollector
) -> None:
catalog_source = _clean(env.get("GOVOPLAN_MODULE_PACKAGE_CATALOG_URL")) or _clean(
env.get("GOVOPLAN_MODULE_PACKAGE_CATALOG")
)
if not runtime.production or not catalog_source:
return
if not _clean(env.get("GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE")):
collector.add("error", "GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE", "A module catalog source is configured without a trusted keyring file.", "Pin the published GovOPlaN catalog keyring locally and set GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE.")
collector.add(
"error",
"GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE",
"A module catalog source is configured without a trusted keyring file.",
"Pin the published GovOPlaN catalog keyring locally and set GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE.",
)
if not _clean(env.get("GOVOPLAN_MODULE_PACKAGE_CATALOG_APPROVED_CHANNEL")):
collector.add("error", "GOVOPLAN_MODULE_PACKAGE_CATALOG_APPROVED_CHANNEL", "A module catalog source is configured without an approved release channel.", "Set GOVOPLAN_MODULE_PACKAGE_CATALOG_APPROVED_CHANNEL=stable or another approved deployment channel.")
collector.add(
"error",
"GOVOPLAN_MODULE_PACKAGE_CATALOG_APPROVED_CHANNEL",
"A module catalog source is configured without an approved release channel.",
"Set GOVOPLAN_MODULE_PACKAGE_CATALOG_APPROVED_CHANNEL=stable or another approved deployment channel.",
)
def _self_hosted_env_template(master_key: str) -> str:
@@ -364,6 +713,13 @@ def _self_hosted_env_template(master_key: str) -> str:
APP_ENV=production
GOVOPLAN_INSTALL_PROFILE=self-hosted
GOVOPLAN_INSTALLATION_ID=govoplan-production
GOVOPLAN_STATE_PROFILE=local
GOVOPLAN_RUNTIME_ROLE=api
GOVOPLAN_RUNTIME_HEARTBEAT_SECONDS=15
GOVOPLAN_RUNTIME_STALE_AFTER_SECONDS=60
GOVOPLAN_EXPECTED_API_REPLICAS=1
GOVOPLAN_EXPECTED_WORKER_REPLICAS=1
MASTER_KEY_B64={master_key}
DATABASE_URL=postgresql+psycopg://govoplan:change-me@127.0.0.1:5432/govoplan
@@ -409,6 +765,7 @@ FILE_STORAGE_BACKEND=local
FILE_STORAGE_LOCAL_ROOT=/var/lib/govoplan/files
FILE_STORAGE_LOCAL_FALLBACK_ROOTS=
FILE_STORAGE_S3_DEPLOYMENT_MANAGED=false
FILE_STORAGE_S3_ENDPOINT_TRUSTED=false
FILE_ARCHIVE_MAX_ENTRIES=10000
FILE_ARCHIVE_MAX_EXPANDED_BYTES=2147483648
FILE_ARCHIVE_MAX_EXPANSION_RATIO=100
@@ -430,6 +787,13 @@ def _production_like_env_template(master_key: str) -> str:
APP_ENV=staging
GOVOPLAN_INSTALL_PROFILE=production-like
GOVOPLAN_INSTALLATION_ID=govoplan-production-like
GOVOPLAN_STATE_PROFILE=local
GOVOPLAN_RUNTIME_ROLE=api
GOVOPLAN_RUNTIME_HEARTBEAT_SECONDS=15
GOVOPLAN_RUNTIME_STALE_AFTER_SECONDS=60
GOVOPLAN_EXPECTED_API_REPLICAS=1
GOVOPLAN_EXPECTED_WORKER_REPLICAS=1
MASTER_KEY_B64={master_key}
GOVOPLAN_PRODUCTION_LIKE_POSTGRES_DB=govoplan
@@ -474,6 +838,7 @@ AUTH_COOKIE_SECURE=false
FILE_STORAGE_BACKEND=local
FILE_STORAGE_LOCAL_ROOT=runtime/production-like/files
FILE_STORAGE_S3_DEPLOYMENT_MANAGED=false
FILE_STORAGE_S3_ENDPOINT_TRUSTED=false
FILE_ARCHIVE_MAX_ENTRIES=10000
FILE_ARCHIVE_MAX_EXPANDED_BYTES=2147483648
FILE_ARCHIVE_MAX_EXPANSION_RATIO=100
@@ -488,7 +853,11 @@ def _clean(value: str | None) -> str:
def _csv(value: str | None) -> tuple[str, ...]:
return tuple(dict.fromkeys(item.strip() for item in str(value or "").split(",") if item.strip()))
return tuple(
dict.fromkeys(
item.strip() for item in str(value or "").split(",") if item.strip()
)
)
def _truthy(value: str | None) -> bool:
File diff suppressed because it is too large Load Diff
@@ -335,6 +335,18 @@ def module_install_preflight(
issues.append(ModuleInstallerIssue("warning", "empty_plan", "No planned package changes are present."))
if not maintenance_mode:
issues.append(ModuleInstallerIssue("blocker", "maintenance_required", "Package changes require maintenance mode."))
if os.getenv("GOVOPLAN_STATE_PROFILE", "local").strip().lower() == "shared":
issues.append(
ModuleInstallerIssue(
"blocker",
"immutable_cluster_release_required",
(
"Shared-state deployments cannot mutate packages on one runtime node. "
"Build and roll out one immutable release image across every API, worker, "
"and scheduler node."
),
)
)
activation_candidates = desired_modules_after_package_plan(desired_sequence, plan)
issues.extend(module_manifest_compatibility_issues(available, module_ids=activation_candidates))
@@ -3,6 +3,7 @@ from __future__ import annotations
import base64
import binascii
from collections import defaultdict
from collections.abc import Mapping
from dataclasses import dataclass
from datetime import UTC, datetime
from pathlib import Path
@@ -16,6 +17,11 @@ from cryptography.hazmat.primitives import serialization
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey, Ed25519PublicKey
from govoplan_core.core.versioning import format_version_range, version_range_is_valid, version_satisfies_range
from govoplan_core.core.provider_governance import (
external_provider_from_mapping,
module_architecture_from_mapping,
module_architecture_issues,
)
from govoplan_core.security.http_fetch import fetch_http_text, is_http_url
_INTERFACE_NAME_RE = re.compile(r"^[a-z][a-z0-9_]*(?:\.[a-z][a-z0-9_]*)+$")
@@ -613,6 +619,72 @@ def _normalize_catalog_item(value: Any) -> dict[str, object]:
"notes": _optional_str(value, "notes"),
"tags": _string_list(value.get("tags")),
}
raw_architecture = value.get("architecture")
if raw_architecture is not None:
if not isinstance(raw_architecture, Mapping):
raise ValueError(
f"Module package catalog architecture for {module_id!r} must be an object."
)
architecture = module_architecture_from_mapping(raw_architecture)
issues = module_architecture_issues(
architecture,
has_migrations=bool(item["migration_tasks"])
or bool(item["migration_notes"]),
)
if issues:
raise ValueError(
f"Module package catalog architecture for {module_id!r} is invalid: "
+ "; ".join(issues)
)
item["architecture"] = architecture.to_dict()
raw_providers = value.get("external_providers")
if raw_providers is not None:
if not isinstance(raw_providers, list):
raise ValueError(
f"Module package catalog external_providers for {module_id!r} must be a list."
)
providers = []
seen_provider_ids: set[str] = set()
for raw_provider in raw_providers:
if not isinstance(raw_provider, Mapping):
raise ValueError(
f"Module package catalog external provider entries for {module_id!r} must be objects."
)
provider = external_provider_from_mapping(raw_provider)
if provider.module_id != module_id:
raise ValueError(
f"Module package catalog provider {provider.id!r} belongs to "
f"{provider.module_id!r}, not {module_id!r}."
)
if provider.id in seen_provider_ids:
raise ValueError(
f"Module package catalog has duplicate provider {provider.id!r}."
)
seen_provider_ids.add(provider.id)
providers.append(provider)
if providers and "architecture" not in item:
raise ValueError(
f"Module package catalog {module_id!r} declares external providers without architecture metadata."
)
if providers:
architecture_payload = item["architecture"]
if not isinstance(architecture_payload, Mapping):
raise ValueError(
f"Module package catalog {module_id!r} has invalid architecture metadata."
)
architecture_modes = set(
_string_list(architecture_payload.get("supported_authority_modes"))
)
provider_modes = {
mode for provider in providers for mode in provider.authority_modes
}
missing_modes = provider_modes - architecture_modes
if missing_modes:
raise ValueError(
f"Module package catalog {module_id!r} provider modes are missing from architecture metadata: "
+ ", ".join(sorted(missing_modes))
)
item["external_providers"] = [provider.to_dict() for provider in providers]
if not version_range_is_valid(
version_min=item["current_version_min"] if isinstance(item["current_version_min"], str) else None,
version_max_exclusive=item["current_version_max_exclusive"] if isinstance(item["current_version_max_exclusive"], str) else None,
+11
View File
@@ -5,6 +5,11 @@ from dataclasses import dataclass, field
from typing import Any, Literal, Protocol, TYPE_CHECKING
from govoplan_core.core.ownership import OwnershipProviderRegistration
from govoplan_core.core.provider_governance import (
ExternalProviderDeclaration,
ExternalProviderStateProviderRegistration,
ModuleArchitectureDeclaration,
)
from govoplan_core.core.views import ViewSurface
if TYPE_CHECKING:
@@ -438,6 +443,12 @@ class ModuleManifest:
"OperationalCheckProviderRegistration",
...,
] = ()
architecture: ModuleArchitectureDeclaration | None = None
external_providers: tuple[ExternalProviderDeclaration, ...] = ()
external_provider_state_providers: tuple[
ExternalProviderStateProviderRegistration,
...,
] = ()
compatibility: ModuleCompatibility = field(default_factory=ModuleCompatibility)
on_activate: LifecycleHook | None = None
on_deactivate: LifecycleHook | None = None
+597
View File
@@ -0,0 +1,597 @@
from __future__ import annotations
from dataclasses import dataclass, field
from heapq import nsmallest
import os
from pathlib import Path
import tempfile
from typing import Any, Iterable, Protocol
from urllib.parse import urlsplit
from govoplan_core.security.outbound_http import (
OutboundHttpError,
response_limit,
validate_unpinned_sdk_http_url,
)
class StorageBackendError(RuntimeError):
"""Base error for the deployment-owned object-storage boundary."""
class StorageObjectMissing(StorageBackendError):
"""Raised when a referenced object no longer exists."""
@dataclass(frozen=True, slots=True)
class StorageObjectInfo:
key: str
size_bytes: int
@dataclass(frozen=True, slots=True)
class StorageObjectPage:
objects: tuple[StorageObjectInfo, ...]
next_cursor: str | None = None
class StorageBackend(Protocol):
"""Shared byte-object storage used by modules without cross-module imports."""
name: str
def put_bytes(
self,
key: str,
data: bytes,
*,
content_type: str | None = None,
) -> None: ...
def get_bytes(self, key: str) -> bytes: ...
def iter_bytes(
self,
key: str,
*,
chunk_size: int = 1024 * 1024,
) -> Iterable[bytes]: ...
def delete(self, key: str) -> None: ...
def exists(self, key: str) -> bool: ...
def stat(self, key: str) -> StorageObjectInfo: ...
def list_objects(
self,
*,
prefix: str,
after: str | None = None,
limit: int = 500,
) -> StorageObjectPage: ...
@dataclass(slots=True)
class LocalFilesystemStorageBackend:
root: Path
fallback_roots: tuple[Path, ...] = field(default_factory=tuple)
name: str = "local"
def __post_init__(self) -> None:
self.root = self.root.expanduser().resolve()
self.fallback_roots = tuple(
root.expanduser().resolve() for root in self.fallback_roots if root
)
self.root.mkdir(parents=True, exist_ok=True)
def _path_for_root(self, root: Path, key: str) -> Path:
normalized = normalize_storage_key(key)
path = (root / normalized).resolve()
if not path.is_relative_to(root):
raise StorageBackendError("Storage key escapes local storage root")
return path
def _path(self, key: str) -> Path:
return self._path_for_root(self.root, key)
def _readable_path(self, key: str) -> Path:
primary = self._path(key)
if primary.exists() and primary.is_file():
return primary
for root in self.fallback_roots:
candidate = self._path_for_root(root, key)
if candidate.exists() and candidate.is_file():
return candidate
raise StorageObjectMissing("Stored object does not exist")
def put_bytes(
self,
key: str,
data: bytes,
*,
content_type: str | None = None,
) -> None:
del content_type
path = self._path(key)
path.parent.mkdir(parents=True, exist_ok=True)
descriptor, temporary_name = tempfile.mkstemp(
prefix=f".{path.name}.",
suffix=".tmp",
dir=path.parent,
)
temporary = Path(temporary_name)
try:
os.fchmod(descriptor, 0o600)
with os.fdopen(descriptor, "wb") as stream:
descriptor = -1
stream.write(data)
stream.flush()
os.fsync(stream.fileno())
temporary.replace(path)
finally:
if descriptor >= 0:
os.close(descriptor)
temporary.unlink(missing_ok=True)
def get_bytes(self, key: str) -> bytes:
return self._readable_path(key).read_bytes()
def iter_bytes(
self,
key: str,
*,
chunk_size: int = 1024 * 1024,
) -> Iterable[bytes]:
path = self._readable_path(key)
with path.open("rb") as handle:
while True:
chunk = handle.read(chunk_size)
if not chunk:
break
yield chunk
def delete(self, key: str) -> None:
path = self._path(key)
if path.exists() and path.is_file():
path.unlink()
def exists(self, key: str) -> bool:
try:
self._readable_path(key)
except StorageObjectMissing:
return False
return True
def stat(self, key: str) -> StorageObjectInfo:
path = self._readable_path(key)
return StorageObjectInfo(
key=normalize_storage_key(key),
size_bytes=path.stat().st_size,
)
def list_objects(
self,
*,
prefix: str,
after: str | None = None,
limit: int = 500,
) -> StorageObjectPage:
normalized_prefix = normalize_storage_prefix(prefix)
normalized_after = normalize_storage_key(after) if after else None
normalized_limit = max(1, min(int(limit), 5000))
def matching_objects() -> Iterable[StorageObjectInfo]:
for path in _iter_local_files(self.root):
key = path.relative_to(self.root).as_posix()
if not key.startswith(normalized_prefix) or (
normalized_after is not None and key <= normalized_after
):
continue
yield StorageObjectInfo(
key=key,
size_bytes=path.stat().st_size,
)
candidates = nsmallest(
normalized_limit + 1,
matching_objects(),
key=lambda item: item.key,
)
has_more = len(candidates) > normalized_limit
page = tuple(candidates[:normalized_limit])
return StorageObjectPage(
objects=page,
next_cursor=page[-1].key if has_more and page else None,
)
@dataclass(slots=True)
class S3StorageBackend:
bucket: str
endpoint_url: str
region_name: str
access_key_id: str
secret_access_key: str
deployment_managed: bool = False
endpoint_trusted: bool = False
name: str = "s3"
_client: Any = field(default=None, init=False, repr=False)
@property
def client(self):
if self._client is not None:
return self._client
if self.deployment_managed:
endpoint_url = _deployment_managed_garage_endpoint(self.endpoint_url)
elif self.endpoint_trusted:
endpoint_url = _trusted_deployment_endpoint(self.endpoint_url)
else:
try:
endpoint_url = validate_unpinned_sdk_http_url(
self.endpoint_url,
label="Object storage S3 endpoint",
)
except OutboundHttpError as exc:
raise StorageBackendError(str(exc)) from exc
try:
import boto3
from botocore.config import Config
except ModuleNotFoundError as exc:
raise StorageBackendError(
"boto3 is required for the S3 storage backend"
) from exc
options: dict[str, object] = {
"endpoint_url": endpoint_url,
"region_name": self.region_name,
"aws_access_key_id": self.access_key_id,
"aws_secret_access_key": self.secret_access_key,
}
if self.deployment_managed:
options["config"] = Config(s3={"addressing_style": "path"})
self._client = boto3.client("s3", **options)
return self._client
def put_bytes(
self,
key: str,
data: bytes,
*,
content_type: str | None = None,
) -> None:
normalized = normalize_storage_key(key)
max_bytes = response_limit("file")
if len(data) > max_bytes:
raise StorageBackendError(
f"Stored object exceeds the deployment limit of {max_bytes} bytes"
)
kwargs: dict[str, object] = {
"Bucket": self.bucket,
"Key": normalized,
"Body": data,
}
if content_type:
kwargs["ContentType"] = content_type
try:
self.client.put_object(**kwargs)
except Exception as exc: # pragma: no cover - depends on S3 backend
raise StorageBackendError(str(exc)) from exc
def get_bytes(self, key: str) -> bytes:
normalized = normalize_storage_key(key)
try:
obj = self.client.get_object(
Bucket=self.bucket,
Key=normalized,
)
max_bytes = response_limit("file")
body = obj["Body"]
try:
_reject_declared_object_size(obj, max_bytes=max_bytes)
data = body.read(max_bytes + 1)
if len(data) > max_bytes:
raise StorageBackendError(
"Stored object exceeds the deployment limit of "
f"{max_bytes} bytes"
)
return data
finally:
if hasattr(body, "close"):
body.close()
except StorageBackendError:
raise
except Exception as exc: # pragma: no cover - depends on S3 backend
if _s3_missing_error(exc):
raise StorageObjectMissing("Stored object does not exist") from exc
raise StorageBackendError(str(exc)) from exc
def iter_bytes(
self,
key: str,
*,
chunk_size: int = 1024 * 1024,
) -> Iterable[bytes]:
normalized = normalize_storage_key(key)
try:
obj = self.client.get_object(
Bucket=self.bucket,
Key=normalized,
)
max_bytes = response_limit("file")
body = obj["Body"]
try:
_reject_declared_object_size(obj, max_bytes=max_bytes)
total = 0
while True:
chunk = body.read(chunk_size)
if not chunk:
break
total += len(chunk)
if total > max_bytes:
raise StorageBackendError(
"Stored object exceeds the deployment limit of "
f"{max_bytes} bytes"
)
yield chunk
finally:
if hasattr(body, "close"):
body.close()
except StorageBackendError:
raise
except Exception as exc: # pragma: no cover - depends on S3 backend
if _s3_missing_error(exc):
raise StorageObjectMissing("Stored object does not exist") from exc
raise StorageBackendError(str(exc)) from exc
def delete(self, key: str) -> None:
try:
self.client.delete_object(
Bucket=self.bucket,
Key=normalize_storage_key(key),
)
except Exception as exc: # pragma: no cover - depends on S3 backend
raise StorageBackendError(str(exc)) from exc
def exists(self, key: str) -> bool:
try:
self.client.head_object(
Bucket=self.bucket,
Key=normalize_storage_key(key),
)
return True
except Exception as exc:
if _s3_missing_error(exc):
return False
raise StorageBackendError(str(exc)) from exc
def stat(self, key: str) -> StorageObjectInfo:
normalized = normalize_storage_key(key)
try:
response = self.client.head_object(
Bucket=self.bucket,
Key=normalized,
)
except Exception as exc: # pragma: no cover - depends on S3 backend
if _s3_missing_error(exc):
raise StorageObjectMissing("Stored object does not exist") from exc
raise StorageBackendError(str(exc)) from exc
try:
size = int(response.get("ContentLength"))
except (AttributeError, TypeError, ValueError) as exc:
raise StorageBackendError(
"S3 object metadata did not include a valid size"
) from exc
return StorageObjectInfo(key=normalized, size_bytes=size)
def list_objects(
self,
*,
prefix: str,
after: str | None = None,
limit: int = 500,
) -> StorageObjectPage:
normalized_prefix = normalize_storage_prefix(prefix)
normalized_limit = max(1, min(int(limit), 1000))
kwargs: dict[str, object] = {
"Bucket": self.bucket,
"Prefix": normalized_prefix,
"MaxKeys": normalized_limit,
}
if after:
kwargs["StartAfter"] = normalize_storage_key(after)
try:
response = self.client.list_objects_v2(**kwargs)
except Exception as exc: # pragma: no cover - depends on S3 backend
raise StorageBackendError(str(exc)) from exc
objects = tuple(
StorageObjectInfo(
key=str(item["Key"]),
size_bytes=int(item.get("Size") or 0),
)
for item in response.get("Contents", ())
if isinstance(item, dict) and item.get("Key")
)
has_more = bool(response.get("IsTruncated"))
return StorageObjectPage(
objects=objects,
next_cursor=objects[-1].key if has_more and objects else None,
)
def configured_storage_backend(settings: object) -> StorageBackend:
"""Build the deployment-wide object store from Core settings.
Modules own their metadata and key namespaces. The deployment owns the
storage endpoint and credentials, so modules do not need to depend on the
Files package merely to persist opaque generated bytes.
"""
configured = (
str(getattr(settings, "file_storage_backend", "local") or "local")
.strip()
.lower()
)
if configured in {"local", "filesystem", "fs"}:
raw_fallbacks = str(
getattr(settings, "file_storage_local_fallback_roots", "") or ""
)
return LocalFilesystemStorageBackend(
Path(
str(
getattr(
settings,
"file_storage_local_root",
"runtime/files",
)
)
),
fallback_roots=tuple(
Path(item.strip()) for item in raw_fallbacks.split(",") if item.strip()
),
)
if configured in {"s3", "garage"}:
return S3StorageBackend(
bucket=str(
getattr(settings, "file_storage_s3_bucket", None)
or getattr(settings, "s3_bucket", "files")
),
endpoint_url=str(
getattr(settings, "file_storage_s3_endpoint_url", None)
or getattr(settings, "s3_endpoint_url", "")
),
region_name=str(
getattr(settings, "file_storage_s3_region", None)
or getattr(settings, "s3_region", "")
),
access_key_id=str(
getattr(settings, "file_storage_s3_access_key_id", None)
or getattr(settings, "s3_access_key_id", "")
),
secret_access_key=str(
getattr(
settings,
"file_storage_s3_secret_access_key",
None,
)
or getattr(settings, "s3_secret_access_key", "")
),
deployment_managed=bool(
getattr(
settings,
"file_storage_s3_deployment_managed",
False,
)
),
endpoint_trusted=bool(
getattr(
settings,
"file_storage_s3_endpoint_trusted",
False,
)
),
)
raise StorageBackendError(f"Unsupported object storage backend: {configured}")
def normalize_storage_key(value: str) -> str:
candidate = str(value or "").strip().replace("\\", "/")
parts = candidate.split("/")
if (
not candidate
or candidate.startswith("/")
or any(part in {"", ".", ".."} for part in parts)
or any(ord(character) < 32 for character in candidate)
):
raise StorageBackendError("Storage key is not a safe relative key")
return "/".join(parts)
def normalize_storage_prefix(value: str) -> str:
candidate = str(value or "").strip().replace("\\", "/")
if not candidate:
return ""
trailing_slash = candidate.endswith("/")
normalized = normalize_storage_key(candidate.rstrip("/"))
return normalized + ("/" if trailing_slash else "")
def _reject_declared_object_size(obj: object, *, max_bytes: int) -> None:
if not isinstance(obj, dict):
return
try:
declared_size = int(obj.get("ContentLength"))
except (TypeError, ValueError):
return
if declared_size > max_bytes:
raise StorageBackendError(
f"Stored object exceeds the deployment limit of {max_bytes} bytes"
)
def _iter_local_files(root: Path):
for entry in sorted(root.iterdir(), key=lambda item: item.name):
if entry.is_symlink():
continue
if entry.is_dir():
yield from _iter_local_files(entry)
elif entry.is_file():
yield entry
def _s3_missing_error(exc: Exception) -> bool:
response = getattr(exc, "response", None)
if not isinstance(response, dict):
return False
error = response.get("Error")
metadata = response.get("ResponseMetadata")
code = str(error.get("Code") if isinstance(error, dict) else "")
status_code = metadata.get("HTTPStatusCode") if isinstance(metadata, dict) else None
return code in {"404", "NoSuchKey", "NotFound"} or status_code == 404
def _deployment_managed_garage_endpoint(value: str) -> str:
endpoint = str(value or "").strip()
if endpoint != "http://garage:3900":
raise StorageBackendError(
"Deployment-managed S3 trust is restricted to http://garage:3900"
)
return endpoint
def _trusted_deployment_endpoint(value: str) -> str:
endpoint = str(value or "").strip()
parsed = urlsplit(endpoint)
try:
parsed.port
except ValueError as exc:
raise StorageBackendError(
"Deployment-trusted S3 endpoint has an invalid port"
) from exc
if (
parsed.scheme.lower() != "https"
or not parsed.hostname
or parsed.username
or parsed.password
or parsed.query
or parsed.fragment
or parsed.path not in {"", "/"}
):
raise StorageBackendError(
"Deployment-trusted S3 endpoint must be an HTTPS origin without "
"credentials, query, fragment, or path"
)
return endpoint.rstrip("/")
__all__ = [
"LocalFilesystemStorageBackend",
"S3StorageBackend",
"StorageBackend",
"StorageBackendError",
"StorageObjectInfo",
"StorageObjectMissing",
"StorageObjectPage",
"configured_storage_backend",
"normalize_storage_key",
"normalize_storage_prefix",
]
File diff suppressed because it is too large Load Diff
+669
View File
@@ -0,0 +1,669 @@
from __future__ import annotations
from dataclasses import dataclass
from datetime import datetime, timezone
from enum import StrEnum
import hashlib
import json
from typing import Any
from uuid import uuid4
from sqlalchemy import (
BigInteger,
DateTime,
ForeignKey,
Index,
Integer,
JSON,
String,
Text,
UniqueConstraint,
select,
)
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Mapped, Session, mapped_column
from govoplan_core.core.runtime_coordination import LeaseClaim, assert_lease_fence
from govoplan_core.db.base import Base, TimestampMixin, utcnow
from govoplan_core.security.redaction import contains_plain_secret
class RecoveryMode(StrEnum):
ATOMIC = "atomic"
COMPENSATION = "compensation"
SNAPSHOT_RESTORE = "snapshot_restore"
FORWARD_RECOVERY = "forward_recovery"
IRREVERSIBLE = "irreversible"
class RecoveryStatus(StrEnum):
PLANNED = "planned"
PREPARED = "prepared"
RUNNING = "running"
SUCCEEDED = "succeeded"
FAILED = "failed"
OUTCOME_UNKNOWN = "outcome_unknown"
RECOVERY_REQUIRED = "recovery_required"
RECOVERING = "recovering"
RECOVERED = "recovered"
MANUAL_INTERVENTION = "manual_intervention"
TERMINAL_RECOVERY_STATUSES = frozenset(
{
RecoveryStatus.SUCCEEDED.value,
RecoveryStatus.FAILED.value,
RecoveryStatus.RECOVERED.value,
RecoveryStatus.MANUAL_INTERVENTION.value,
}
)
_TRANSITIONS: dict[str, frozenset[str]] = {
RecoveryStatus.PLANNED.value: frozenset(
{RecoveryStatus.PREPARED.value, RecoveryStatus.FAILED.value}
),
RecoveryStatus.PREPARED.value: frozenset(
{RecoveryStatus.RUNNING.value, RecoveryStatus.FAILED.value}
),
RecoveryStatus.RUNNING.value: frozenset(
{
RecoveryStatus.SUCCEEDED.value,
RecoveryStatus.FAILED.value,
RecoveryStatus.OUTCOME_UNKNOWN.value,
RecoveryStatus.RECOVERY_REQUIRED.value,
}
),
RecoveryStatus.OUTCOME_UNKNOWN.value: frozenset(
{
RecoveryStatus.SUCCEEDED.value,
RecoveryStatus.RECOVERY_REQUIRED.value,
RecoveryStatus.MANUAL_INTERVENTION.value,
}
),
RecoveryStatus.RECOVERY_REQUIRED.value: frozenset(
{
RecoveryStatus.RECOVERING.value,
RecoveryStatus.MANUAL_INTERVENTION.value,
}
),
RecoveryStatus.RECOVERING.value: frozenset(
{
RecoveryStatus.RECOVERED.value,
RecoveryStatus.MANUAL_INTERVENTION.value,
}
),
}
class RecoveryGuaranteeError(ValueError):
pass
class RecoveryIdempotencyConflict(RecoveryGuaranteeError):
pass
class RecoveryOperation(Base, TimestampMixin):
__tablename__ = "core_recovery_operations"
__table_args__ = (
UniqueConstraint(
"installation_id",
"module_id",
"idempotency_key",
name="uq_core_recovery_operation_idempotency",
),
Index(
"ix_core_recovery_operations_status_updated",
"installation_id",
"status",
"updated_at",
),
Index(
"ix_core_recovery_operations_resource",
"module_id",
"resource_type",
"resource_id",
),
)
id: Mapped[str] = mapped_column(
String(36),
primary_key=True,
default=lambda: str(uuid4()),
)
installation_id: Mapped[str] = mapped_column(
String(100), nullable=False, index=True
)
module_id: Mapped[str] = mapped_column(String(100), nullable=False, index=True)
operation_type: Mapped[str] = mapped_column(String(100), nullable=False, index=True)
resource_type: Mapped[str | None] = mapped_column(String(100), index=True)
resource_id: Mapped[str | None] = mapped_column(String(255), index=True)
mode: Mapped[str] = mapped_column(String(40), nullable=False, index=True)
status: Mapped[str] = mapped_column(
String(40),
default=RecoveryStatus.PLANNED.value,
nullable=False,
index=True,
)
idempotency_key: Mapped[str] = mapped_column(String(200), nullable=False)
request_sha256: Mapped[str] = mapped_column(String(64), nullable=False)
plan: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict, nullable=False)
backup_reference: Mapped[str | None] = mapped_column(String(1000))
approval_reference: Mapped[str | None] = mapped_column(String(1000))
lease_resource_key: Mapped[str | None] = mapped_column(String(255))
holder_node_id: Mapped[str | None] = mapped_column(String(200))
holder_incarnation: Mapped[str | None] = mapped_column(String(36))
fencing_token: Mapped[int | None] = mapped_column(
BigInteger().with_variant(Integer, "sqlite")
)
checkpoint_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
evidence_head_sha256: Mapped[str | None] = mapped_column(String(64))
failure_summary: Mapped[str | None] = mapped_column(Text)
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
completed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
recovery_started_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True)
)
recovered_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
revision: Mapped[int] = mapped_column(Integer, default=1, nullable=False)
metadata_: Mapped[dict[str, Any]] = mapped_column(
"metadata", JSON, default=dict, nullable=False
)
class RecoveryCheckpoint(Base):
__tablename__ = "core_recovery_checkpoints"
__table_args__ = (
UniqueConstraint(
"operation_id",
"sequence",
name="uq_core_recovery_checkpoint_sequence",
),
Index(
"ix_core_recovery_checkpoints_operation_created",
"operation_id",
"created_at",
),
)
id: Mapped[str] = mapped_column(
String(36),
primary_key=True,
default=lambda: str(uuid4()),
)
operation_id: Mapped[str] = mapped_column(
ForeignKey("core_recovery_operations.id", ondelete="CASCADE"),
nullable=False,
index=True,
)
sequence: Mapped[int] = mapped_column(Integer, nullable=False)
status: Mapped[str] = mapped_column(String(40), nullable=False, index=True)
kind: Mapped[str] = mapped_column(String(80), nullable=False)
summary: Mapped[str] = mapped_column(Text, nullable=False)
evidence: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict, nullable=False)
previous_sha256: Mapped[str | None] = mapped_column(String(64))
checkpoint_sha256: Mapped[str] = mapped_column(
String(64), nullable=False, index=True
)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), default=utcnow, nullable=False
)
@dataclass(frozen=True, slots=True)
class RecoveryPlan:
mode: RecoveryMode
preconditions: tuple[str, ...] = ()
compensation_steps: tuple[str, ...] = ()
forward_recovery_steps: tuple[str, ...] = ()
verification_steps: tuple[str, ...] = ()
backup_reference: str | None = None
approval_reference: str | None = None
def validate(self) -> None:
if not self.verification_steps:
raise RecoveryGuaranteeError(
"Recovery plans require at least one verification step"
)
if self.mode == RecoveryMode.COMPENSATION and not self.compensation_steps:
raise RecoveryGuaranteeError(
"Compensation recovery requires explicit compensation steps"
)
if self.mode == RecoveryMode.SNAPSHOT_RESTORE and not self.backup_reference:
raise RecoveryGuaranteeError(
"Snapshot restore requires a verified backup reference"
)
if (
self.mode == RecoveryMode.FORWARD_RECOVERY
and not self.forward_recovery_steps
):
raise RecoveryGuaranteeError(
"Forward recovery requires explicit forward-recovery steps"
)
if self.mode == RecoveryMode.IRREVERSIBLE and not self.approval_reference:
raise RecoveryGuaranteeError(
"Irreversible operations require an approval reference"
)
def as_dict(self) -> dict[str, Any]:
return {
"mode": self.mode.value,
"preconditions": list(self.preconditions),
"compensation_steps": list(self.compensation_steps),
"forward_recovery_steps": list(self.forward_recovery_steps),
"verification_steps": list(self.verification_steps),
"backup_reference": self.backup_reference,
"approval_reference": self.approval_reference,
}
def plan_recovery_operation(
session: Session,
*,
installation_id: str,
module_id: str,
operation_type: str,
idempotency_key: str,
request: dict[str, Any],
recovery_plan: RecoveryPlan,
resource_type: str | None = None,
resource_id: str | None = None,
lease_claim: LeaseClaim | None = None,
metadata: dict[str, Any] | None = None,
) -> RecoveryOperation:
recovery_plan.validate()
request_sha256 = _canonical_sha256(request)
existing = session.execute(
select(RecoveryOperation).where(
RecoveryOperation.installation_id == installation_id,
RecoveryOperation.module_id == module_id,
RecoveryOperation.idempotency_key == idempotency_key,
)
).scalar_one_or_none()
if existing is not None:
if existing.request_sha256 != request_sha256:
raise RecoveryIdempotencyConflict(
"Recovery operation idempotency key was reused for another request"
)
return existing
if contains_plain_secret(metadata or {}):
raise RecoveryGuaranteeError(
"Recovery metadata must contain secret references, not plaintext secrets"
)
if lease_claim is not None:
if lease_claim.installation_id != installation_id:
raise RecoveryGuaranteeError(
"Recovery operation and lease belong to different installations"
)
assert_lease_fence(session, lease_claim)
operation = RecoveryOperation(
installation_id=installation_id,
module_id=module_id,
operation_type=operation_type,
resource_type=resource_type,
resource_id=resource_id,
mode=recovery_plan.mode.value,
status=RecoveryStatus.PLANNED.value,
idempotency_key=idempotency_key,
request_sha256=request_sha256,
plan=recovery_plan.as_dict(),
backup_reference=recovery_plan.backup_reference,
approval_reference=recovery_plan.approval_reference,
lease_resource_key=lease_claim.resource_key if lease_claim else None,
holder_node_id=lease_claim.holder_node_id if lease_claim else None,
holder_incarnation=lease_claim.holder_incarnation if lease_claim else None,
fencing_token=lease_claim.fencing_token if lease_claim else None,
metadata_=dict(metadata or {}),
)
try:
with session.begin_nested():
session.add(operation)
session.flush()
except IntegrityError:
existing = session.execute(
select(RecoveryOperation).where(
RecoveryOperation.installation_id == installation_id,
RecoveryOperation.module_id == module_id,
RecoveryOperation.idempotency_key == idempotency_key,
)
).scalar_one_or_none()
if existing is None:
raise
if existing.request_sha256 != request_sha256:
raise RecoveryIdempotencyConflict(
"Recovery operation idempotency key was reused for another request"
)
return existing
record_recovery_checkpoint(
session,
operation,
kind="plan",
summary="Recovery contract recorded before side effects",
evidence={"request_sha256": request_sha256, "plan": recovery_plan.as_dict()},
lease_claim=lease_claim,
)
return operation
def prepare_recovery_operation(
session: Session,
operation: RecoveryOperation,
*,
evidence: dict[str, Any],
lease_claim: LeaseClaim | None = None,
) -> RecoveryOperation:
_verify_operation_fence(session, operation, lease_claim)
if not evidence:
raise RecoveryGuaranteeError(
"Recovery preparation requires durable precondition evidence"
)
return transition_recovery_operation(
session,
operation,
status=RecoveryStatus.PREPARED,
kind="prepared",
summary="Preconditions and recovery material verified",
evidence=evidence,
lease_claim=lease_claim,
)
def start_recovery_operation(
session: Session,
operation: RecoveryOperation,
*,
evidence: dict[str, Any] | None = None,
lease_claim: LeaseClaim | None = None,
now: datetime | None = None,
) -> RecoveryOperation:
_verify_operation_fence(session, operation, lease_claim)
operation.started_at = _as_utc(now or utcnow())
return transition_recovery_operation(
session,
operation,
status=RecoveryStatus.RUNNING,
kind="started",
summary="Guarded operation started",
evidence=evidence or {},
lease_claim=lease_claim,
)
def transition_recovery_operation(
session: Session,
operation: RecoveryOperation,
*,
status: RecoveryStatus,
kind: str,
summary: str,
evidence: dict[str, Any] | None = None,
failure_summary: str | None = None,
lease_claim: LeaseClaim | None = None,
now: datetime | None = None,
) -> RecoveryOperation:
locked = session.execute(
select(RecoveryOperation)
.where(RecoveryOperation.id == operation.id)
.with_for_update()
).scalar_one()
_verify_operation_fence(session, locked, lease_claim)
allowed = _TRANSITIONS.get(locked.status, frozenset())
if status.value not in allowed:
raise RecoveryGuaranteeError(
f"Recovery transition {locked.status!r} -> {status.value!r} is not allowed"
)
_validate_mode_transition(locked, status)
_validate_transition_evidence(
status=status,
evidence=evidence or {},
failure_summary=failure_summary,
)
observed_at = _as_utc(now or utcnow())
locked.status = status.value
locked.revision = int(locked.revision or 0) + 1
if failure_summary is not None:
locked.failure_summary = failure_summary
if status == RecoveryStatus.SUCCEEDED:
locked.completed_at = observed_at
elif status == RecoveryStatus.RECOVERING:
locked.recovery_started_at = observed_at
elif status == RecoveryStatus.RECOVERED:
locked.recovered_at = observed_at
locked.completed_at = observed_at
elif status in {RecoveryStatus.FAILED, RecoveryStatus.MANUAL_INTERVENTION}:
locked.completed_at = observed_at
session.add(locked)
record_recovery_checkpoint(
session,
locked,
kind=kind,
summary=summary,
evidence=evidence or {},
lease_claim=lease_claim,
now=observed_at,
)
session.flush()
return locked
def record_recovery_checkpoint(
session: Session,
operation: RecoveryOperation,
*,
kind: str,
summary: str,
evidence: dict[str, Any],
lease_claim: LeaseClaim | None = None,
now: datetime | None = None,
) -> RecoveryCheckpoint:
if contains_plain_secret(evidence):
raise RecoveryGuaranteeError(
"Recovery evidence must contain secret references, not plaintext secrets"
)
locked = session.execute(
select(RecoveryOperation)
.where(RecoveryOperation.id == operation.id)
.with_for_update()
).scalar_one()
_verify_operation_fence(session, locked, lease_claim)
observed_at = _as_utc(now or utcnow())
sequence = int(locked.checkpoint_count or 0) + 1
payload = {
"operation_id": locked.id,
"sequence": sequence,
"status": locked.status,
"kind": kind,
"summary": summary,
"evidence": evidence,
"previous_sha256": locked.evidence_head_sha256,
"created_at": observed_at.isoformat(),
}
checkpoint_hash = _canonical_sha256(payload)
checkpoint = RecoveryCheckpoint(
operation_id=locked.id,
sequence=sequence,
status=locked.status,
kind=kind,
summary=summary,
evidence=dict(evidence),
previous_sha256=locked.evidence_head_sha256,
checkpoint_sha256=checkpoint_hash,
created_at=observed_at,
)
locked.checkpoint_count = sequence
locked.evidence_head_sha256 = checkpoint_hash
session.add(locked)
session.add(checkpoint)
session.flush()
return checkpoint
def verify_recovery_evidence_chain(
session: Session,
operation_id: str,
) -> bool:
operation = session.get(RecoveryOperation, operation_id)
if operation is None:
raise RecoveryGuaranteeError("Recovery operation was not found")
checkpoints = (
session.execute(
select(RecoveryCheckpoint)
.where(RecoveryCheckpoint.operation_id == operation_id)
.order_by(RecoveryCheckpoint.sequence)
)
.scalars()
.all()
)
previous: str | None = None
for expected_sequence, checkpoint in enumerate(checkpoints, start=1):
if (
checkpoint.sequence != expected_sequence
or checkpoint.previous_sha256 != previous
):
return False
payload = {
"operation_id": checkpoint.operation_id,
"sequence": checkpoint.sequence,
"status": checkpoint.status,
"kind": checkpoint.kind,
"summary": checkpoint.summary,
"evidence": checkpoint.evidence,
"previous_sha256": checkpoint.previous_sha256,
"created_at": _as_utc(checkpoint.created_at).isoformat(),
}
if _canonical_sha256(payload) != checkpoint.checkpoint_sha256:
return False
previous = checkpoint.checkpoint_sha256
return (
len(checkpoints) == int(operation.checkpoint_count or 0)
and previous == operation.evidence_head_sha256
)
def operation_recovery_action(mode: RecoveryMode) -> RecoveryStatus:
if mode == RecoveryMode.ATOMIC:
return RecoveryStatus.FAILED
if mode in {
RecoveryMode.COMPENSATION,
RecoveryMode.SNAPSHOT_RESTORE,
RecoveryMode.FORWARD_RECOVERY,
}:
return RecoveryStatus.RECOVERY_REQUIRED
return RecoveryStatus.MANUAL_INTERVENTION
def _validate_mode_transition(
operation: RecoveryOperation,
status: RecoveryStatus,
) -> None:
mode = RecoveryMode(operation.mode)
if (
operation.status == RecoveryStatus.RUNNING.value
and status == RecoveryStatus.FAILED
and mode != RecoveryMode.ATOMIC
):
raise RecoveryGuaranteeError(
"A non-atomic operation cannot be marked failed after it starts; "
"record recovery_required, outcome_unknown, or manual_intervention"
)
if mode == RecoveryMode.ATOMIC and status in {
RecoveryStatus.RECOVERY_REQUIRED,
RecoveryStatus.RECOVERING,
RecoveryStatus.RECOVERED,
}:
raise RecoveryGuaranteeError(
"Atomic operations must roll back in their transaction instead of entering recovery"
)
if mode == RecoveryMode.IRREVERSIBLE and status in {
RecoveryStatus.RECOVERY_REQUIRED,
RecoveryStatus.RECOVERING,
RecoveryStatus.RECOVERED,
}:
raise RecoveryGuaranteeError(
"Irreversible operations cannot claim automated recovery"
)
def _validate_transition_evidence(
*,
status: RecoveryStatus,
evidence: dict[str, Any],
failure_summary: str | None,
) -> None:
if status in {RecoveryStatus.SUCCEEDED, RecoveryStatus.RECOVERED}:
checks = evidence.get("checks")
if (
evidence.get("verified") is not True
or not isinstance(
checks,
(dict, list),
)
or not checks
):
raise RecoveryGuaranteeError(
"Successful recovery transitions require verified evidence and check results"
)
if status == RecoveryStatus.MANUAL_INTERVENTION and not failure_summary:
raise RecoveryGuaranteeError(
"Manual intervention requires an operator-facing failure summary"
)
def _verify_operation_fence(
session: Session,
operation: RecoveryOperation,
lease_claim: LeaseClaim | None,
) -> None:
if operation.lease_resource_key is None:
return
if lease_claim is None:
raise RecoveryGuaranteeError(
"This recovery operation requires its distributed lease fence"
)
if (
lease_claim.resource_key != operation.lease_resource_key
or lease_claim.holder_node_id != operation.holder_node_id
or lease_claim.holder_incarnation != operation.holder_incarnation
or lease_claim.fencing_token != operation.fencing_token
):
raise RecoveryGuaranteeError(
"Recovery operation lease does not match its recorded fence"
)
assert_lease_fence(session, lease_claim)
def _canonical_sha256(value: dict[str, Any]) -> str:
encoded = json.dumps(
value,
sort_keys=True,
separators=(",", ":"),
ensure_ascii=False,
default=str,
).encode("utf-8")
return hashlib.sha256(encoded).hexdigest()
def _as_utc(value: datetime) -> datetime:
if value.tzinfo is None:
return value.replace(tzinfo=timezone.utc)
return value.astimezone(timezone.utc)
__all__ = [
"RecoveryCheckpoint",
"RecoveryGuaranteeError",
"RecoveryIdempotencyConflict",
"RecoveryMode",
"RecoveryOperation",
"RecoveryPlan",
"RecoveryStatus",
"TERMINAL_RECOVERY_STATUSES",
"operation_recovery_action",
"plan_recovery_operation",
"prepare_recovery_operation",
"record_recovery_checkpoint",
"start_recovery_operation",
"transition_recovery_operation",
"verify_recovery_evidence_chain",
]
+184
View File
@@ -29,6 +29,12 @@ from govoplan_core.core.ownership import (
OwnershipProviderRegistration,
ResourceOwnershipProvider,
)
from govoplan_core.core.provider_governance import (
ExternalProviderDeclaration,
ExternalProviderStateProviderRegistration,
ModuleArchitectureDeclaration,
module_architecture_issues,
)
from govoplan_core.core.versioning import format_version_range, version_range_is_valid, version_satisfies_range
from govoplan_core.core.search import (
RegisteredSearchProvider,
@@ -163,6 +169,33 @@ class PlatformRegistry:
def manifests(self) -> tuple[ModuleManifest, ...]:
return tuple(self._topologically_sorted())
def module_architectures(
self,
) -> tuple[tuple[str, ModuleArchitectureDeclaration], ...]:
return tuple(
(manifest.id, manifest.architecture)
for manifest in self.manifests()
if manifest.architecture is not None
)
def external_provider_declarations(
self,
) -> tuple[ExternalProviderDeclaration, ...]:
return tuple(
declaration
for manifest in self.manifests()
for declaration in manifest.external_providers
)
def external_provider_state_providers(
self,
) -> tuple[ExternalProviderStateProviderRegistration, ...]:
return tuple(
registration
for manifest in self.manifests()
for registration in manifest.external_provider_state_providers
)
def permissions(self) -> tuple[PermissionDefinition, ...]:
return tuple(permission for manifest in self.manifests() for permission in manifest.permissions)
@@ -629,9 +662,160 @@ def _validate_manifest_shape(manifest: ModuleManifest) -> None:
f"Module {manifest.id!r} documentation topic {topic.id!r}: {issue}"
)
_validate_documentation_extensions(manifest)
_validate_architecture_declarations(manifest)
_validate_workflow_definition_contributions(manifest)
def _validate_architecture_declarations(manifest: ModuleManifest) -> None:
architecture = manifest.architecture
if architecture is not None:
for issue in module_architecture_issues(
architecture,
has_migrations=manifest.migration_spec is not None,
):
raise RegistryError(
f"Module {manifest.id!r} architecture declaration: {issue}"
)
provider_ids: set[str] = set()
declared_capabilities = {
*manifest.required_capabilities,
*manifest.optional_capabilities,
*manifest.capability_factories,
}
declared_interfaces = {
*(item.name for item in manifest.provides_interfaces),
*(item.name for item in manifest.requires_interfaces),
}
operational_check_ids = {
item.check_id for item in manifest.operational_check_providers
}
documentation_topic_ids = {item.id for item in manifest.documentation}
ownership_resource_types = {
item.resource_type for item in manifest.ownership_providers
}
provider_authority_modes: set[str] = set()
for declaration in manifest.external_providers:
if declaration.module_id != manifest.id:
raise RegistryError(
f"Provider declaration {declaration.id!r} belongs to "
f"{declaration.module_id!r}, not module {manifest.id!r}"
)
if declaration.id in provider_ids:
raise RegistryError(
f"Module {manifest.id!r} declares duplicate external provider "
f"{declaration.id!r}"
)
provider_ids.add(declaration.id)
provider_authority_modes.update(declaration.authority_modes)
_validate_provider_references(
manifest.id,
declaration,
declared_capabilities=declared_capabilities,
declared_interfaces=declared_interfaces,
operational_check_ids=operational_check_ids,
documentation_topic_ids=documentation_topic_ids,
ownership_resource_types=ownership_resource_types,
)
state_provider_ids: set[str] = set()
for registration in manifest.external_provider_state_providers:
if registration.module_id != manifest.id:
raise RegistryError(
f"Provider state registration {registration.provider_id!r} belongs "
f"to {registration.module_id!r}, not module {manifest.id!r}"
)
if registration.provider_id not in provider_ids:
raise RegistryError(
f"Module {manifest.id!r} registers runtime state for undeclared "
f"provider {registration.provider_id!r}"
)
if registration.provider_id in state_provider_ids:
raise RegistryError(
f"Module {manifest.id!r} registers duplicate runtime state for "
f"provider {registration.provider_id!r}"
)
state_provider_ids.add(registration.provider_id)
missing_state_provider_ids = provider_ids - state_provider_ids
if missing_state_provider_ids:
raise RegistryError(
f"Module {manifest.id!r} external providers require sanitized runtime "
"state providers: " + ", ".join(sorted(missing_state_provider_ids))
)
if architecture is None:
if manifest.external_providers or manifest.external_provider_state_providers:
raise RegistryError(
f"Module {manifest.id!r} declares external providers without a "
"module architecture declaration"
)
return
missing_modes = provider_authority_modes - set(
architecture.supported_authority_modes
)
if missing_modes:
raise RegistryError(
f"Module {manifest.id!r} provider authority modes are not declared "
"by its architecture metadata: " + ", ".join(sorted(missing_modes))
)
unknown_target_providers = set(
architecture.target_tested_providers
) - provider_ids
if unknown_target_providers:
raise RegistryError(
f"Module {manifest.id!r} target-tested providers are not declared: "
+ ", ".join(sorted(unknown_target_providers))
)
def _validate_provider_references(
module_id: str,
declaration: ExternalProviderDeclaration,
*,
declared_capabilities: set[str],
declared_interfaces: set[str],
operational_check_ids: set[str],
documentation_topic_ids: set[str],
ownership_resource_types: set[str],
) -> None:
references = (
(
"capabilities",
set(declaration.capability_names),
declared_capabilities,
),
(
"interfaces",
set(declaration.interface_names),
declared_interfaces,
),
(
"operational checks",
set(declaration.operational_check_ids),
operational_check_ids,
),
(
"documentation topics",
set(declaration.documentation_topic_ids),
documentation_topic_ids,
),
(
"ownership resource types",
set(declaration.ownership_resource_types),
ownership_resource_types,
),
)
for label, requested, available in references:
missing = requested - available
if missing:
raise RegistryError(
f"Module {module_id!r} provider {declaration.id!r} references "
f"undeclared {label}: " + ", ".join(sorted(missing))
)
def _validate_workflow_definition_contributions(
manifest: ModuleManifest,
) -> None:
@@ -0,0 +1,605 @@
from __future__ import annotations
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from enum import StrEnum
import hashlib
import json
import os
import socket
from typing import Any
from uuid import uuid4
from sqlalchemy import BigInteger, DateTime, Index, Integer, JSON, String, UniqueConstraint, select
from sqlalchemy.orm import Mapped, Session, mapped_column
from govoplan_core.db.base import Base, TimestampMixin, utcnow
class RuntimeNodeState(StrEnum):
ACTIVE = "active"
DRAINING = "draining"
STOPPED = "stopped"
class RuntimeCoordinationError(RuntimeError):
pass
class LeaseUnavailable(RuntimeCoordinationError):
pass
class StaleFence(RuntimeCoordinationError):
pass
class RuntimeNode(Base, TimestampMixin):
__tablename__ = "core_runtime_nodes"
__table_args__ = (
UniqueConstraint(
"installation_id",
"node_id",
name="uq_core_runtime_node_installation_node",
),
Index(
"ix_core_runtime_nodes_installation_state_heartbeat",
"installation_id",
"state",
"last_heartbeat_at",
),
)
id: Mapped[str] = mapped_column(
String(36),
primary_key=True,
default=lambda: str(uuid4()),
)
installation_id: Mapped[str] = mapped_column(String(100), nullable=False, index=True)
node_id: Mapped[str] = mapped_column(String(200), nullable=False, index=True)
incarnation: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
role: Mapped[str] = mapped_column(String(40), nullable=False, index=True)
software_version: Mapped[str] = mapped_column(String(80), nullable=False)
composition_hash: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
queues: Mapped[list[str]] = mapped_column(JSON, default=list, nullable=False)
state: Mapped[str] = mapped_column(
String(30),
default=RuntimeNodeState.ACTIVE.value,
nullable=False,
index=True,
)
started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, nullable=False)
last_heartbeat_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, nullable=False)
drain_requested_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
drain_reason: Mapped[str | None] = mapped_column(String(500))
stopped_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
metadata_: Mapped[dict[str, Any]] = mapped_column("metadata", JSON, default=dict, nullable=False)
class DistributedLease(Base, TimestampMixin):
__tablename__ = "core_distributed_leases"
__table_args__ = (
UniqueConstraint(
"installation_id",
"resource_key",
name="uq_core_distributed_lease_resource",
),
Index(
"ix_core_distributed_leases_expiry",
"installation_id",
"expires_at",
),
)
id: Mapped[str] = mapped_column(
String(36),
primary_key=True,
default=lambda: str(uuid4()),
)
installation_id: Mapped[str] = mapped_column(String(100), nullable=False, index=True)
resource_key: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
holder_node_id: Mapped[str | None] = mapped_column(String(200), index=True)
holder_incarnation: Mapped[str | None] = mapped_column(String(36), index=True)
fencing_token: Mapped[int] = mapped_column(
BigInteger().with_variant(Integer, "sqlite"),
default=0,
nullable=False,
)
acquired_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
renewed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, nullable=False)
metadata_: Mapped[dict[str, Any]] = mapped_column("metadata", JSON, default=dict, nullable=False)
@dataclass(frozen=True, slots=True)
class RuntimeIdentity:
installation_id: str
node_id: str
incarnation: str
role: str
software_version: str
composition_hash: str
queues: tuple[str, ...] = ()
@dataclass(frozen=True, slots=True)
class LeaseClaim:
installation_id: str
resource_key: str
holder_node_id: str
holder_incarnation: str
fencing_token: int
expires_at: datetime
def runtime_identity(
settings: object,
*,
software_version: str,
module_ids: tuple[str, ...] = (),
role: str | None = None,
node_id: str | None = None,
incarnation: str | None = None,
queues: tuple[str, ...] | None = None,
) -> RuntimeIdentity:
effective_role = str(
role or getattr(settings, "runtime_role", "api") or "api"
).strip().lower()
effective_node_id = str(
node_id
or getattr(settings, "runtime_node_id", None)
or os.getenv("HOSTNAME")
or socket.gethostname()
).strip()
installation_id = str(
getattr(settings, "installation_id", "govoplan-local")
or "govoplan-local"
).strip()
if not installation_id or not effective_node_id or not effective_role:
raise RuntimeCoordinationError(
"Runtime identity requires installation, node, and role identifiers"
)
effective_queues = queues
if effective_queues is None:
effective_queues = tuple(
item.strip()
for item in str(getattr(settings, "celery_queues", "") or "").split(",")
if item.strip()
)
composition_hash = hashlib.sha256(
json.dumps(sorted(module_ids), separators=(",", ":")).encode("utf-8")
).hexdigest()
return RuntimeIdentity(
installation_id=installation_id,
node_id=effective_node_id,
incarnation=incarnation or str(uuid4()),
role=effective_role,
software_version=str(software_version),
composition_hash=composition_hash,
queues=tuple(sorted(set(effective_queues))),
)
def register_runtime_node(
session: Session,
identity: RuntimeIdentity,
*,
metadata: dict[str, Any] | None = None,
now: datetime | None = None,
) -> RuntimeNode:
observed_at = _as_utc(now or utcnow())
_insert_node_placeholder(session, identity, observed_at)
node = _locked_node(session, identity.installation_id, identity.node_id)
if node is None: # pragma: no cover - guarded by insert/upsert
raise RuntimeCoordinationError("Runtime node registration was not persisted")
node.incarnation = identity.incarnation
node.role = identity.role
node.software_version = identity.software_version
node.composition_hash = identity.composition_hash
node.queues = list(identity.queues)
node.state = RuntimeNodeState.ACTIVE.value
node.started_at = observed_at
node.last_heartbeat_at = observed_at
node.drain_requested_at = None
node.drain_reason = None
node.stopped_at = None
node.metadata_ = dict(metadata or {})
session.add(node)
session.flush()
return node
def heartbeat_runtime_node(
session: Session,
identity: RuntimeIdentity,
*,
now: datetime | None = None,
metadata: dict[str, Any] | None = None,
) -> RuntimeNode:
node = _locked_node(session, identity.installation_id, identity.node_id)
if node is None or node.incarnation != identity.incarnation:
raise RuntimeCoordinationError(
"Runtime heartbeat was rejected for a stale node incarnation"
)
if node.state == RuntimeNodeState.STOPPED.value:
raise RuntimeCoordinationError("Stopped runtime node cannot heartbeat")
node.last_heartbeat_at = _as_utc(now or utcnow())
if metadata is not None:
node.metadata_ = dict(metadata)
session.add(node)
session.flush()
return node
def request_runtime_node_drain(
session: Session,
*,
installation_id: str,
node_id: str,
reason: str | None = None,
now: datetime | None = None,
) -> RuntimeNode:
node = _locked_node(session, installation_id, node_id)
if node is None:
raise RuntimeCoordinationError("Runtime node was not found")
if node.state == RuntimeNodeState.STOPPED.value:
raise RuntimeCoordinationError("Stopped runtime node cannot be drained")
node.state = RuntimeNodeState.DRAINING.value
node.drain_requested_at = _as_utc(now or utcnow())
node.drain_reason = str(reason or "operator request")[:500]
session.add(node)
session.flush()
return node
def cancel_runtime_node_drain(
session: Session,
*,
installation_id: str,
node_id: str,
) -> RuntimeNode:
node = _locked_node(session, installation_id, node_id)
if node is None:
raise RuntimeCoordinationError("Runtime node was not found")
if node.state != RuntimeNodeState.DRAINING.value:
return node
node.state = RuntimeNodeState.ACTIVE.value
node.drain_requested_at = None
node.drain_reason = None
session.add(node)
session.flush()
return node
def stop_runtime_node(
session: Session,
identity: RuntimeIdentity,
*,
now: datetime | None = None,
) -> bool:
node = _locked_node(session, identity.installation_id, identity.node_id)
if node is None or node.incarnation != identity.incarnation:
return False
observed_at = _as_utc(now or utcnow())
node.state = RuntimeNodeState.STOPPED.value
node.stopped_at = observed_at
node.last_heartbeat_at = observed_at
session.add(node)
session.flush()
return True
def runtime_node_is_draining(
session: Session,
identity: RuntimeIdentity,
) -> bool:
node = session.execute(
select(RuntimeNode).where(
RuntimeNode.installation_id == identity.installation_id,
RuntimeNode.node_id == identity.node_id,
RuntimeNode.incarnation == identity.incarnation,
)
).scalar_one_or_none()
return node is not None and node.state == RuntimeNodeState.DRAINING.value
def list_runtime_nodes(
session: Session,
*,
installation_id: str,
stale_after_seconds: int,
now: datetime | None = None,
) -> list[dict[str, Any]]:
observed_at = _as_utc(now or utcnow())
stale_before = observed_at - timedelta(seconds=max(1, stale_after_seconds))
nodes = session.execute(
select(RuntimeNode)
.where(RuntimeNode.installation_id == installation_id)
.order_by(RuntimeNode.role, RuntimeNode.node_id)
).scalars().all()
return [
{
"node_id": node.node_id,
"incarnation": node.incarnation,
"role": node.role,
"software_version": node.software_version,
"composition_hash": node.composition_hash,
"queues": list(node.queues or []),
"state": node.state,
"started_at": _iso(node.started_at),
"last_heartbeat_at": _iso(node.last_heartbeat_at),
"drain_requested_at": _iso(node.drain_requested_at),
"drain_reason": node.drain_reason,
"stopped_at": _iso(node.stopped_at),
"stale": (
node.state != RuntimeNodeState.STOPPED.value
and _as_utc(node.last_heartbeat_at) < stale_before
),
"metadata": dict(node.metadata_ or {}),
}
for node in nodes
]
def acquire_lease(
session: Session,
*,
installation_id: str,
resource_key: str,
holder_node_id: str,
holder_incarnation: str,
ttl_seconds: int,
metadata: dict[str, Any] | None = None,
now: datetime | None = None,
) -> LeaseClaim | None:
if ttl_seconds < 1:
raise ValueError("Lease TTL must be at least one second")
observed_at = _as_utc(now or utcnow())
_insert_lease_placeholder(session, installation_id, resource_key, observed_at)
lease = _locked_lease(session, installation_id, resource_key)
if lease is None: # pragma: no cover - guarded by insert/upsert
raise RuntimeCoordinationError("Distributed lease was not persisted")
same_holder = (
lease.holder_node_id == holder_node_id
and lease.holder_incarnation == holder_incarnation
)
expired = _as_utc(lease.expires_at) <= observed_at
if lease.holder_node_id is not None and not same_holder and not expired:
return None
if not same_holder or expired:
lease.fencing_token = int(lease.fencing_token or 0) + 1
lease.acquired_at = observed_at
lease.holder_node_id = holder_node_id
lease.holder_incarnation = holder_incarnation
lease.renewed_at = observed_at
lease.expires_at = observed_at + timedelta(seconds=ttl_seconds)
lease.metadata_ = dict(metadata or {})
session.add(lease)
session.flush()
return _lease_claim(lease)
def renew_lease(
session: Session,
claim: LeaseClaim,
*,
ttl_seconds: int,
now: datetime | None = None,
) -> LeaseClaim:
if ttl_seconds < 1:
raise ValueError("Lease TTL must be at least one second")
observed_at = _as_utc(now or utcnow())
lease = _locked_lease(session, claim.installation_id, claim.resource_key)
_assert_matching_fence(lease, claim, observed_at=observed_at)
lease.renewed_at = observed_at
lease.expires_at = observed_at + timedelta(seconds=ttl_seconds)
session.add(lease)
session.flush()
return _lease_claim(lease)
def release_lease(
session: Session,
claim: LeaseClaim,
*,
now: datetime | None = None,
) -> None:
observed_at = _as_utc(now or utcnow())
lease = _locked_lease(session, claim.installation_id, claim.resource_key)
_assert_matching_fence(lease, claim, observed_at=observed_at)
lease.holder_node_id = None
lease.holder_incarnation = None
lease.renewed_at = observed_at
lease.expires_at = observed_at
session.add(lease)
session.flush()
def assert_lease_fence(
session: Session,
claim: LeaseClaim,
*,
now: datetime | None = None,
) -> None:
lease = _locked_lease(session, claim.installation_id, claim.resource_key)
_assert_matching_fence(
lease,
claim,
observed_at=_as_utc(now or utcnow()),
)
def _insert_node_placeholder(
session: Session,
identity: RuntimeIdentity,
observed_at: datetime,
) -> None:
values = {
"id": str(uuid4()),
"installation_id": identity.installation_id,
"node_id": identity.node_id,
"incarnation": identity.incarnation,
"role": identity.role,
"software_version": identity.software_version,
"composition_hash": identity.composition_hash,
"queues": list(identity.queues),
"state": RuntimeNodeState.ACTIVE.value,
"started_at": observed_at,
"last_heartbeat_at": observed_at,
"metadata": {},
"created_at": observed_at,
"updated_at": observed_at,
}
_insert_do_nothing(
session,
RuntimeNode.__table__,
values,
conflict_columns=("installation_id", "node_id"),
)
def _insert_lease_placeholder(
session: Session,
installation_id: str,
resource_key: str,
observed_at: datetime,
) -> None:
values = {
"id": str(uuid4()),
"installation_id": installation_id,
"resource_key": resource_key,
"fencing_token": 0,
"expires_at": observed_at,
"metadata": {},
"created_at": observed_at,
"updated_at": observed_at,
}
_insert_do_nothing(
session,
DistributedLease.__table__,
values,
conflict_columns=("installation_id", "resource_key"),
)
def _insert_do_nothing(
session: Session,
table: Any,
values: dict[str, Any],
*,
conflict_columns: tuple[str, ...],
) -> None:
dialect = session.get_bind().dialect.name
if dialect == "postgresql":
from sqlalchemy.dialects.postgresql import insert
statement = insert(table).values(**values).on_conflict_do_nothing(
index_elements=list(conflict_columns)
)
elif dialect == "sqlite":
from sqlalchemy.dialects.sqlite import insert
statement = insert(table).values(**values).on_conflict_do_nothing(
index_elements=list(conflict_columns)
)
else: # pragma: no cover - GovOPlaN supports PostgreSQL and dev SQLite
raise RuntimeCoordinationError(
f"Distributed coordination is unsupported on {dialect!r}"
)
session.execute(statement)
session.flush()
def _locked_node(
session: Session,
installation_id: str,
node_id: str,
) -> RuntimeNode | None:
return session.execute(
select(RuntimeNode)
.where(
RuntimeNode.installation_id == installation_id,
RuntimeNode.node_id == node_id,
)
.with_for_update()
).scalar_one_or_none()
def _locked_lease(
session: Session,
installation_id: str,
resource_key: str,
) -> DistributedLease | None:
return session.execute(
select(DistributedLease)
.where(
DistributedLease.installation_id == installation_id,
DistributedLease.resource_key == resource_key,
)
.with_for_update()
).scalar_one_or_none()
def _assert_matching_fence(
lease: DistributedLease | None,
claim: LeaseClaim,
*,
observed_at: datetime,
) -> None:
if (
lease is None
or lease.holder_node_id != claim.holder_node_id
or lease.holder_incarnation != claim.holder_incarnation
or int(lease.fencing_token) != claim.fencing_token
or _as_utc(lease.expires_at) <= observed_at
):
raise StaleFence(
f"Lease fence is stale for resource {claim.resource_key!r}"
)
def _lease_claim(lease: DistributedLease) -> LeaseClaim:
if lease.holder_node_id is None or lease.holder_incarnation is None:
raise LeaseUnavailable("Distributed lease has no active holder")
return LeaseClaim(
installation_id=lease.installation_id,
resource_key=lease.resource_key,
holder_node_id=lease.holder_node_id,
holder_incarnation=lease.holder_incarnation,
fencing_token=int(lease.fencing_token),
expires_at=_as_utc(lease.expires_at),
)
def _as_utc(value: datetime) -> datetime:
if value.tzinfo is None:
return value.replace(tzinfo=timezone.utc)
return value.astimezone(timezone.utc)
def _iso(value: datetime | None) -> str | None:
return _as_utc(value).isoformat() if value is not None else None
__all__ = [
"DistributedLease",
"LeaseClaim",
"LeaseUnavailable",
"RuntimeCoordinationError",
"RuntimeIdentity",
"RuntimeNode",
"RuntimeNodeState",
"StaleFence",
"acquire_lease",
"assert_lease_fence",
"cancel_runtime_node_drain",
"heartbeat_runtime_node",
"list_runtime_nodes",
"register_runtime_node",
"release_lease",
"renew_lease",
"request_runtime_node_drain",
"runtime_identity",
"runtime_node_is_draining",
"stop_runtime_node",
]