Release v0.1.7
All checks were successful
Dependency Audit / dependency-audit (push) Successful in 1m35s
All checks were successful
Dependency Audit / dependency-audit (push) Successful in 1m35s
This commit is contained in:
@@ -1,9 +1,17 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from govoplan_core.db.bootstrap import bootstrap_dev_data
|
||||
from govoplan_core.db.migrations import migrate_database
|
||||
from govoplan_core.db.migrations import (
|
||||
ModuleMigrationTaskExecutionError,
|
||||
POST_MIGRATION_TASK_PHASES,
|
||||
PRE_MIGRATION_TASK_PHASES,
|
||||
migrate_database,
|
||||
run_registered_module_migration_tasks,
|
||||
)
|
||||
from govoplan_core.db.session import configure_database, get_database
|
||||
from govoplan_core.settings import settings
|
||||
|
||||
@@ -11,12 +19,43 @@ from govoplan_core.settings import settings
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Initialize the GovOPlaN database")
|
||||
parser.add_argument("--database-url", default=settings.database_url, help="Database URL to migrate")
|
||||
parser.add_argument("--enabled-module", action="append", default=[], help="Target enabled module id used to discover module migrations; may be repeated.")
|
||||
parser.add_argument("--migration-module", action="append", default=[], help="Module id whose migration heads should be upgraded in this order before final heads.")
|
||||
parser.add_argument("--migration-task-record-output", type=Path, help="Write executed module migration task records to this JSON file.")
|
||||
parser.add_argument("--with-dev-data", action="store_true", help="Create default tenant/user/roles and a development API key")
|
||||
parser.add_argument("--dev-api-key", default=settings.dev_bootstrap_api_key, help="Development API key secret to create")
|
||||
args = parser.parse_args()
|
||||
|
||||
configure_database(args.database_url)
|
||||
migration = migrate_database(database_url=args.database_url)
|
||||
enabled_modules = tuple(args.enabled_module) if args.enabled_module else None
|
||||
migration_order = tuple(args.migration_module) if args.migration_module else None
|
||||
task_records: list[dict[str, object]] = []
|
||||
try:
|
||||
_run_migration_tasks(
|
||||
task_records,
|
||||
database_url=args.database_url,
|
||||
enabled_modules=enabled_modules,
|
||||
migration_order=migration_order,
|
||||
phases=PRE_MIGRATION_TASK_PHASES,
|
||||
)
|
||||
migration = migrate_database(
|
||||
database_url=args.database_url,
|
||||
enabled_modules=enabled_modules,
|
||||
migration_module_order=migration_order,
|
||||
)
|
||||
_run_migration_tasks(
|
||||
task_records,
|
||||
database_url=args.database_url,
|
||||
enabled_modules=enabled_modules,
|
||||
migration_order=migration_order,
|
||||
phases=POST_MIGRATION_TASK_PHASES,
|
||||
)
|
||||
finally:
|
||||
if args.migration_task_record_output:
|
||||
args.migration_task_record_output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.migration_task_record_output.write_text(json.dumps(task_records, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
if task_records:
|
||||
print(f"Executed {len(task_records)} module migration task(s).")
|
||||
if migration.reconciled_revision:
|
||||
print(f"Reconciled legacy database marker to {migration.reconciled_revision}.")
|
||||
print(f"Database schema upgraded to {migration.current_revision}.")
|
||||
@@ -33,5 +72,26 @@ def main() -> None:
|
||||
print("Development API key already exists or was not requested.")
|
||||
|
||||
|
||||
def _run_migration_tasks(
|
||||
target: list[dict[str, object]],
|
||||
*,
|
||||
database_url: str,
|
||||
enabled_modules: tuple[str, ...] | None,
|
||||
migration_order: tuple[str, ...] | None,
|
||||
phases: tuple[str, ...],
|
||||
) -> None:
|
||||
try:
|
||||
records = run_registered_module_migration_tasks(
|
||||
database_url=database_url,
|
||||
enabled_modules=enabled_modules,
|
||||
migration_module_order=migration_order,
|
||||
phases=phases,
|
||||
)
|
||||
except ModuleMigrationTaskExecutionError as exc:
|
||||
target.extend(exc.records)
|
||||
raise
|
||||
target.extend(records)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
@@ -12,6 +12,29 @@ from govoplan_core.core.registry import PlatformRegistry
|
||||
class MigrationMetadataPlan:
|
||||
metadata: tuple[MetaData, ...]
|
||||
script_locations: tuple[str, ...]
|
||||
modules: tuple[ModuleMigrationMetadata, ...] = ()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ModuleMigrationMetadata:
|
||||
module_id: str
|
||||
script_location: str | None = None
|
||||
has_metadata: bool = False
|
||||
migration_after: tuple[str, ...] = ()
|
||||
migration_before: tuple[str, ...] = ()
|
||||
migration_tasks: tuple[ModuleMigrationTaskMetadata, ...] = ()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ModuleMigrationTaskMetadata:
|
||||
task_id: str
|
||||
phase: str
|
||||
summary: str
|
||||
task_version: str = "1"
|
||||
safety: str = "automatic"
|
||||
idempotent: bool = True
|
||||
timeout_seconds: int | None = None
|
||||
has_executor: bool = False
|
||||
|
||||
|
||||
def migration_metadata_plan(registry: PlatformRegistry, *, extra_metadata: Iterable[MetaData] = ()) -> MigrationMetadataPlan:
|
||||
@@ -28,4 +51,28 @@ def migration_metadata_plan(registry: PlatformRegistry, *, extra_metadata: Itera
|
||||
metadata.append(spec.metadata)
|
||||
if spec.script_location:
|
||||
script_locations.append(spec.script_location)
|
||||
return MigrationMetadataPlan(metadata=tuple(metadata), script_locations=tuple(script_locations))
|
||||
modules = tuple(
|
||||
ModuleMigrationMetadata(
|
||||
module_id=manifest.id,
|
||||
script_location=manifest.migration_spec.script_location,
|
||||
has_metadata=isinstance(manifest.migration_spec.metadata, MetaData),
|
||||
migration_after=tuple(manifest.migration_spec.migration_after),
|
||||
migration_before=tuple(manifest.migration_spec.migration_before),
|
||||
migration_tasks=tuple(
|
||||
ModuleMigrationTaskMetadata(
|
||||
task_id=task.task_id,
|
||||
phase=task.phase,
|
||||
summary=task.summary,
|
||||
task_version=task.task_version,
|
||||
safety=task.safety,
|
||||
idempotent=task.idempotent,
|
||||
timeout_seconds=task.timeout_seconds,
|
||||
has_executor=task.executor is not None,
|
||||
)
|
||||
for task in manifest.migration_spec.migration_tasks
|
||||
),
|
||||
)
|
||||
for manifest in registry.manifests()
|
||||
if manifest.migration_spec is not None
|
||||
)
|
||||
return MigrationMetadataPlan(metadata=tuple(metadata), script_locations=tuple(script_locations), modules=modules)
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -20,6 +20,12 @@ from govoplan_core.core.versioning import format_version_range, version_range_is
|
||||
|
||||
_INTERFACE_NAME_RE = re.compile(r"^[a-z][a-z0-9_]*(?:\.[a-z][a-z0-9_]*)+$")
|
||||
CATALOG_MIGRATION_SAFETY = ("automatic", "requires_review", "forward_only", "destructive")
|
||||
CATALOG_MIGRATION_TASK_PHASES = (
|
||||
"pre_migration_check",
|
||||
"pre_migration_prepare",
|
||||
"post_migration_backfill",
|
||||
"post_migration_verify",
|
||||
)
|
||||
|
||||
|
||||
def module_package_catalog(
|
||||
@@ -611,6 +617,17 @@ def _normalize_catalog_item(value: Any) -> dict[str, object]:
|
||||
"optional_dependencies": _string_list(value.get("optional_dependencies")),
|
||||
"migration_safety": _catalog_migration_safety(value.get("migration_safety"), module_id=module_id),
|
||||
"migration_notes": _optional_str(value, "migration_notes"),
|
||||
"migration_after": _string_list(value.get("migration_after")),
|
||||
"migration_before": _string_list(value.get("migration_before")),
|
||||
"migration_tasks": _normalize_catalog_migration_tasks(value.get("migration_tasks"), module_id=module_id),
|
||||
"current_version_min": _optional_str(value, "current_version_min"),
|
||||
"current_version_max_exclusive": _optional_str(value, "current_version_max_exclusive"),
|
||||
"bridge_release": _optional_bool(value, "bridge_release"),
|
||||
"bridge_notes": _optional_str(value, "bridge_notes"),
|
||||
"allow_downgrade": _optional_bool(value, "allow_downgrade"),
|
||||
"allow_same_version": _optional_bool(value, "allow_same_version"),
|
||||
"recovery_tested": _optional_bool(value, "recovery_tested"),
|
||||
"recovery_notes": _optional_str(value, "recovery_notes"),
|
||||
"python_package": _optional_str(value, "python_package"),
|
||||
"python_ref": _optional_str(value, "python_ref"),
|
||||
"webui_package": _optional_str(value, "webui_package"),
|
||||
@@ -621,6 +638,15 @@ def _normalize_catalog_item(value: Any) -> dict[str, object]:
|
||||
"notes": _optional_str(value, "notes"),
|
||||
"tags": _string_list(value.get("tags")),
|
||||
}
|
||||
if not version_range_is_valid(
|
||||
version_min=item["current_version_min"] if isinstance(item["current_version_min"], str) else None,
|
||||
version_max_exclusive=item["current_version_max_exclusive"] if isinstance(item["current_version_max_exclusive"], str) else None,
|
||||
):
|
||||
version_range = format_version_range(
|
||||
version_min=item["current_version_min"] if isinstance(item["current_version_min"], str) else None,
|
||||
version_max_exclusive=item["current_version_max_exclusive"] if isinstance(item["current_version_max_exclusive"], str) else None,
|
||||
)
|
||||
raise ValueError(f"Module package catalog entry {module_id!r} has invalid current-version range {version_range!r}.")
|
||||
artifact_integrity = _normalize_artifact_integrity(value.get("artifact_integrity"))
|
||||
if artifact_integrity:
|
||||
item["artifact_integrity"] = artifact_integrity
|
||||
@@ -637,6 +663,66 @@ def _catalog_migration_safety(value: Any, *, module_id: str) -> str:
|
||||
return safety
|
||||
|
||||
|
||||
def _normalize_catalog_migration_tasks(value: Any, *, module_id: str) -> list[dict[str, object]]:
|
||||
if value is None:
|
||||
return []
|
||||
if not isinstance(value, list):
|
||||
raise ValueError(f"Module package catalog migration_tasks for {module_id!r} must be a list.")
|
||||
normalized: list[dict[str, object]] = []
|
||||
seen: set[str] = set()
|
||||
for raw in value:
|
||||
if not isinstance(raw, dict):
|
||||
raise ValueError(f"Module package catalog migration_tasks entries for {module_id!r} must be objects.")
|
||||
task_id = _catalog_task_required_str(raw, "task_id", module_id=module_id)
|
||||
if task_id in seen:
|
||||
raise ValueError(f"Module package catalog entry {module_id!r} declares migration task {task_id!r} more than once.")
|
||||
seen.add(task_id)
|
||||
phase = _catalog_task_required_str(raw, "phase", module_id=module_id)
|
||||
if phase not in CATALOG_MIGRATION_TASK_PHASES:
|
||||
allowed = ", ".join(CATALOG_MIGRATION_TASK_PHASES)
|
||||
raise ValueError(f"Unsupported migration task phase for {module_id!r}/{task_id!r}: {phase!r}; expected one of {allowed}.")
|
||||
summary = _catalog_task_required_str(raw, "summary", module_id=module_id)
|
||||
safety = _catalog_migration_safety(raw.get("safety"), module_id=module_id)
|
||||
item: dict[str, object] = {
|
||||
"task_id": task_id,
|
||||
"phase": phase,
|
||||
"summary": summary,
|
||||
"task_version": _optional_str(raw, "task_version") or "1",
|
||||
"safety": safety,
|
||||
"idempotent": _catalog_optional_bool_default(raw, "idempotent", default=True),
|
||||
}
|
||||
timeout_seconds = _catalog_optional_positive_int(raw, "timeout_seconds", module_id=module_id, task_id=task_id)
|
||||
if timeout_seconds is not None:
|
||||
item["timeout_seconds"] = timeout_seconds
|
||||
normalized.append(item)
|
||||
return normalized
|
||||
|
||||
|
||||
def _catalog_task_required_str(value: dict[str, Any], key: str, *, module_id: str) -> str:
|
||||
item = _optional_str(value, key)
|
||||
if not item:
|
||||
raise ValueError(f"Module package catalog migration task for {module_id!r} is missing {key!r}.")
|
||||
return item
|
||||
|
||||
|
||||
def _catalog_optional_bool_default(value: dict[str, Any], key: str, *, default: bool) -> bool:
|
||||
if key not in value:
|
||||
return default
|
||||
return _optional_bool(value, key)
|
||||
|
||||
|
||||
def _catalog_optional_positive_int(value: dict[str, Any], key: str, *, module_id: str, task_id: str) -> int | None:
|
||||
if value.get(key) is None:
|
||||
return None
|
||||
try:
|
||||
integer = int(value[key])
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise ValueError(f"Module package catalog migration task {module_id!r}/{task_id!r} has invalid {key!r}.") from exc
|
||||
if integer <= 0:
|
||||
raise ValueError(f"Module package catalog migration task {module_id!r}/{task_id!r} requires positive {key!r}.")
|
||||
return integer
|
||||
|
||||
|
||||
def _required_str(value: dict[str, Any], key: str) -> str:
|
||||
item = _optional_str(value, key)
|
||||
if not item:
|
||||
|
||||
@@ -13,6 +13,14 @@ SUPPORTED_FRONTEND_ASSET_MANIFEST_CONTRACT_VERSION = "1"
|
||||
|
||||
PermissionLevel = Literal["system", "tenant"]
|
||||
SubjectType = Literal["account", "membership", "group", "service_account", "tenant"]
|
||||
MigrationTaskPhase = Literal[
|
||||
"pre_migration_check",
|
||||
"pre_migration_prepare",
|
||||
"post_migration_backfill",
|
||||
"post_migration_verify",
|
||||
]
|
||||
MigrationTaskSafety = Literal["automatic", "requires_review", "forward_only", "destructive"]
|
||||
MigrationTaskStatus = Literal["ok", "warning", "blocked", "skipped"]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
@@ -91,11 +99,49 @@ MigrationRetirementExecutor = Callable[[object, str], None]
|
||||
MigrationRetirementProvider = Callable[[object | None, str], MigrationRetirementPlan]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ModuleMigrationTaskContext:
|
||||
module_id: str
|
||||
task_id: str
|
||||
phase: MigrationTaskPhase
|
||||
database_url: str
|
||||
target_version: str | None = None
|
||||
session: object | None = None
|
||||
dry_run: bool = False
|
||||
metadata: Mapping[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ModuleMigrationTaskResult:
|
||||
status: MigrationTaskStatus = "ok"
|
||||
message: str | None = None
|
||||
details: Mapping[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
ModuleMigrationTaskExecutor = Callable[[ModuleMigrationTaskContext], ModuleMigrationTaskResult | None]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ModuleMigrationTask:
|
||||
task_id: str
|
||||
phase: MigrationTaskPhase
|
||||
summary: str
|
||||
task_version: str = "1"
|
||||
safety: MigrationTaskSafety = "automatic"
|
||||
idempotent: bool = True
|
||||
timeout_seconds: int | None = None
|
||||
executor: ModuleMigrationTaskExecutor | None = None
|
||||
metadata: Mapping[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class MigrationSpec:
|
||||
module_id: str
|
||||
metadata: object | None = None
|
||||
script_location: str | None = None
|
||||
migration_after: tuple[str, ...] = ()
|
||||
migration_before: tuple[str, ...] = ()
|
||||
migration_tasks: tuple[ModuleMigrationTask, ...] = ()
|
||||
retirement_supported: bool = False
|
||||
retirement_provider: MigrationRetirementProvider | None = None
|
||||
retirement_notes: str | None = None
|
||||
@@ -215,7 +261,28 @@ class ResourceAclProvider(Protocol):
|
||||
|
||||
|
||||
TenantSummaryProvider = Callable[[object, str], Mapping[str, int]]
|
||||
DeleteVetoProvider = Callable[[object, str, str], None]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DeleteVetoIssue:
|
||||
severity: Literal["blocker", "warning", "info"]
|
||||
code: str
|
||||
message: str
|
||||
module_id: str | None = None
|
||||
details: Mapping[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
DeleteVetoProviderResult = DeleteVetoIssue | Iterable[DeleteVetoIssue] | None
|
||||
DeleteVetoProvider = Callable[[object, str, str], DeleteVetoProviderResult]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DeleteVetoProviderRegistration:
|
||||
module_id: str
|
||||
resource_type: str
|
||||
provider: DeleteVetoProvider
|
||||
|
||||
|
||||
RouteFactory = Callable[[ModuleContext], "APIRouter"]
|
||||
CapabilityFactory = Callable[[ModuleContext], object]
|
||||
DocumentationProvider = Callable[[DocumentationContext], Iterable[DocumentationTopic]]
|
||||
|
||||
@@ -3,11 +3,13 @@ from __future__ import annotations
|
||||
import re
|
||||
from collections import defaultdict, deque
|
||||
from collections.abc import Iterable, Mapping
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, replace
|
||||
|
||||
from govoplan_core.core.modules import (
|
||||
CapabilityFactory,
|
||||
DeleteVetoIssue,
|
||||
DeleteVetoProvider,
|
||||
DeleteVetoProviderRegistration,
|
||||
ModuleInterfaceProvider,
|
||||
ModuleInterfaceRequirement,
|
||||
ModuleContext,
|
||||
@@ -45,7 +47,7 @@ class PlatformRegistry:
|
||||
def __init__(self) -> None:
|
||||
self._manifests: dict[str, ModuleManifest] = {}
|
||||
self._tenant_summary_providers: dict[str, TenantSummaryProvider] = {}
|
||||
self._delete_veto_providers: dict[str, list[DeleteVetoProvider]] = defaultdict(list)
|
||||
self._delete_veto_providers: dict[str, list[DeleteVetoProviderRegistration]] = defaultdict(list)
|
||||
self._capability_factories: dict[str, CapabilityFactory] = {}
|
||||
self._capabilities: dict[str, object] = {}
|
||||
self._capability_context: ModuleContext | None = None
|
||||
@@ -58,7 +60,7 @@ class PlatformRegistry:
|
||||
self.register_tenant_summary_provider(manifest.id, provider)
|
||||
for resource_type, providers in manifest.delete_veto_providers.items():
|
||||
for provider in providers:
|
||||
self.register_delete_veto(resource_type, provider)
|
||||
self.register_delete_veto(manifest.id, resource_type, provider)
|
||||
for name, factory in manifest.capability_factories.items():
|
||||
self.register_capability_factory(manifest.id, name, factory)
|
||||
return manifest
|
||||
@@ -148,12 +150,39 @@ class PlatformRegistry:
|
||||
def tenant_summary_providers(self) -> Mapping[str, TenantSummaryProvider]:
|
||||
return dict(self._tenant_summary_providers)
|
||||
|
||||
def register_delete_veto(self, resource_type: str, provider: DeleteVetoProvider) -> None:
|
||||
self._delete_veto_providers[resource_type].append(provider)
|
||||
def register_delete_veto(self, module_id: str, resource_type: str, provider: DeleteVetoProvider) -> None:
|
||||
self._delete_veto_providers[resource_type].append(DeleteVetoProviderRegistration(
|
||||
module_id=module_id,
|
||||
resource_type=resource_type,
|
||||
provider=provider,
|
||||
))
|
||||
|
||||
def delete_veto_providers(self, resource_type: str) -> tuple[DeleteVetoProvider, ...]:
|
||||
return tuple(registration.provider for registration in self._delete_veto_providers.get(resource_type, ()))
|
||||
|
||||
def delete_veto_provider_registrations(self, resource_type: str) -> tuple[DeleteVetoProviderRegistration, ...]:
|
||||
return tuple(self._delete_veto_providers.get(resource_type, ()))
|
||||
|
||||
def collect_delete_veto_issues(self, resource_type: str, session: object, tenant_id: str, resource_id: str) -> tuple[DeleteVetoIssue, ...]:
|
||||
issues: list[DeleteVetoIssue] = []
|
||||
for registration in self.delete_veto_provider_registrations(resource_type):
|
||||
try:
|
||||
result = registration.provider(session, tenant_id, resource_id)
|
||||
except Exception as exc:
|
||||
issues.append(DeleteVetoIssue(
|
||||
severity="blocker",
|
||||
code="module_veto",
|
||||
message=str(exc),
|
||||
module_id=registration.module_id,
|
||||
details={
|
||||
"resource_type": resource_type,
|
||||
"resource_id": resource_id,
|
||||
},
|
||||
))
|
||||
continue
|
||||
issues.extend(_normalize_delete_veto_result(result, registration=registration, resource_id=resource_id))
|
||||
return tuple(issues)
|
||||
|
||||
def validate(self) -> RegistrySnapshot:
|
||||
ordered = tuple(self._topologically_sorted())
|
||||
available_capabilities = set(self._capability_factories)
|
||||
@@ -224,6 +253,52 @@ class PlatformRegistry:
|
||||
return (self._manifests[module_id] for module_id in ordered)
|
||||
|
||||
|
||||
def _normalize_delete_veto_result(
|
||||
result: object,
|
||||
*,
|
||||
registration: DeleteVetoProviderRegistration,
|
||||
resource_id: str,
|
||||
) -> tuple[DeleteVetoIssue, ...]:
|
||||
if result is None:
|
||||
return ()
|
||||
if isinstance(result, DeleteVetoIssue):
|
||||
return (_attribute_delete_veto_issue(result, registration=registration, resource_id=resource_id),)
|
||||
if isinstance(result, str):
|
||||
return (DeleteVetoIssue(
|
||||
severity="blocker",
|
||||
code="module_veto",
|
||||
message=result,
|
||||
module_id=registration.module_id,
|
||||
details={
|
||||
"resource_type": registration.resource_type,
|
||||
"resource_id": resource_id,
|
||||
},
|
||||
),)
|
||||
if isinstance(result, Iterable):
|
||||
return tuple(
|
||||
_attribute_delete_veto_issue(issue, registration=registration, resource_id=resource_id)
|
||||
for issue in result
|
||||
if isinstance(issue, DeleteVetoIssue)
|
||||
)
|
||||
return ()
|
||||
|
||||
|
||||
def _attribute_delete_veto_issue(
|
||||
issue: DeleteVetoIssue,
|
||||
*,
|
||||
registration: DeleteVetoProviderRegistration,
|
||||
resource_id: str,
|
||||
) -> DeleteVetoIssue:
|
||||
details = {
|
||||
"resource_type": registration.resource_type,
|
||||
"resource_id": resource_id,
|
||||
**dict(issue.details),
|
||||
}
|
||||
if issue.module_id == registration.module_id and issue.details == details:
|
||||
return issue
|
||||
return replace(issue, module_id=issue.module_id or registration.module_id, details=details)
|
||||
|
||||
|
||||
def _validate_manifest_shape(manifest: ModuleManifest) -> None:
|
||||
if not _MODULE_ID_RE.match(manifest.id):
|
||||
raise RegistryError(f"Module manifest id must match {_MODULE_ID_RE.pattern}: {manifest.id!r}")
|
||||
|
||||
@@ -1,19 +1,24 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from alembic import command
|
||||
from alembic.config import Config
|
||||
from alembic.runtime.migration import MigrationContext
|
||||
from alembic.script import ScriptDirectory
|
||||
from sqlalchemy import create_engine, inspect, text
|
||||
|
||||
from govoplan_core.core.migrations import MigrationMetadataPlan, migration_metadata_plan
|
||||
from govoplan_core.core import change_sequence as core_change_sequence_models # noqa: F401 - populate core metadata
|
||||
from govoplan_core.core.change_sequence import ChangeSequenceEntry, ChangeSequenceRetentionFloor
|
||||
from govoplan_core.core.module_management import load_startup_enabled_modules, startup_candidate_module_ids
|
||||
from govoplan_core.db.session import configure_database
|
||||
from govoplan_core.core.modules import ModuleMigrationTaskContext, ModuleMigrationTaskResult
|
||||
from govoplan_core.db.session import configure_database, get_database
|
||||
from govoplan_core.server.default_config import get_server_config
|
||||
from govoplan_core.server.config import ManifestFactory
|
||||
from govoplan_core.server.registry import available_module_manifests, build_platform_registry
|
||||
@@ -33,6 +38,9 @@ REVISION_CORE_CHANGE_SEQUENCE = "3f4a5b6c7d8e"
|
||||
REVISION_HIERARCHICAL_SETTINGS = "f5a6b7c8d9e0"
|
||||
LEGACY_SCOPE_TABLE = "tenancy_tenants"
|
||||
CORE_SCOPE_TABLE = "core_scopes"
|
||||
PRE_MIGRATION_TASK_PHASES = ("pre_migration_check", "pre_migration_prepare")
|
||||
POST_MIGRATION_TASK_PHASES = ("post_migration_backfill", "post_migration_verify")
|
||||
MIGRATION_TASK_PHASES = (*PRE_MIGRATION_TASK_PHASES, *POST_MIGRATION_TASK_PHASES)
|
||||
|
||||
_NAMESPACE_TABLE_RENAMES = (
|
||||
("tenants", "tenancy_tenants"),
|
||||
@@ -160,12 +168,31 @@ class MigrationResult:
|
||||
current_revision: str | None
|
||||
|
||||
|
||||
class ModuleMigrationTaskExecutionError(RuntimeError):
|
||||
def __init__(self, message: str, *, records: tuple[dict[str, object], ...]) -> None:
|
||||
super().__init__(message)
|
||||
self.records = records
|
||||
|
||||
|
||||
def registered_module_migration_plan(
|
||||
database_url: str | None = None,
|
||||
*,
|
||||
enabled_modules: tuple[str, ...] | list[str] | None = None,
|
||||
manifest_factories: tuple[ManifestFactory, ...] = (),
|
||||
) -> MigrationMetadataPlan:
|
||||
return migration_metadata_plan(_registered_module_registry(
|
||||
database_url=database_url,
|
||||
enabled_modules=enabled_modules,
|
||||
manifest_factories=manifest_factories,
|
||||
))
|
||||
|
||||
|
||||
def _registered_module_registry(
|
||||
*,
|
||||
database_url: str | None = None,
|
||||
enabled_modules: tuple[str, ...] | list[str] | None = None,
|
||||
manifest_factories: tuple[ManifestFactory, ...] = (),
|
||||
):
|
||||
server_config = get_server_config()
|
||||
active_database_url = database_url or settings.database_url
|
||||
if active_database_url:
|
||||
@@ -187,7 +214,130 @@ def registered_module_migration_plan(
|
||||
active_enabled_modules,
|
||||
manifest_factories=active_manifest_factories,
|
||||
)
|
||||
return migration_metadata_plan(registry)
|
||||
return registry
|
||||
|
||||
|
||||
def run_registered_module_migration_tasks(
|
||||
*,
|
||||
database_url: str | None = None,
|
||||
enabled_modules: tuple[str, ...] | list[str] | None = None,
|
||||
migration_module_order: tuple[str, ...] | list[str] | None = None,
|
||||
phases: tuple[str, ...] | list[str] = MIGRATION_TASK_PHASES,
|
||||
dry_run: bool = False,
|
||||
manifest_factories: tuple[ManifestFactory, ...] = (),
|
||||
) -> tuple[dict[str, object], ...]:
|
||||
active_phases = tuple(dict.fromkeys(str(item).strip() for item in phases if str(item).strip()))
|
||||
invalid_phases = tuple(phase for phase in active_phases if phase not in MIGRATION_TASK_PHASES)
|
||||
if invalid_phases:
|
||||
raise ValueError("Unsupported module migration task phase(s): " + ", ".join(invalid_phases))
|
||||
url = database_url or settings.database_url
|
||||
registry = _registered_module_registry(
|
||||
database_url=url,
|
||||
enabled_modules=enabled_modules,
|
||||
manifest_factories=manifest_factories,
|
||||
)
|
||||
manifests = {manifest.id: manifest for manifest in registry.manifests()}
|
||||
ordered_ids = tuple(dict.fromkeys([
|
||||
*(str(item).strip() for item in (migration_module_order or ()) if str(item).strip()),
|
||||
*manifests.keys(),
|
||||
]))
|
||||
records: list[dict[str, object]] = []
|
||||
database = get_database()
|
||||
with database.SessionLocal() as session:
|
||||
for phase in active_phases:
|
||||
for module_id in ordered_ids:
|
||||
manifest = manifests.get(module_id)
|
||||
if manifest is None or manifest.migration_spec is None:
|
||||
continue
|
||||
for task in manifest.migration_spec.migration_tasks:
|
||||
if task.phase != phase:
|
||||
continue
|
||||
record: dict[str, object] = {
|
||||
"module_id": manifest.id,
|
||||
"task_id": task.task_id,
|
||||
"phase": task.phase,
|
||||
"summary": task.summary,
|
||||
"task_version": task.task_version,
|
||||
"safety": task.safety,
|
||||
"idempotent": task.idempotent,
|
||||
"dry_run": dry_run,
|
||||
}
|
||||
if task.timeout_seconds is not None:
|
||||
record["timeout_seconds"] = task.timeout_seconds
|
||||
if not task.idempotent:
|
||||
record.update({"status": "blocked", "message": "Task is not idempotent."})
|
||||
records.append(record)
|
||||
raise ModuleMigrationTaskExecutionError(
|
||||
f"Module migration task {manifest.id}/{task.task_id} is not idempotent.",
|
||||
records=tuple(records),
|
||||
)
|
||||
if dry_run:
|
||||
record.update({"status": "skipped", "message": "Dry run; executor was not called."})
|
||||
records.append(record)
|
||||
continue
|
||||
if task.executor is None:
|
||||
record.update({"status": "blocked", "message": "Task has no executor."})
|
||||
records.append(record)
|
||||
raise ModuleMigrationTaskExecutionError(
|
||||
f"Module migration task {manifest.id}/{task.task_id} has no executor.",
|
||||
records=tuple(records),
|
||||
)
|
||||
context = ModuleMigrationTaskContext(
|
||||
module_id=manifest.id,
|
||||
task_id=task.task_id,
|
||||
phase=task.phase,
|
||||
database_url=url,
|
||||
target_version=manifest.version,
|
||||
session=session,
|
||||
dry_run=dry_run,
|
||||
metadata=task.metadata,
|
||||
)
|
||||
try:
|
||||
result = task.executor(context)
|
||||
normalized = _normalize_migration_task_result(result)
|
||||
except Exception as exc:
|
||||
session.rollback()
|
||||
record.update({
|
||||
"status": "blocked",
|
||||
"message": f"{type(exc).__name__}: {exc}",
|
||||
})
|
||||
records.append(record)
|
||||
raise ModuleMigrationTaskExecutionError(
|
||||
f"Module migration task {manifest.id}/{task.task_id} failed.",
|
||||
records=tuple(records),
|
||||
) from exc
|
||||
record.update({
|
||||
"status": normalized.status,
|
||||
"message": normalized.message,
|
||||
"details": _jsonable_migration_task_details(normalized.details),
|
||||
})
|
||||
records.append(record)
|
||||
if normalized.status == "blocked":
|
||||
session.rollback()
|
||||
raise ModuleMigrationTaskExecutionError(
|
||||
normalized.message or f"Module migration task {manifest.id}/{task.task_id} blocked migration.",
|
||||
records=tuple(records),
|
||||
)
|
||||
session.commit()
|
||||
return tuple(records)
|
||||
|
||||
|
||||
def _normalize_migration_task_result(result: ModuleMigrationTaskResult | None) -> ModuleMigrationTaskResult:
|
||||
if result is None:
|
||||
return ModuleMigrationTaskResult()
|
||||
if not isinstance(result, ModuleMigrationTaskResult):
|
||||
raise TypeError("Module migration task executors must return ModuleMigrationTaskResult or None.")
|
||||
if result.status not in {"ok", "warning", "blocked", "skipped"}:
|
||||
raise ValueError(f"Unsupported module migration task status: {result.status!r}")
|
||||
return result
|
||||
|
||||
|
||||
def _jsonable_migration_task_details(value: Mapping[str, Any]) -> Mapping[str, Any] | str:
|
||||
try:
|
||||
json.dumps(value)
|
||||
except (TypeError, ValueError):
|
||||
return str(dict(value))
|
||||
return value
|
||||
|
||||
|
||||
def _repo_root() -> Path:
|
||||
@@ -518,6 +668,7 @@ def migrate_database(
|
||||
database_url: str | None = None,
|
||||
reconcile_legacy_schema: bool = True,
|
||||
enabled_modules: tuple[str, ...] | list[str] | None = None,
|
||||
migration_module_order: tuple[str, ...] | list[str] | None = None,
|
||||
manifest_factories: tuple[ManifestFactory, ...] = (),
|
||||
) -> MigrationResult:
|
||||
url = database_url or settings.database_url
|
||||
@@ -526,14 +677,19 @@ def migrate_database(
|
||||
reconcile_change_sequence_retention_floor_drift(url)
|
||||
previous = database_revision(url)
|
||||
reconciled = reconcile_legacy_create_all_schema(url) if reconcile_legacy_schema else None
|
||||
command.upgrade(
|
||||
alembic_config(
|
||||
database_url=url,
|
||||
config = alembic_config(
|
||||
database_url=url,
|
||||
enabled_modules=enabled_modules,
|
||||
manifest_factories=manifest_factories,
|
||||
)
|
||||
if migration_module_order:
|
||||
plan = registered_module_migration_plan(
|
||||
url,
|
||||
enabled_modules=enabled_modules,
|
||||
manifest_factories=manifest_factories,
|
||||
),
|
||||
"heads",
|
||||
)
|
||||
)
|
||||
_upgrade_ordered_module_heads(config, plan, tuple(migration_module_order))
|
||||
command.upgrade(config, "heads")
|
||||
if reconcile_legacy_schema:
|
||||
reconcile_scope_table_retirement_drift(url)
|
||||
current = database_revision(url)
|
||||
@@ -542,3 +698,43 @@ def migrate_database(
|
||||
reconciled_revision=reconciled,
|
||||
current_revision=current,
|
||||
)
|
||||
|
||||
|
||||
def _upgrade_ordered_module_heads(
|
||||
config: Config,
|
||||
plan: MigrationMetadataPlan,
|
||||
module_order: tuple[str, ...],
|
||||
) -> None:
|
||||
module_heads = _module_heads_by_id(config, plan)
|
||||
upgraded: set[str] = set()
|
||||
for module_id in dict.fromkeys(module_order):
|
||||
for head in module_heads.get(module_id, ()):
|
||||
if head in upgraded:
|
||||
continue
|
||||
command.upgrade(config, head)
|
||||
upgraded.add(head)
|
||||
|
||||
|
||||
def _module_heads_by_id(config: Config, plan: MigrationMetadataPlan) -> dict[str, tuple[str, ...]]:
|
||||
script = ScriptDirectory.from_config(config)
|
||||
heads = script.get_heads()
|
||||
module_locations = {
|
||||
item.module_id: Path(item.script_location).resolve()
|
||||
for item in plan.modules
|
||||
if item.script_location
|
||||
}
|
||||
grouped: dict[str, list[str]] = {module_id: [] for module_id in module_locations}
|
||||
for head in heads:
|
||||
revision = script.get_revision(head)
|
||||
path_text = getattr(revision, "path", None)
|
||||
if revision is None or not path_text:
|
||||
continue
|
||||
revision_path = Path(path_text).resolve()
|
||||
for module_id, location in module_locations.items():
|
||||
try:
|
||||
revision_path.relative_to(location)
|
||||
except ValueError:
|
||||
continue
|
||||
grouped[module_id].append(head)
|
||||
break
|
||||
return {module_id: tuple(sorted(values)) for module_id, values in grouped.items() if values}
|
||||
|
||||
Reference in New Issue
Block a user