Release v0.1.7
All checks were successful
Dependency Audit / dependency-audit (push) Successful in 1m35s

This commit is contained in:
2026-07-11 02:46:04 +02:00
parent edb4687826
commit a00ef54821
25 changed files with 2740 additions and 201 deletions

View File

@@ -1,9 +1,17 @@
from __future__ import annotations
import argparse
import json
from pathlib import Path
from govoplan_core.db.bootstrap import bootstrap_dev_data
from govoplan_core.db.migrations import migrate_database
from govoplan_core.db.migrations import (
ModuleMigrationTaskExecutionError,
POST_MIGRATION_TASK_PHASES,
PRE_MIGRATION_TASK_PHASES,
migrate_database,
run_registered_module_migration_tasks,
)
from govoplan_core.db.session import configure_database, get_database
from govoplan_core.settings import settings
@@ -11,12 +19,43 @@ from govoplan_core.settings import settings
def main() -> None:
parser = argparse.ArgumentParser(description="Initialize the GovOPlaN database")
parser.add_argument("--database-url", default=settings.database_url, help="Database URL to migrate")
parser.add_argument("--enabled-module", action="append", default=[], help="Target enabled module id used to discover module migrations; may be repeated.")
parser.add_argument("--migration-module", action="append", default=[], help="Module id whose migration heads should be upgraded in this order before final heads.")
parser.add_argument("--migration-task-record-output", type=Path, help="Write executed module migration task records to this JSON file.")
parser.add_argument("--with-dev-data", action="store_true", help="Create default tenant/user/roles and a development API key")
parser.add_argument("--dev-api-key", default=settings.dev_bootstrap_api_key, help="Development API key secret to create")
args = parser.parse_args()
configure_database(args.database_url)
migration = migrate_database(database_url=args.database_url)
enabled_modules = tuple(args.enabled_module) if args.enabled_module else None
migration_order = tuple(args.migration_module) if args.migration_module else None
task_records: list[dict[str, object]] = []
try:
_run_migration_tasks(
task_records,
database_url=args.database_url,
enabled_modules=enabled_modules,
migration_order=migration_order,
phases=PRE_MIGRATION_TASK_PHASES,
)
migration = migrate_database(
database_url=args.database_url,
enabled_modules=enabled_modules,
migration_module_order=migration_order,
)
_run_migration_tasks(
task_records,
database_url=args.database_url,
enabled_modules=enabled_modules,
migration_order=migration_order,
phases=POST_MIGRATION_TASK_PHASES,
)
finally:
if args.migration_task_record_output:
args.migration_task_record_output.parent.mkdir(parents=True, exist_ok=True)
args.migration_task_record_output.write_text(json.dumps(task_records, indent=2, sort_keys=True) + "\n", encoding="utf-8")
if task_records:
print(f"Executed {len(task_records)} module migration task(s).")
if migration.reconciled_revision:
print(f"Reconciled legacy database marker to {migration.reconciled_revision}.")
print(f"Database schema upgraded to {migration.current_revision}.")
@@ -33,5 +72,26 @@ def main() -> None:
print("Development API key already exists or was not requested.")
def _run_migration_tasks(
target: list[dict[str, object]],
*,
database_url: str,
enabled_modules: tuple[str, ...] | None,
migration_order: tuple[str, ...] | None,
phases: tuple[str, ...],
) -> None:
try:
records = run_registered_module_migration_tasks(
database_url=database_url,
enabled_modules=enabled_modules,
migration_module_order=migration_order,
phases=phases,
)
except ModuleMigrationTaskExecutionError as exc:
target.extend(exc.records)
raise
target.extend(records)
if __name__ == "__main__":
main()

View File

@@ -12,6 +12,29 @@ from govoplan_core.core.registry import PlatformRegistry
class MigrationMetadataPlan:
metadata: tuple[MetaData, ...]
script_locations: tuple[str, ...]
modules: tuple[ModuleMigrationMetadata, ...] = ()
@dataclass(frozen=True, slots=True)
class ModuleMigrationMetadata:
module_id: str
script_location: str | None = None
has_metadata: bool = False
migration_after: tuple[str, ...] = ()
migration_before: tuple[str, ...] = ()
migration_tasks: tuple[ModuleMigrationTaskMetadata, ...] = ()
@dataclass(frozen=True, slots=True)
class ModuleMigrationTaskMetadata:
task_id: str
phase: str
summary: str
task_version: str = "1"
safety: str = "automatic"
idempotent: bool = True
timeout_seconds: int | None = None
has_executor: bool = False
def migration_metadata_plan(registry: PlatformRegistry, *, extra_metadata: Iterable[MetaData] = ()) -> MigrationMetadataPlan:
@@ -28,4 +51,28 @@ def migration_metadata_plan(registry: PlatformRegistry, *, extra_metadata: Itera
metadata.append(spec.metadata)
if spec.script_location:
script_locations.append(spec.script_location)
return MigrationMetadataPlan(metadata=tuple(metadata), script_locations=tuple(script_locations))
modules = tuple(
ModuleMigrationMetadata(
module_id=manifest.id,
script_location=manifest.migration_spec.script_location,
has_metadata=isinstance(manifest.migration_spec.metadata, MetaData),
migration_after=tuple(manifest.migration_spec.migration_after),
migration_before=tuple(manifest.migration_spec.migration_before),
migration_tasks=tuple(
ModuleMigrationTaskMetadata(
task_id=task.task_id,
phase=task.phase,
summary=task.summary,
task_version=task.task_version,
safety=task.safety,
idempotent=task.idempotent,
timeout_seconds=task.timeout_seconds,
has_executor=task.executor is not None,
)
for task in manifest.migration_spec.migration_tasks
),
)
for manifest in registry.manifests()
if manifest.migration_spec is not None
)
return MigrationMetadataPlan(metadata=tuple(metadata), script_locations=tuple(script_locations), modules=modules)

File diff suppressed because it is too large Load Diff

View File

@@ -20,6 +20,12 @@ from govoplan_core.core.versioning import format_version_range, version_range_is
_INTERFACE_NAME_RE = re.compile(r"^[a-z][a-z0-9_]*(?:\.[a-z][a-z0-9_]*)+$")
CATALOG_MIGRATION_SAFETY = ("automatic", "requires_review", "forward_only", "destructive")
CATALOG_MIGRATION_TASK_PHASES = (
"pre_migration_check",
"pre_migration_prepare",
"post_migration_backfill",
"post_migration_verify",
)
def module_package_catalog(
@@ -611,6 +617,17 @@ def _normalize_catalog_item(value: Any) -> dict[str, object]:
"optional_dependencies": _string_list(value.get("optional_dependencies")),
"migration_safety": _catalog_migration_safety(value.get("migration_safety"), module_id=module_id),
"migration_notes": _optional_str(value, "migration_notes"),
"migration_after": _string_list(value.get("migration_after")),
"migration_before": _string_list(value.get("migration_before")),
"migration_tasks": _normalize_catalog_migration_tasks(value.get("migration_tasks"), module_id=module_id),
"current_version_min": _optional_str(value, "current_version_min"),
"current_version_max_exclusive": _optional_str(value, "current_version_max_exclusive"),
"bridge_release": _optional_bool(value, "bridge_release"),
"bridge_notes": _optional_str(value, "bridge_notes"),
"allow_downgrade": _optional_bool(value, "allow_downgrade"),
"allow_same_version": _optional_bool(value, "allow_same_version"),
"recovery_tested": _optional_bool(value, "recovery_tested"),
"recovery_notes": _optional_str(value, "recovery_notes"),
"python_package": _optional_str(value, "python_package"),
"python_ref": _optional_str(value, "python_ref"),
"webui_package": _optional_str(value, "webui_package"),
@@ -621,6 +638,15 @@ def _normalize_catalog_item(value: Any) -> dict[str, object]:
"notes": _optional_str(value, "notes"),
"tags": _string_list(value.get("tags")),
}
if not version_range_is_valid(
version_min=item["current_version_min"] if isinstance(item["current_version_min"], str) else None,
version_max_exclusive=item["current_version_max_exclusive"] if isinstance(item["current_version_max_exclusive"], str) else None,
):
version_range = format_version_range(
version_min=item["current_version_min"] if isinstance(item["current_version_min"], str) else None,
version_max_exclusive=item["current_version_max_exclusive"] if isinstance(item["current_version_max_exclusive"], str) else None,
)
raise ValueError(f"Module package catalog entry {module_id!r} has invalid current-version range {version_range!r}.")
artifact_integrity = _normalize_artifact_integrity(value.get("artifact_integrity"))
if artifact_integrity:
item["artifact_integrity"] = artifact_integrity
@@ -637,6 +663,66 @@ def _catalog_migration_safety(value: Any, *, module_id: str) -> str:
return safety
def _normalize_catalog_migration_tasks(value: Any, *, module_id: str) -> list[dict[str, object]]:
if value is None:
return []
if not isinstance(value, list):
raise ValueError(f"Module package catalog migration_tasks for {module_id!r} must be a list.")
normalized: list[dict[str, object]] = []
seen: set[str] = set()
for raw in value:
if not isinstance(raw, dict):
raise ValueError(f"Module package catalog migration_tasks entries for {module_id!r} must be objects.")
task_id = _catalog_task_required_str(raw, "task_id", module_id=module_id)
if task_id in seen:
raise ValueError(f"Module package catalog entry {module_id!r} declares migration task {task_id!r} more than once.")
seen.add(task_id)
phase = _catalog_task_required_str(raw, "phase", module_id=module_id)
if phase not in CATALOG_MIGRATION_TASK_PHASES:
allowed = ", ".join(CATALOG_MIGRATION_TASK_PHASES)
raise ValueError(f"Unsupported migration task phase for {module_id!r}/{task_id!r}: {phase!r}; expected one of {allowed}.")
summary = _catalog_task_required_str(raw, "summary", module_id=module_id)
safety = _catalog_migration_safety(raw.get("safety"), module_id=module_id)
item: dict[str, object] = {
"task_id": task_id,
"phase": phase,
"summary": summary,
"task_version": _optional_str(raw, "task_version") or "1",
"safety": safety,
"idempotent": _catalog_optional_bool_default(raw, "idempotent", default=True),
}
timeout_seconds = _catalog_optional_positive_int(raw, "timeout_seconds", module_id=module_id, task_id=task_id)
if timeout_seconds is not None:
item["timeout_seconds"] = timeout_seconds
normalized.append(item)
return normalized
def _catalog_task_required_str(value: dict[str, Any], key: str, *, module_id: str) -> str:
item = _optional_str(value, key)
if not item:
raise ValueError(f"Module package catalog migration task for {module_id!r} is missing {key!r}.")
return item
def _catalog_optional_bool_default(value: dict[str, Any], key: str, *, default: bool) -> bool:
if key not in value:
return default
return _optional_bool(value, key)
def _catalog_optional_positive_int(value: dict[str, Any], key: str, *, module_id: str, task_id: str) -> int | None:
if value.get(key) is None:
return None
try:
integer = int(value[key])
except (TypeError, ValueError) as exc:
raise ValueError(f"Module package catalog migration task {module_id!r}/{task_id!r} has invalid {key!r}.") from exc
if integer <= 0:
raise ValueError(f"Module package catalog migration task {module_id!r}/{task_id!r} requires positive {key!r}.")
return integer
def _required_str(value: dict[str, Any], key: str) -> str:
item = _optional_str(value, key)
if not item:

View File

@@ -13,6 +13,14 @@ SUPPORTED_FRONTEND_ASSET_MANIFEST_CONTRACT_VERSION = "1"
PermissionLevel = Literal["system", "tenant"]
SubjectType = Literal["account", "membership", "group", "service_account", "tenant"]
MigrationTaskPhase = Literal[
"pre_migration_check",
"pre_migration_prepare",
"post_migration_backfill",
"post_migration_verify",
]
MigrationTaskSafety = Literal["automatic", "requires_review", "forward_only", "destructive"]
MigrationTaskStatus = Literal["ok", "warning", "blocked", "skipped"]
@dataclass(frozen=True, slots=True)
@@ -91,11 +99,49 @@ MigrationRetirementExecutor = Callable[[object, str], None]
MigrationRetirementProvider = Callable[[object | None, str], MigrationRetirementPlan]
@dataclass(frozen=True, slots=True)
class ModuleMigrationTaskContext:
module_id: str
task_id: str
phase: MigrationTaskPhase
database_url: str
target_version: str | None = None
session: object | None = None
dry_run: bool = False
metadata: Mapping[str, Any] = field(default_factory=dict)
@dataclass(frozen=True, slots=True)
class ModuleMigrationTaskResult:
status: MigrationTaskStatus = "ok"
message: str | None = None
details: Mapping[str, Any] = field(default_factory=dict)
ModuleMigrationTaskExecutor = Callable[[ModuleMigrationTaskContext], ModuleMigrationTaskResult | None]
@dataclass(frozen=True, slots=True)
class ModuleMigrationTask:
task_id: str
phase: MigrationTaskPhase
summary: str
task_version: str = "1"
safety: MigrationTaskSafety = "automatic"
idempotent: bool = True
timeout_seconds: int | None = None
executor: ModuleMigrationTaskExecutor | None = None
metadata: Mapping[str, Any] = field(default_factory=dict)
@dataclass(frozen=True, slots=True)
class MigrationSpec:
module_id: str
metadata: object | None = None
script_location: str | None = None
migration_after: tuple[str, ...] = ()
migration_before: tuple[str, ...] = ()
migration_tasks: tuple[ModuleMigrationTask, ...] = ()
retirement_supported: bool = False
retirement_provider: MigrationRetirementProvider | None = None
retirement_notes: str | None = None
@@ -215,7 +261,28 @@ class ResourceAclProvider(Protocol):
TenantSummaryProvider = Callable[[object, str], Mapping[str, int]]
DeleteVetoProvider = Callable[[object, str, str], None]
@dataclass(frozen=True, slots=True)
class DeleteVetoIssue:
severity: Literal["blocker", "warning", "info"]
code: str
message: str
module_id: str | None = None
details: Mapping[str, Any] = field(default_factory=dict)
DeleteVetoProviderResult = DeleteVetoIssue | Iterable[DeleteVetoIssue] | None
DeleteVetoProvider = Callable[[object, str, str], DeleteVetoProviderResult]
@dataclass(frozen=True, slots=True)
class DeleteVetoProviderRegistration:
module_id: str
resource_type: str
provider: DeleteVetoProvider
RouteFactory = Callable[[ModuleContext], "APIRouter"]
CapabilityFactory = Callable[[ModuleContext], object]
DocumentationProvider = Callable[[DocumentationContext], Iterable[DocumentationTopic]]

View File

@@ -3,11 +3,13 @@ from __future__ import annotations
import re
from collections import defaultdict, deque
from collections.abc import Iterable, Mapping
from dataclasses import dataclass
from dataclasses import dataclass, replace
from govoplan_core.core.modules import (
CapabilityFactory,
DeleteVetoIssue,
DeleteVetoProvider,
DeleteVetoProviderRegistration,
ModuleInterfaceProvider,
ModuleInterfaceRequirement,
ModuleContext,
@@ -45,7 +47,7 @@ class PlatformRegistry:
def __init__(self) -> None:
self._manifests: dict[str, ModuleManifest] = {}
self._tenant_summary_providers: dict[str, TenantSummaryProvider] = {}
self._delete_veto_providers: dict[str, list[DeleteVetoProvider]] = defaultdict(list)
self._delete_veto_providers: dict[str, list[DeleteVetoProviderRegistration]] = defaultdict(list)
self._capability_factories: dict[str, CapabilityFactory] = {}
self._capabilities: dict[str, object] = {}
self._capability_context: ModuleContext | None = None
@@ -58,7 +60,7 @@ class PlatformRegistry:
self.register_tenant_summary_provider(manifest.id, provider)
for resource_type, providers in manifest.delete_veto_providers.items():
for provider in providers:
self.register_delete_veto(resource_type, provider)
self.register_delete_veto(manifest.id, resource_type, provider)
for name, factory in manifest.capability_factories.items():
self.register_capability_factory(manifest.id, name, factory)
return manifest
@@ -148,12 +150,39 @@ class PlatformRegistry:
def tenant_summary_providers(self) -> Mapping[str, TenantSummaryProvider]:
return dict(self._tenant_summary_providers)
def register_delete_veto(self, resource_type: str, provider: DeleteVetoProvider) -> None:
self._delete_veto_providers[resource_type].append(provider)
def register_delete_veto(self, module_id: str, resource_type: str, provider: DeleteVetoProvider) -> None:
self._delete_veto_providers[resource_type].append(DeleteVetoProviderRegistration(
module_id=module_id,
resource_type=resource_type,
provider=provider,
))
def delete_veto_providers(self, resource_type: str) -> tuple[DeleteVetoProvider, ...]:
return tuple(registration.provider for registration in self._delete_veto_providers.get(resource_type, ()))
def delete_veto_provider_registrations(self, resource_type: str) -> tuple[DeleteVetoProviderRegistration, ...]:
return tuple(self._delete_veto_providers.get(resource_type, ()))
def collect_delete_veto_issues(self, resource_type: str, session: object, tenant_id: str, resource_id: str) -> tuple[DeleteVetoIssue, ...]:
issues: list[DeleteVetoIssue] = []
for registration in self.delete_veto_provider_registrations(resource_type):
try:
result = registration.provider(session, tenant_id, resource_id)
except Exception as exc:
issues.append(DeleteVetoIssue(
severity="blocker",
code="module_veto",
message=str(exc),
module_id=registration.module_id,
details={
"resource_type": resource_type,
"resource_id": resource_id,
},
))
continue
issues.extend(_normalize_delete_veto_result(result, registration=registration, resource_id=resource_id))
return tuple(issues)
def validate(self) -> RegistrySnapshot:
ordered = tuple(self._topologically_sorted())
available_capabilities = set(self._capability_factories)
@@ -224,6 +253,52 @@ class PlatformRegistry:
return (self._manifests[module_id] for module_id in ordered)
def _normalize_delete_veto_result(
result: object,
*,
registration: DeleteVetoProviderRegistration,
resource_id: str,
) -> tuple[DeleteVetoIssue, ...]:
if result is None:
return ()
if isinstance(result, DeleteVetoIssue):
return (_attribute_delete_veto_issue(result, registration=registration, resource_id=resource_id),)
if isinstance(result, str):
return (DeleteVetoIssue(
severity="blocker",
code="module_veto",
message=result,
module_id=registration.module_id,
details={
"resource_type": registration.resource_type,
"resource_id": resource_id,
},
),)
if isinstance(result, Iterable):
return tuple(
_attribute_delete_veto_issue(issue, registration=registration, resource_id=resource_id)
for issue in result
if isinstance(issue, DeleteVetoIssue)
)
return ()
def _attribute_delete_veto_issue(
issue: DeleteVetoIssue,
*,
registration: DeleteVetoProviderRegistration,
resource_id: str,
) -> DeleteVetoIssue:
details = {
"resource_type": registration.resource_type,
"resource_id": resource_id,
**dict(issue.details),
}
if issue.module_id == registration.module_id and issue.details == details:
return issue
return replace(issue, module_id=issue.module_id or registration.module_id, details=details)
def _validate_manifest_shape(manifest: ModuleManifest) -> None:
if not _MODULE_ID_RE.match(manifest.id):
raise RegistryError(f"Module manifest id must match {_MODULE_ID_RE.pattern}: {manifest.id!r}")

View File

@@ -1,19 +1,24 @@
from __future__ import annotations
from collections.abc import Mapping
from dataclasses import dataclass
import json
import os
from pathlib import Path
from typing import Any
from alembic import command
from alembic.config import Config
from alembic.runtime.migration import MigrationContext
from alembic.script import ScriptDirectory
from sqlalchemy import create_engine, inspect, text
from govoplan_core.core.migrations import MigrationMetadataPlan, migration_metadata_plan
from govoplan_core.core import change_sequence as core_change_sequence_models # noqa: F401 - populate core metadata
from govoplan_core.core.change_sequence import ChangeSequenceEntry, ChangeSequenceRetentionFloor
from govoplan_core.core.module_management import load_startup_enabled_modules, startup_candidate_module_ids
from govoplan_core.db.session import configure_database
from govoplan_core.core.modules import ModuleMigrationTaskContext, ModuleMigrationTaskResult
from govoplan_core.db.session import configure_database, get_database
from govoplan_core.server.default_config import get_server_config
from govoplan_core.server.config import ManifestFactory
from govoplan_core.server.registry import available_module_manifests, build_platform_registry
@@ -33,6 +38,9 @@ REVISION_CORE_CHANGE_SEQUENCE = "3f4a5b6c7d8e"
REVISION_HIERARCHICAL_SETTINGS = "f5a6b7c8d9e0"
LEGACY_SCOPE_TABLE = "tenancy_tenants"
CORE_SCOPE_TABLE = "core_scopes"
PRE_MIGRATION_TASK_PHASES = ("pre_migration_check", "pre_migration_prepare")
POST_MIGRATION_TASK_PHASES = ("post_migration_backfill", "post_migration_verify")
MIGRATION_TASK_PHASES = (*PRE_MIGRATION_TASK_PHASES, *POST_MIGRATION_TASK_PHASES)
_NAMESPACE_TABLE_RENAMES = (
("tenants", "tenancy_tenants"),
@@ -160,12 +168,31 @@ class MigrationResult:
current_revision: str | None
class ModuleMigrationTaskExecutionError(RuntimeError):
def __init__(self, message: str, *, records: tuple[dict[str, object], ...]) -> None:
super().__init__(message)
self.records = records
def registered_module_migration_plan(
database_url: str | None = None,
*,
enabled_modules: tuple[str, ...] | list[str] | None = None,
manifest_factories: tuple[ManifestFactory, ...] = (),
) -> MigrationMetadataPlan:
return migration_metadata_plan(_registered_module_registry(
database_url=database_url,
enabled_modules=enabled_modules,
manifest_factories=manifest_factories,
))
def _registered_module_registry(
*,
database_url: str | None = None,
enabled_modules: tuple[str, ...] | list[str] | None = None,
manifest_factories: tuple[ManifestFactory, ...] = (),
):
server_config = get_server_config()
active_database_url = database_url or settings.database_url
if active_database_url:
@@ -187,7 +214,130 @@ def registered_module_migration_plan(
active_enabled_modules,
manifest_factories=active_manifest_factories,
)
return migration_metadata_plan(registry)
return registry
def run_registered_module_migration_tasks(
*,
database_url: str | None = None,
enabled_modules: tuple[str, ...] | list[str] | None = None,
migration_module_order: tuple[str, ...] | list[str] | None = None,
phases: tuple[str, ...] | list[str] = MIGRATION_TASK_PHASES,
dry_run: bool = False,
manifest_factories: tuple[ManifestFactory, ...] = (),
) -> tuple[dict[str, object], ...]:
active_phases = tuple(dict.fromkeys(str(item).strip() for item in phases if str(item).strip()))
invalid_phases = tuple(phase for phase in active_phases if phase not in MIGRATION_TASK_PHASES)
if invalid_phases:
raise ValueError("Unsupported module migration task phase(s): " + ", ".join(invalid_phases))
url = database_url or settings.database_url
registry = _registered_module_registry(
database_url=url,
enabled_modules=enabled_modules,
manifest_factories=manifest_factories,
)
manifests = {manifest.id: manifest for manifest in registry.manifests()}
ordered_ids = tuple(dict.fromkeys([
*(str(item).strip() for item in (migration_module_order or ()) if str(item).strip()),
*manifests.keys(),
]))
records: list[dict[str, object]] = []
database = get_database()
with database.SessionLocal() as session:
for phase in active_phases:
for module_id in ordered_ids:
manifest = manifests.get(module_id)
if manifest is None or manifest.migration_spec is None:
continue
for task in manifest.migration_spec.migration_tasks:
if task.phase != phase:
continue
record: dict[str, object] = {
"module_id": manifest.id,
"task_id": task.task_id,
"phase": task.phase,
"summary": task.summary,
"task_version": task.task_version,
"safety": task.safety,
"idempotent": task.idempotent,
"dry_run": dry_run,
}
if task.timeout_seconds is not None:
record["timeout_seconds"] = task.timeout_seconds
if not task.idempotent:
record.update({"status": "blocked", "message": "Task is not idempotent."})
records.append(record)
raise ModuleMigrationTaskExecutionError(
f"Module migration task {manifest.id}/{task.task_id} is not idempotent.",
records=tuple(records),
)
if dry_run:
record.update({"status": "skipped", "message": "Dry run; executor was not called."})
records.append(record)
continue
if task.executor is None:
record.update({"status": "blocked", "message": "Task has no executor."})
records.append(record)
raise ModuleMigrationTaskExecutionError(
f"Module migration task {manifest.id}/{task.task_id} has no executor.",
records=tuple(records),
)
context = ModuleMigrationTaskContext(
module_id=manifest.id,
task_id=task.task_id,
phase=task.phase,
database_url=url,
target_version=manifest.version,
session=session,
dry_run=dry_run,
metadata=task.metadata,
)
try:
result = task.executor(context)
normalized = _normalize_migration_task_result(result)
except Exception as exc:
session.rollback()
record.update({
"status": "blocked",
"message": f"{type(exc).__name__}: {exc}",
})
records.append(record)
raise ModuleMigrationTaskExecutionError(
f"Module migration task {manifest.id}/{task.task_id} failed.",
records=tuple(records),
) from exc
record.update({
"status": normalized.status,
"message": normalized.message,
"details": _jsonable_migration_task_details(normalized.details),
})
records.append(record)
if normalized.status == "blocked":
session.rollback()
raise ModuleMigrationTaskExecutionError(
normalized.message or f"Module migration task {manifest.id}/{task.task_id} blocked migration.",
records=tuple(records),
)
session.commit()
return tuple(records)
def _normalize_migration_task_result(result: ModuleMigrationTaskResult | None) -> ModuleMigrationTaskResult:
if result is None:
return ModuleMigrationTaskResult()
if not isinstance(result, ModuleMigrationTaskResult):
raise TypeError("Module migration task executors must return ModuleMigrationTaskResult or None.")
if result.status not in {"ok", "warning", "blocked", "skipped"}:
raise ValueError(f"Unsupported module migration task status: {result.status!r}")
return result
def _jsonable_migration_task_details(value: Mapping[str, Any]) -> Mapping[str, Any] | str:
try:
json.dumps(value)
except (TypeError, ValueError):
return str(dict(value))
return value
def _repo_root() -> Path:
@@ -518,6 +668,7 @@ def migrate_database(
database_url: str | None = None,
reconcile_legacy_schema: bool = True,
enabled_modules: tuple[str, ...] | list[str] | None = None,
migration_module_order: tuple[str, ...] | list[str] | None = None,
manifest_factories: tuple[ManifestFactory, ...] = (),
) -> MigrationResult:
url = database_url or settings.database_url
@@ -526,14 +677,19 @@ def migrate_database(
reconcile_change_sequence_retention_floor_drift(url)
previous = database_revision(url)
reconciled = reconcile_legacy_create_all_schema(url) if reconcile_legacy_schema else None
command.upgrade(
alembic_config(
database_url=url,
config = alembic_config(
database_url=url,
enabled_modules=enabled_modules,
manifest_factories=manifest_factories,
)
if migration_module_order:
plan = registered_module_migration_plan(
url,
enabled_modules=enabled_modules,
manifest_factories=manifest_factories,
),
"heads",
)
)
_upgrade_ordered_module_heads(config, plan, tuple(migration_module_order))
command.upgrade(config, "heads")
if reconcile_legacy_schema:
reconcile_scope_table_retirement_drift(url)
current = database_revision(url)
@@ -542,3 +698,43 @@ def migrate_database(
reconciled_revision=reconciled,
current_revision=current,
)
def _upgrade_ordered_module_heads(
config: Config,
plan: MigrationMetadataPlan,
module_order: tuple[str, ...],
) -> None:
module_heads = _module_heads_by_id(config, plan)
upgraded: set[str] = set()
for module_id in dict.fromkeys(module_order):
for head in module_heads.get(module_id, ()):
if head in upgraded:
continue
command.upgrade(config, head)
upgraded.add(head)
def _module_heads_by_id(config: Config, plan: MigrationMetadataPlan) -> dict[str, tuple[str, ...]]:
script = ScriptDirectory.from_config(config)
heads = script.get_heads()
module_locations = {
item.module_id: Path(item.script_location).resolve()
for item in plan.modules
if item.script_location
}
grouped: dict[str, list[str]] = {module_id: [] for module_id in module_locations}
for head in heads:
revision = script.get_revision(head)
path_text = getattr(revision, "path", None)
if revision is None or not path_text:
continue
revision_path = Path(path_text).resolve()
for module_id, location in module_locations.items():
try:
revision_path.relative_to(location)
except ValueError:
continue
grouped[module_id].append(head)
break
return {module_id: tuple(sorted(values)) for module_id, values in grouped.items() if values}