from __future__ import annotations import tempfile import unittest from pathlib import Path from alembic import command from alembic.runtime.migration import MigrationContext from sqlalchemy import create_engine, inspect, text from govoplan_core.db.base import Base from govoplan_core.db.migrations import ( REVISION_AUTH_RBAC, REVISION_FILE_FOLDERS, REVISION_HIERARCHICAL_SETTINGS, alembic_config, migrate_database, ) class DatabaseMigrationTests(unittest.TestCase): def test_repairs_create_all_schema_drift_and_upgrades_to_head(self) -> None: with tempfile.TemporaryDirectory(prefix="msm-migration-test-") as directory: database = Path(directory) / "legacy.db" url = f"sqlite:///{database}" # Reproduce the historical development database: Alembic was run # through auth/RBAC, then create_all() created later file tables # without advancing alembic_version and without altering the # already-existing campaign_versions table. command.upgrade(alembic_config(database_url=url), REVISION_AUTH_RBAC) engine = create_engine(url) try: Base.metadata.create_all(bind=engine) with engine.connect() as connection: self.assertEqual( MigrationContext.configure(connection).get_current_revision(), REVISION_AUTH_RBAC, ) self.assertIn("file_blobs", inspect(connection).get_table_names()) self.assertNotIn( "user_lock_state", {column["name"] for column in inspect(connection).get_columns("campaign_versions")}, ) finally: engine.dispose() result = migrate_database(database_url=url) self.assertEqual(result.previous_revision, REVISION_AUTH_RBAC) self.assertEqual(result.reconciled_revision, REVISION_FILE_FOLDERS) engine = create_engine(url) try: with engine.connect() as connection: current = MigrationContext.configure(connection).get_current_revision() inspector = inspect(connection) columns = { column["name"] for column in inspector.get_columns("campaign_versions") } folder_indexes = {index["name"] for index in inspector.get_indexes("file_folders")} job_columns = {column["name"] for column in inspector.get_columns("campaign_jobs")} job_indexes = {index["name"] for index in inspector.get_indexes("campaign_jobs")} attempt_columns = {column["name"] for column in inspector.get_columns("send_attempts")} tables = set(inspector.get_table_names()) tenant_columns = {column["name"] for column in inspector.get_columns("tenants")} user_columns = {column["name"] for column in inspector.get_columns("users")} session_columns = {column["name"] for column in inspector.get_columns("auth_sessions")} group_columns = {column["name"] for column in inspector.get_columns("groups")} role_columns = {column["name"] for column in inspector.get_columns("roles")} role_indexes = {index["name"] for index in inspector.get_indexes("roles")} campaign_columns = {column["name"] for column in inspector.get_columns("campaigns")} audit_columns = {column["name"] for column in inspector.get_columns("audit_log")} audit_indexes = {index["name"] for index in inspector.get_indexes("audit_log")} system_role_flags = { row["slug"]: bool(row["is_builtin"]) for row in connection.execute(text( "SELECT slug, is_builtin FROM roles WHERE tenant_id IS NULL AND slug IN ('system_owner', 'system_admin', 'system_auditor')" )).mappings().all() } system_settings_columns = {column["name"] for column in inspector.get_columns("system_settings")} governance_assignment_columns = {column["name"] for column in inspector.get_columns("governance_template_assignments")} import_profile_columns = {column["name"] for column in inspector.get_columns("campaign_recipient_import_mapping_profiles")} import_profile_indexes = {index["name"] for index in inspector.get_indexes("campaign_recipient_import_mapping_profiles")} account_count = connection.execute(text("SELECT COUNT(*) FROM accounts")).scalar_one() user_count = connection.execute(text("SELECT COUNT(*) FROM users")).scalar_one() system_owner_count = connection.execute(text( """ SELECT COUNT(*) FROM system_role_assignments assignment JOIN roles role ON role.id = assignment.role_id JOIN accounts account ON account.id = assignment.account_id WHERE role.slug = 'system_owner' AND account.is_active = 1 """ )).scalar_one() self.assertEqual(current, result.current_revision) self.assertIn("user_lock_state", columns) self.assertIn("user_locked_at", columns) self.assertIn("user_locked_by_user_id", columns) self.assertIn("uq_file_folders_active_user_path", folder_indexes) self.assertIn("uq_file_folders_active_group_path", folder_indexes) self.assertIn("execution_snapshot", columns) self.assertIn("execution_snapshot_hash", columns) self.assertIn("claimed_at", job_columns) self.assertIn("claim_token", job_columns) self.assertIn("smtp_started_at", job_columns) self.assertIn("outcome_unknown_at", job_columns) self.assertIn("eml_sha256", job_columns) self.assertIn("ix_campaign_jobs_claim_token", job_indexes) self.assertIn("ix_campaign_jobs_eml_sha256", job_indexes) self.assertIn("status", attempt_columns) self.assertIn("claim_token", attempt_columns) self.assertIn("accounts", tables) self.assertIn("system_role_assignments", tables) self.assertIn("system_settings", tables) self.assertIn("governance_templates", tables) self.assertIn("governance_template_assignments", tables) self.assertIn("campaign_shares", tables) self.assertIn("campaign_recipient_import_mapping_profiles", tables) self.assertIn("owner_user_id", campaign_columns) self.assertIn("owner_group_id", campaign_columns) self.assertIn("ordered_header_fingerprint", import_profile_columns) self.assertIn("unordered_header_fingerprint", import_profile_columns) self.assertIn("mappings", import_profile_columns) self.assertIn("ix_recipient_import_profiles_ordered_fp", import_profile_indexes) self.assertIn("scope", audit_columns) self.assertIn("ix_audit_log_scope", audit_indexes) self.assertIn("ix_audit_log_scope_created_at", audit_indexes) self.assertIn("ix_audit_log_tenant_scope_created_at", audit_indexes) self.assertTrue(system_role_flags["system_owner"]) self.assertFalse(system_role_flags["system_admin"]) self.assertFalse(system_role_flags["system_auditor"]) self.assertIn("allow_custom_groups", tenant_columns) self.assertIn("allow_custom_roles", tenant_columns) self.assertIn("allow_api_keys", tenant_columns) self.assertIn("description", tenant_columns) self.assertIn("default_locale", tenant_columns) self.assertIn("settings", tenant_columns) self.assertIn("settings", user_columns) self.assertIn("settings", group_columns) self.assertIn("settings", campaign_columns) self.assertIn("account_id", user_columns) self.assertIn("account_id", session_columns) self.assertIn("description", group_columns) self.assertIn("is_active", group_columns) self.assertIn("system_template_id", group_columns) self.assertIn("system_required", group_columns) self.assertIn("description", role_columns) self.assertIn("is_builtin", role_columns) self.assertIn("is_assignable", role_columns) self.assertIn("system_template_id", role_columns) self.assertIn("system_required", role_columns) self.assertIn("allow_tenant_custom_groups", system_settings_columns) self.assertIn("allow_tenant_custom_roles", system_settings_columns) self.assertIn("allow_tenant_api_keys", system_settings_columns) self.assertIn("mode", governance_assignment_columns) self.assertIn("uq_roles_system_slug", role_indexes) self.assertEqual(account_count, user_count) self.assertEqual(system_owner_count, 1 if user_count else 0) finally: engine.dispose() def test_repairs_current_schema_stamped_at_file_folders(self) -> None: with tempfile.TemporaryDirectory(prefix="msm-current-schema-marker-test-") as directory: database = Path(directory) / "current-schema-marker.db" url = f"sqlite:///{database}" command.upgrade(alembic_config(database_url=url), REVISION_HIERARCHICAL_SETTINGS) engine = create_engine(url) try: with engine.begin() as connection: connection.execute( text( """ INSERT INTO mail_server_profiles (id, tenant_id, scope_type, scope_id, name, slug, description, is_active, smtp_config, smtp_password_encrypted, imap_config, imap_password_encrypted, created_by_user_id, updated_by_user_id, created_at, updated_at) VALUES ('profile-1', NULL, 'system', 'system', 'System Mail', 'system-mail', NULL, 1, :smtp_config, NULL, :imap_config, NULL, NULL, NULL, '2026-06-25 15:00:00', '2026-06-25 15:00:00') """ ), { "smtp_config": '{"host":"smtp.example.test","port":587,"security":"starttls","username":"smtp-user","enabled":true}', "imap_config": '{"host":"imap.example.test","port":993,"security":"ssl","username":"imap-user","enabled":true}', }, ) connection.execute( text("UPDATE alembic_version SET version_num = :revision"), {"revision": REVISION_FILE_FOLDERS}, ) with engine.connect() as connection: self.assertEqual( MigrationContext.configure(connection).get_current_revision(), REVISION_FILE_FOLDERS, ) profile_columns = { column["name"] for column in inspect(connection).get_columns("mail_server_profiles") } self.assertNotIn("smtp_username", profile_columns) self.assertNotIn("imap_username", profile_columns) finally: engine.dispose() result = migrate_database(database_url=url) self.assertEqual(result.previous_revision, REVISION_FILE_FOLDERS) self.assertEqual(result.reconciled_revision, REVISION_HIERARCHICAL_SETTINGS) engine = create_engine(url) try: with engine.connect() as connection: self.assertEqual( MigrationContext.configure(connection).get_current_revision(), result.current_revision, ) profile_columns = { column["name"] for column in inspect(connection).get_columns("mail_server_profiles") } profile = connection.execute(text( """ SELECT smtp_username, smtp_config, imap_username, imap_config FROM mail_server_profiles WHERE id = 'profile-1' """ )).mappings().one() self.assertIn("smtp_username", profile_columns) self.assertIn("imap_username", profile_columns) self.assertEqual(profile["smtp_username"], "smtp-user") self.assertEqual(profile["imap_username"], "imap-user") self.assertNotIn("username", profile["smtp_config"]) self.assertNotIn("enabled", profile["smtp_config"]) self.assertNotIn("username", profile["imap_config"]) self.assertNotIn("enabled", profile["imap_config"]) finally: engine.dispose() def test_migrates_legacy_login_identity_and_bootstraps_system_owner(self) -> None: with tempfile.TemporaryDirectory(prefix="msm-account-migration-test-") as directory: database = Path(directory) / "accounts.db" url = f"sqlite:///{database}" command.upgrade(alembic_config(database_url=url), "7b8c9d0e1f2a") engine = create_engine(url) try: with engine.begin() as connection: timestamps = {"created_at": "2026-06-14 12:00:00", "updated_at": "2026-06-14 12:00:00"} connection.execute(text( """ INSERT INTO tenants (id, slug, name, is_active, created_at, updated_at) VALUES ('tenant-1', 'legacy', 'Legacy', 1, :created_at, :updated_at) """ ), timestamps) connection.execute(text( """ INSERT INTO users (id, tenant_id, email, display_name, is_active, is_tenant_admin, auth_provider, password_hash, last_login_at, created_at, updated_at) VALUES ('user-1', 'tenant-1', 'Owner@Example.Local', 'Legacy Owner', 1, 1, 'local', 'legacy-password-hash', NULL, :created_at, :updated_at) """ ), timestamps) connection.execute(text( """ INSERT INTO auth_sessions (id, tenant_id, user_id, token_hash, expires_at, last_seen_at, revoked_at, user_agent, ip_address, created_at, updated_at) VALUES ('session-1', 'tenant-1', 'user-1', 'token-hash', '2026-06-15 12:00:00', NULL, NULL, NULL, NULL, :created_at, :updated_at) """ ), timestamps) finally: engine.dispose() migrate_database(database_url=url, reconcile_legacy_schema=False) engine = create_engine(url) try: with engine.connect() as connection: account = connection.execute(text( "SELECT id, normalized_email, password_hash FROM accounts" )).mappings().one() membership = connection.execute(text( "SELECT account_id FROM users WHERE id = 'user-1'" )).mappings().one() migrated_session = connection.execute(text( "SELECT account_id FROM auth_sessions WHERE id = 'session-1'" )).mappings().one() owner_assignment = connection.execute(text( """ SELECT role.slug FROM user_role_assignments assignment JOIN roles role ON role.id = assignment.role_id WHERE assignment.user_id = 'user-1' """ )).scalar_one() owner_permissions = connection.execute(text( "SELECT permissions FROM roles WHERE tenant_id = 'tenant-1' AND slug = 'owner'" )).scalar_one() system_assignment = connection.execute(text( """ SELECT role.slug FROM system_role_assignments assignment JOIN roles role ON role.id = assignment.role_id WHERE assignment.account_id = :account_id """ ), {"account_id": account["id"]}).scalar_one() system_role_flags = { row["slug"]: bool(row["is_builtin"]) for row in connection.execute(text( "SELECT slug, is_builtin FROM roles WHERE tenant_id IS NULL AND slug IN ('system_owner', 'system_admin', 'system_auditor')" )).mappings().all() } self.assertEqual(account["normalized_email"], "owner@example.local") self.assertEqual(account["password_hash"], "legacy-password-hash") self.assertEqual(membership["account_id"], account["id"]) self.assertEqual(migrated_session["account_id"], account["id"]) self.assertEqual(owner_assignment, "owner") self.assertIn("tenant:*", owner_permissions) self.assertEqual(system_assignment, "system_owner") self.assertTrue(system_role_flags["system_owner"]) self.assertFalse(system_role_flags["system_admin"]) self.assertFalse(system_role_flags["system_auditor"]) finally: engine.dispose() def test_backfills_explicit_system_and_tenant_audit_scopes(self) -> None: with tempfile.TemporaryDirectory(prefix="msm-audit-scope-migration-test-") as directory: database = Path(directory) / "audit-scope.db" url = f"sqlite:///{database}" command.upgrade(alembic_config(database_url=url), "a0b1c2d3e4f5") engine = create_engine(url) try: with engine.begin() as connection: timestamps = { "created_at": "2026-06-15 12:00:00", "updated_at": "2026-06-15 12:00:00", } connection.execute(text( """ INSERT INTO audit_log (id, tenant_id, user_id, api_key_id, action, object_type, object_id, details, created_at, updated_at) VALUES ('audit-system', NULL, NULL, NULL, 'system_settings.updated', 'system_settings', 'system', '{}', :created_at, :updated_at), ('audit-tenant', NULL, NULL, NULL, 'user.updated', 'user', 'user-1', '{}', :created_at, :updated_at) """ ), timestamps) finally: engine.dispose() migrate_database(database_url=url, reconcile_legacy_schema=False) engine = create_engine(url) try: with engine.connect() as connection: scopes = dict(connection.execute(text( "SELECT id, scope FROM audit_log WHERE id IN ('audit-system', 'audit-tenant')" )).all()) self.assertEqual(scopes["audit-system"], "system") self.assertEqual(scopes["audit-tenant"], "tenant") finally: engine.dispose() def test_deduplicates_active_folder_paths_before_unique_indexes(self) -> None: with tempfile.TemporaryDirectory(prefix="msm-folder-migration-test-") as directory: database = Path(directory) / "duplicates.db" url = f"sqlite:///{database}" command.upgrade(alembic_config(database_url=url), "5f6a7b8c9d0e") engine = create_engine(url) try: with engine.begin() as connection: parameters = { "tenant_id": "tenant-1", "owner_user_id": "user-1", "path": "archive", "created_at": "2026-06-13 20:00:00", "updated_at": "2026-06-13 20:00:00", } connection.execute(text( """ INSERT INTO file_folders (id, tenant_id, owner_type, owner_user_id, owner_group_id, path, created_by_user_id, deleted_at, metadata, created_at, updated_at) VALUES ('folder-1', :tenant_id, 'user', :owner_user_id, NULL, :path, NULL, NULL, '{}', :created_at, :updated_at), ('folder-2', :tenant_id, 'user', :owner_user_id, NULL, :path, NULL, NULL, '{}', :created_at, :updated_at) """ ), parameters) finally: engine.dispose() migrate_database(database_url=url, reconcile_legacy_schema=False) engine = create_engine(url) try: with engine.connect() as connection: active_count = connection.execute(text( "SELECT COUNT(*) FROM file_folders WHERE path = 'archive' AND deleted_at IS NULL" )).scalar_one() deleted_count = connection.execute(text( "SELECT COUNT(*) FROM file_folders WHERE path = 'archive' AND deleted_at IS NOT NULL" )).scalar_one() folder_indexes = {index["name"] for index in inspect(connection).get_indexes("file_folders")} self.assertEqual(active_count, 1) self.assertEqual(deleted_count, 1) self.assertIn("uq_file_folders_active_user_path", folder_indexes) self.assertIn("uq_file_folders_active_group_path", folder_indexes) finally: engine.dispose()