diff --git a/docs/POLICY_CONTRACTS.md b/docs/POLICY_CONTRACTS.md index ca34936..06ce589 100644 --- a/docs/POLICY_CONTRACTS.md +++ b/docs/POLICY_CONTRACTS.md @@ -97,6 +97,20 @@ New backend code should import policy-owned retention behavior from `govoplan-policy` or request the capability, not add new implementation logic to core. +The retention API DTOs live in `govoplan_core.privacy.schemas`. +`PrivacyRetentionPolicyItem`, `PrivacyRetentionPolicyPatchItem`, +`RETENTION_POLICY_FIELD_KEYS`, and `default_allow_lower_level_limits()` are +platform contracts because admin, access compatibility, and policy routes expose +the same stable retention payload shape. The policy engine's internal +`PrivacyRetentionPolicy` and `PrivacyRetentionPolicyPatch` models stay in +`govoplan-policy`, because they carry implementation validators and merge +behavior that are not generic API contracts. + +Tenant administration DTOs remain owned by `govoplan-tenancy`; access keeps +matching compatibility DTOs only for its legacy admin surface. Admin overview +responses remain module-local because the same counters are exposed from +different menu contexts and are not yet a separately versioned platform API. + ## Frontend Contract Policy UIs must: diff --git a/docs/RELEASE_DEPENDENCIES.md b/docs/RELEASE_DEPENDENCIES.md index 66d18bf..3b2f652 100644 --- a/docs/RELEASE_DEPENDENCIES.md +++ b/docs/RELEASE_DEPENDENCIES.md @@ -126,7 +126,6 @@ are not listed in `requirements-release.txt` or `webui/package.release.json`. Current tag-only module repositories: -- `govoplan-addresses` - `govoplan-appointments` - `govoplan-cases` - `govoplan-connectors` diff --git a/docs/UI_UX_DECISION_LEDGER.md b/docs/UI_UX_DECISION_LEDGER.md index 094b630..9c32f2c 100644 --- a/docs/UI_UX_DECISION_LEDGER.md +++ b/docs/UI_UX_DECISION_LEDGER.md @@ -38,6 +38,7 @@ contestability, responsibility, and traceability at the point of action. | UX-012 | Automated actions must remain inspectable. The UI must show the system actor, trigger, policy result, observed effects, and failure/manual-intervention state when automation changes administrative state. | Accepted | Workflow, automation, connectors, tasks, audit | | UX-013 | Contestable decisions must expose provenance. Denials, locks, generated outputs, calculated defaults, policy decisions, access decisions, and retention decisions need a reachable source path. | Accepted | Policy, access, templates, workflow, retention, records | | UX-014 | Retraction, expiry, undo, rollback, and delete controls must state the real limit of the operation. Corrective or future-only actions must not be described as if they undo already observed effects. | Accepted | Postbox, files, records, installer, workflow, payments | +| UX-015 | Core owns the platform appearance contract. Modules must use shared CSS tokens and shared controls for theme-aware UI; they must not define independent light/dark palette systems. | Accepted | Core shell and all module WebUIs | ## Confirmed Implementation Decisions @@ -138,6 +139,23 @@ adaptive form, not force a linear wizard. - Wizard shells remain available for assisted setup, first-run guidance, imports, discovery-heavy flows, and operational preflight workflows. +### DUE-008: Platform Theme Contract + +Decision: the WebUI shell exposes a small, stable appearance contract based on +shared CSS tokens and persisted user preference selection. + +- Core applies `system`, `light`, and `dark` preferences at the document root. +- Core owns shared tokens such as `--bg`, `--bar`, `--panel`, `--surface`, + `--line`, `--line-dark`, `--text`, `--text-strong`, `--muted`, semantic + status colors, radii, shadows, and disabled-control colors. +- Modules must style new UI with these tokens and shared controls. Module-local + CSS may tune layout and spacing, but it must not introduce a separate + appearance system. +- Appearance controls live in user settings first. Tenant defaults and policy + enforcement can be added later without changing the token contract. +- Visual preview in settings is illustrative; it must reflect token families, + not become a second theme implementation. + ## Implementation Sequence | Phase | Scope | Output | diff --git a/docs/audits/2026-07-09-dependency-audit.md b/docs/audits/2026-07-09-dependency-audit.md index adeb08b..6137843 100644 --- a/docs/audits/2026-07-09-dependency-audit.md +++ b/docs/audits/2026-07-09-dependency-audit.md @@ -45,7 +45,7 @@ Remediation applied on 2026-07-09: - upgraded the campaign ZIP dependency to `pyzipper>=0.4,<1`, resolving to `pyzipper==0.4.0` - upgraded the local audit environment to `pip==26.1.2` -- removed the obsolete local `govoplan-module-multimailer` editable install +- removed the obsolete local mailer-module editable install from the audit environment so the audit reflects the split module product Post-remediation result: diff --git a/src/govoplan_core/api/v1/schemas.py b/src/govoplan_core/api/v1/schemas.py index cbc2022..5cf7e4f 100644 --- a/src/govoplan_core/api/v1/schemas.py +++ b/src/govoplan_core/api/v1/schemas.py @@ -94,6 +94,28 @@ class UserInfo(BaseModel): ui_preferences: UserUiPreferences = Field(default_factory=UserUiPreferences) +class AuthSessionUserInfo(BaseModel): + id: str + account_id: str + email: str + display_name: str | None = None + tenant_display_name: str | None = None + is_tenant_admin: bool = False + password_reset_required: bool = False + + +class AuthSessionResponse(BaseModel): + authenticated: bool = True + auth_method: Literal["session", "api_key"] = "session" + user: AuthSessionUserInfo + # Backwards-compatible alias for the active tenant. + tenant: TenantInfo + active_tenant: TenantInfo + session_id: str | None = None + api_key_id: str | None = None + expires_at: datetime | None = None + + class ProfileUpdateRequest(BaseModel): model_config = ConfigDict(extra="forbid") @@ -143,6 +165,40 @@ class PrincipalContextInfo(BaseModel): display_name: str | None = None +class AuthShellResponse(BaseModel): + user: UserInfo + # Backwards-compatible alias for the active tenant. + tenant: TenantInfo + active_tenant: TenantInfo + tenants: list[TenantMembershipInfo] = Field(default_factory=list) + scopes: list[str] + principal: PrincipalContextInfo | None = None + profile_loaded: bool = False + roles_loaded: bool = False + groups_loaded: bool = False + + +class AuthProfileResponse(BaseModel): + user: UserInfo + # Backwards-compatible alias for the active tenant. + tenant: TenantInfo + active_tenant: TenantInfo + available_languages: list[LanguageInfo] = Field(default_factory=list) + enabled_language_codes: list[str] = Field(default_factory=list) + default_language: str = "en" + profile_loaded: bool = True + + +class AuthRolesResponse(BaseModel): + roles: list[RoleInfo] = Field(default_factory=list) + roles_loaded: bool = True + + +class AuthGroupsResponse(BaseModel): + groups: list[GroupInfo] = Field(default_factory=list) + groups_loaded: bool = True + + class LoginResponse(BaseModel): access_token: str token_type: str = "bearer" @@ -159,6 +215,9 @@ class LoginResponse(BaseModel): available_languages: list[LanguageInfo] = Field(default_factory=list) enabled_language_codes: list[str] = Field(default_factory=list) default_language: str = "en" + profile_loaded: bool = True + roles_loaded: bool = True + groups_loaded: bool = True class MeResponse(BaseModel): @@ -174,3 +233,6 @@ class MeResponse(BaseModel): available_languages: list[LanguageInfo] = Field(default_factory=list) enabled_language_codes: list[str] = Field(default_factory=list) default_language: str = "en" + profile_loaded: bool = True + roles_loaded: bool = True + groups_loaded: bool = True diff --git a/src/govoplan_core/celery_app.py b/src/govoplan_core/celery_app.py index ad29c60..45751a6 100644 --- a/src/govoplan_core/celery_app.py +++ b/src/govoplan_core/celery_app.py @@ -5,6 +5,8 @@ from celery import Celery from govoplan_core.core.campaigns import CAPABILITY_CAMPAIGNS_DELIVERY_TASKS, CampaignDeliveryTaskProvider from govoplan_core.core.module_management import load_startup_enabled_modules, startup_candidate_module_ids from govoplan_core.core.modules import ModuleContext +from govoplan_core.core.notifications import CAPABILITY_NOTIFICATIONS_DISPATCH, NotificationDispatchProvider +from govoplan_core.core.registry import PlatformRegistry from govoplan_core.core.runtime import configure_runtime from govoplan_core.settings import settings from govoplan_core.db.session import configure_database @@ -13,7 +15,7 @@ from govoplan_core.server.registry import available_module_manifests, build_plat configure_database(settings.database_url) celery = Celery( - "multimailer", + "govoplan", broker=settings.redis_url, backend=settings.redis_url, ) @@ -21,8 +23,10 @@ celery = Celery( celery.conf.update( task_default_queue="default", task_routes={ - "multimailer.send_email": {"queue": "send_email"}, - "multimailer.append_sent": {"queue": "append_sent"}, + "govoplan.campaigns.send_email": {"queue": "send_email"}, + "govoplan.campaigns.append_sent": {"queue": "append_sent"}, + "govoplan.notifications.deliver": {"queue": "notifications"}, + "govoplan.notifications.deliver_pending": {"queue": "notifications"}, }, worker_prefetch_multiplier=1, task_acks_late=True, @@ -30,12 +34,12 @@ celery.conf.update( ) -@celery.task(name="multimailer.ping") +@celery.task(name="govoplan.ping") def ping(): return "pong" -def _campaign_delivery_tasks() -> CampaignDeliveryTaskProvider: +def _platform_registry() -> PlatformRegistry: raw_enabled_modules = load_startup_enabled_modules(settings.enabled_modules) candidate_modules = startup_candidate_module_ids(settings.enabled_modules, raw_enabled_modules) available_modules = available_module_manifests(enabled_modules=candidate_modules, ignore_load_errors=True) @@ -44,13 +48,26 @@ def _campaign_delivery_tasks() -> CampaignDeliveryTaskProvider: context = ModuleContext(registry=registry, settings=settings) configure_runtime(context) registry.configure_capability_context(context) + return registry + + +def _campaign_delivery_tasks() -> CampaignDeliveryTaskProvider: + registry = _platform_registry() capability = registry.require_capability(CAPABILITY_CAMPAIGNS_DELIVERY_TASKS) if not isinstance(capability, CampaignDeliveryTaskProvider): raise RuntimeError("Campaign delivery task capability is invalid") return capability -@celery.task(name="multimailer.send_email", bind=True, max_retries=0) +def _notification_dispatch() -> NotificationDispatchProvider: + registry = _platform_registry() + capability = registry.require_capability(CAPABILITY_NOTIFICATIONS_DISPATCH) + if not isinstance(capability, NotificationDispatchProvider): + raise RuntimeError("Notification dispatch capability is invalid") + return capability + + +@celery.task(name="govoplan.campaigns.send_email", bind=True, max_retries=0) def send_email(self, job_id: str): """Send one explicitly queued campaign job. @@ -65,7 +82,7 @@ def send_email(self, job_id: str): return dict(_campaign_delivery_tasks().send_campaign_job(session, job_id=job_id, enqueue_imap_task=True)) -@celery.task(name="multimailer.append_sent", bind=True, max_retries=None) +@celery.task(name="govoplan.campaigns.append_sent", bind=True, max_retries=None) def append_sent(self, job_id: str): """Append the exact sent MIME to the configured IMAP Sent folder.""" @@ -78,3 +95,19 @@ def append_sent(self, job_id: str): if getattr(exc, "temporary", None) is True: raise self.retry(exc=exc, countdown=300) raise + + +@celery.task(name="govoplan.notifications.deliver", bind=True, max_retries=0) +def deliver_notification(self, notification_id: str): + from govoplan_core.db.session import get_database + + with get_database().SessionLocal() as session: + return dict(_notification_dispatch().deliver_notification(session, notification_id=notification_id)) + + +@celery.task(name="govoplan.notifications.deliver_pending", bind=True, max_retries=0) +def deliver_pending_notifications(self, tenant_id: str | None = None, limit: int = 50): + from govoplan_core.db.session import get_database + + with get_database().SessionLocal() as session: + return dict(_notification_dispatch().deliver_pending(session, tenant_id=tenant_id, limit=limit)) diff --git a/src/govoplan_core/commands/module_installer.py b/src/govoplan_core/commands/module_installer.py index 35c19b3..0ecbf89 100644 --- a/src/govoplan_core/commands/module_installer.py +++ b/src/govoplan_core/commands/module_installer.py @@ -5,7 +5,6 @@ import json from pathlib import Path import sys import time -from typing import Any from govoplan_core.core.module_installer import ( ModuleInstallerError, @@ -27,6 +26,13 @@ from govoplan_core.core.module_installer import ( update_module_installer_request, update_module_installer_daemon_status, ) +from govoplan_core.core.module_installer_notifications import ( + build_runtime_notification_registry, + emit_module_installer_notification, + installer_notification_body, + installer_notification_priority, + installer_notification_subject, +) from govoplan_core.core.module_license import issue_module_license, module_license_diagnostics from govoplan_core.core.module_package_catalog import sign_module_package_catalog, validate_module_package_catalog from govoplan_core.core.module_management import ( @@ -39,7 +45,7 @@ from govoplan_core.server.registry import available_module_manifests from govoplan_core.settings import settings -def main() -> int: +def _build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description="Preflight, apply, or roll back a GovOPlaN module package install plan.") parser.add_argument("--database-url", default=settings.database_url, help="Database URL containing system_settings.") parser.add_argument("--runtime-dir", type=Path, help="Directory for installer locks and run snapshots.") @@ -95,202 +101,295 @@ def main() -> int: parser.add_argument("--license-signing-private-key", type=Path, help="PEM Ed25519 private key for --issue-license.") parser.add_argument("--license-issuer", help="Optional issuer string for --issue-license.") parser.add_argument("--license-notes", help="Optional operator note for --issue-license.") - args = parser.parse_args() + return parser + +def main() -> int: + args = _build_parser().parse_args() runtime_dir = args.runtime_dir or default_installer_runtime_dir(args.database_url) try: - if args.issue_license: - if not args.license_id or not args.license_subject or not args.license_valid_until: - raise ModuleInstallerError("--issue-license requires --license-id, --license-subject, and --license-valid-until.") - if not args.license_signing_key_id or not args.license_signing_private_key: - raise ModuleInstallerError("--issue-license requires --license-signing-key-id and --license-signing-private-key.") - if not [item for item in args.license_feature if item.strip()]: - raise ModuleInstallerError("--issue-license requires at least one --license-feature.") - try: - path = issue_module_license( - path=args.issue_license, - license_id=args.license_id, - subject=args.license_subject, - features=args.license_feature, - valid_from=args.license_valid_from, - valid_until=args.license_valid_until, - signing_key_id=args.license_signing_key_id, - signing_private_key_path=args.license_signing_private_key, - issuer=args.license_issuer, - notes=args.license_notes, - ) - except (OSError, ValueError) as exc: - raise ModuleInstallerError(str(exc)) from exc - _print_result( - { - "issued": True, - "path": str(path), - "license_id": args.license_id, - "subject": args.license_subject, - "features": list(dict.fromkeys(item.strip() for item in args.license_feature if item.strip())), - "valid_until": args.license_valid_until, - "key_id": args.license_signing_key_id, - }, - output_format=args.format, - ) - return 0 - if args.validate_license is not None: - path = Path(args.validate_license).expanduser() if args.validate_license else None - result = module_license_diagnostics( - path, - require_trusted=args.require_trusted_license, - trusted_keys=_trusted_keys_from_args(args.license_trusted_key, flag_name="--license-trusted-key"), - required_features=args.license_required_feature, - ) - _print_result(result, output_format=args.format) - return 0 if result.get("valid") and not result.get("missing_features") else 1 - if args.sign_package_catalog: - if not args.catalog_signing_key_id or not args.catalog_signing_private_key: - raise ModuleInstallerError("--sign-package-catalog requires --catalog-signing-key-id and --catalog-signing-private-key.") - path = sign_module_package_catalog( - path=args.sign_package_catalog, - key_id=args.catalog_signing_key_id, - private_key_path=args.catalog_signing_private_key, - output_path=args.catalog_output, - ) - _print_result({"signed": True, "path": str(path), "key_id": args.catalog_signing_key_id}, output_format=args.format) - return 0 - if args.validate_package_catalog is not None: - path = Path(args.validate_package_catalog).expanduser() if args.validate_package_catalog else None - result = validate_module_package_catalog( - path, - require_trusted=args.require_signed_catalog, - approved_channels=tuple(args.approved_catalog_channel), - trusted_keys=_trusted_keys_from_args(args.catalog_trusted_key, flag_name="--catalog-trusted-key"), - ) - _print_result(result, output_format=args.format) - return 0 if result.get("valid") else 1 - if args.daemon_status: - _print_result(module_installer_daemon_status(runtime_dir=runtime_dir), output_format=args.format) - return 0 - if args.daemon or args.daemon_once: - return _run_daemon(args=args, runtime_dir=runtime_dir) - if args.list_requests: - _print_result({ - "requests": list(list_module_installer_requests(runtime_dir=runtime_dir)), - "daemon": module_installer_daemon_status(runtime_dir=runtime_dir), - }, output_format=args.format) - return 0 - if args.show_request: - _print_result(read_module_installer_request(runtime_dir=runtime_dir, request_id=args.show_request), output_format=args.format) - return 0 - if args.cancel_request: - _print_result(cancel_module_installer_request(runtime_dir=runtime_dir, request_id=args.cancel_request, cancelled_by="cli"), output_format=args.format) - return 0 - if args.retry_request: - _print_result(retry_module_installer_request(runtime_dir=runtime_dir, request_id=args.retry_request, requested_by="cli"), output_format=args.format) - return 0 - if args.enqueue_supervised: - request = queue_module_installer_request( - runtime_dir=runtime_dir, - requested_by="cli", - options=_request_options_from_args(args), - ) - _print_result(request, output_format=args.format) - return 0 - if args.list_runs: - _print_result({ - "runs": list(list_module_installer_runs(runtime_dir=runtime_dir)), - "lock": module_installer_lock_status(runtime_dir=runtime_dir), - }, output_format=args.format) - return 0 - if args.show_run: - _print_result(read_module_installer_run(runtime_dir=runtime_dir, run_id=args.show_run), output_format=args.format) - return 0 - if args.lock_status: - _print_result(module_installer_lock_status(runtime_dir=runtime_dir), output_format=args.format) - return 0 - if args.rollback: - result = rollback_module_install_run( - run_id=args.rollback, - runtime_dir=runtime_dir, - npm_bin=args.npm_bin, - build_webui=args.build_webui, - database_restore_command=args.database_restore_command, - database_url=str(args.database_url), - ) - _print_result(result.as_dict(), output_format=args.format) - return 0 if result.return_code == 0 else 1 - - configure_database(str(args.database_url)) - available = available_module_manifests(ignore_load_errors=True) - with get_database().session() as session: - configured = configured_enabled_modules(settings.enabled_modules) - desired = saved_desired_enabled_modules(session, configured) - plan = saved_module_install_plan(session) - if args.supervise: - result = supervise_module_install_plan( - session=session, - plan=plan, - available=available, - current_enabled=desired, - desired_enabled=desired, - database_url=str(args.database_url), - runtime_dir=runtime_dir, - webui_root=args.webui_root, - npm_bin=args.npm_bin, - build_webui=args.build_webui, - migrate_database=args.migrate, - database_backup_command=args.database_backup_command, - database_restore_command=args.database_restore_command, - database_restore_check_command=args.database_restore_check_command, - activate_installed_modules=not args.no_activate_installed_modules, - remove_uninstalled_modules_from_desired=not args.keep_uninstalled_modules_in_desired, - restart_command=args.restart_command, - health_url=args.health_url, - health_timeout_seconds=args.health_timeout_seconds, - health_interval_seconds=args.health_interval_seconds, - ) - _print_result(result.as_dict(), output_format=args.format) - return 0 if result.return_code == 0 else 1 - if args.apply or args.dry_run: - result = run_module_install_plan( - session=session, - plan=plan, - available=available, - current_enabled=desired, - desired_enabled=desired, - database_url=str(args.database_url), - runtime_dir=runtime_dir, - webui_root=args.webui_root, - npm_bin=args.npm_bin, - build_webui=args.build_webui, - migrate_database=args.migrate, - database_backup_command=args.database_backup_command, - database_restore_command=args.database_restore_command, - database_restore_check_command=args.database_restore_check_command, - activate_installed_modules=not args.no_activate_installed_modules, - remove_uninstalled_modules_from_desired=not args.keep_uninstalled_modules_in_desired, - dry_run=args.dry_run, - ) - _print_result(result.as_dict(), output_format=args.format) - return 0 if result.return_code == 0 else 1 - - from govoplan_core.core.maintenance import saved_maintenance_mode - - maintenance = saved_maintenance_mode(session) - preflight = module_install_preflight( - plan=plan, - available=available, - current_enabled=desired, - desired_enabled=desired, - maintenance_mode=maintenance.enabled, - session=session, - webui_root=args.webui_root, - runtime_dir=runtime_dir, - ) - _print_preflight(preflight.as_dict(), output_format=args.format) - return 0 if preflight.allowed else 1 + return _dispatch_command(args=args, runtime_dir=runtime_dir) except ModuleInstallerError as exc: print(f"error: {exc}", file=sys.stderr) return 1 +def _dispatch_command(*, args: argparse.Namespace, runtime_dir: Path) -> int: + for handler in ( + _handle_license_command, + _handle_catalog_command, + _handle_runtime_command, + _handle_queue_command, + _handle_run_history_command, + ): + result = handler(args=args, runtime_dir=runtime_dir) + if result is not None: + return result + return _handle_install_command(args=args, runtime_dir=runtime_dir) + + +def _handle_license_command(*, args: argparse.Namespace, runtime_dir: Path) -> int | None: + del runtime_dir + if args.issue_license: + return _issue_license_command(args) + if args.validate_license is not None: + return _validate_license_command(args) + return None + + +def _issue_license_command(args: argparse.Namespace) -> int: + if not args.license_id or not args.license_subject or not args.license_valid_until: + raise ModuleInstallerError("--issue-license requires --license-id, --license-subject, and --license-valid-until.") + if not args.license_signing_key_id or not args.license_signing_private_key: + raise ModuleInstallerError("--issue-license requires --license-signing-key-id and --license-signing-private-key.") + if not [item for item in args.license_feature if item.strip()]: + raise ModuleInstallerError("--issue-license requires at least one --license-feature.") + try: + path = issue_module_license( + path=args.issue_license, + license_id=args.license_id, + subject=args.license_subject, + features=args.license_feature, + valid_from=args.license_valid_from, + valid_until=args.license_valid_until, + signing_key_id=args.license_signing_key_id, + signing_private_key_path=args.license_signing_private_key, + issuer=args.license_issuer, + notes=args.license_notes, + ) + except (OSError, ValueError) as exc: + raise ModuleInstallerError(str(exc)) from exc + _print_result( + { + "issued": True, + "path": str(path), + "license_id": args.license_id, + "subject": args.license_subject, + "features": list(dict.fromkeys(item.strip() for item in args.license_feature if item.strip())), + "valid_until": args.license_valid_until, + "key_id": args.license_signing_key_id, + }, + output_format=args.format, + ) + return 0 + + +def _validate_license_command(args: argparse.Namespace) -> int: + path = Path(args.validate_license).expanduser() if args.validate_license else None + result = module_license_diagnostics( + path, + require_trusted=args.require_trusted_license, + trusted_keys=_trusted_keys_from_args(args.license_trusted_key, flag_name="--license-trusted-key"), + required_features=args.license_required_feature, + ) + _print_result(result, output_format=args.format) + return 0 if result.get("valid") and not result.get("missing_features") else 1 + + +def _handle_catalog_command(*, args: argparse.Namespace, runtime_dir: Path) -> int | None: + del runtime_dir + if args.sign_package_catalog: + return _sign_package_catalog_command(args) + if args.validate_package_catalog is not None: + return _validate_package_catalog_command(args) + return None + + +def _sign_package_catalog_command(args: argparse.Namespace) -> int: + if not args.catalog_signing_key_id or not args.catalog_signing_private_key: + raise ModuleInstallerError("--sign-package-catalog requires --catalog-signing-key-id and --catalog-signing-private-key.") + path = sign_module_package_catalog( + path=args.sign_package_catalog, + key_id=args.catalog_signing_key_id, + private_key_path=args.catalog_signing_private_key, + output_path=args.catalog_output, + ) + _print_result({"signed": True, "path": str(path), "key_id": args.catalog_signing_key_id}, output_format=args.format) + return 0 + + +def _validate_package_catalog_command(args: argparse.Namespace) -> int: + path = Path(args.validate_package_catalog).expanduser() if args.validate_package_catalog else None + result = validate_module_package_catalog( + path, + require_trusted=args.require_signed_catalog, + approved_channels=tuple(args.approved_catalog_channel), + trusted_keys=_trusted_keys_from_args(args.catalog_trusted_key, flag_name="--catalog-trusted-key"), + ) + _print_result(result, output_format=args.format) + return 0 if result.get("valid") else 1 + + +def _handle_runtime_command(*, args: argparse.Namespace, runtime_dir: Path) -> int | None: + if args.daemon_status: + _print_result(module_installer_daemon_status(runtime_dir=runtime_dir), output_format=args.format) + return 0 + if args.daemon or args.daemon_once: + return _run_daemon(args=args, runtime_dir=runtime_dir) + return None + + +def _handle_queue_command(*, args: argparse.Namespace, runtime_dir: Path) -> int | None: + if args.list_requests: + _print_result({ + "requests": list(list_module_installer_requests(runtime_dir=runtime_dir)), + "daemon": module_installer_daemon_status(runtime_dir=runtime_dir), + }, output_format=args.format) + return 0 + if args.show_request: + _print_result(read_module_installer_request(runtime_dir=runtime_dir, request_id=args.show_request), output_format=args.format) + return 0 + if args.cancel_request: + _print_result(cancel_module_installer_request(runtime_dir=runtime_dir, request_id=args.cancel_request, cancelled_by="cli"), output_format=args.format) + return 0 + if args.retry_request: + _print_result(retry_module_installer_request(runtime_dir=runtime_dir, request_id=args.retry_request, requested_by="cli"), output_format=args.format) + return 0 + if args.enqueue_supervised: + request = queue_module_installer_request(runtime_dir=runtime_dir, requested_by="cli", options=_request_options_from_args(args)) + _print_result(request, output_format=args.format) + return 0 + return None + + +def _handle_run_history_command(*, args: argparse.Namespace, runtime_dir: Path) -> int | None: + if args.list_runs: + _print_result({ + "runs": list(list_module_installer_runs(runtime_dir=runtime_dir)), + "lock": module_installer_lock_status(runtime_dir=runtime_dir), + }, output_format=args.format) + return 0 + if args.show_run: + _print_result(read_module_installer_run(runtime_dir=runtime_dir, run_id=args.show_run), output_format=args.format) + return 0 + if args.lock_status: + _print_result(module_installer_lock_status(runtime_dir=runtime_dir), output_format=args.format) + return 0 + if args.rollback: + return _rollback_command(args=args, runtime_dir=runtime_dir) + return None + + +def _rollback_command(*, args: argparse.Namespace, runtime_dir: Path) -> int: + result = rollback_module_install_run( + run_id=args.rollback, + runtime_dir=runtime_dir, + npm_bin=args.npm_bin, + build_webui=args.build_webui, + database_restore_command=args.database_restore_command, + database_url=str(args.database_url), + ) + _print_result(result.as_dict(), output_format=args.format) + return 0 if result.return_code == 0 else 1 + + +def _handle_install_command(*, args: argparse.Namespace, runtime_dir: Path) -> int: + configure_database(str(args.database_url)) + available = available_module_manifests(ignore_load_errors=True) + with get_database().session() as session: + configured = configured_enabled_modules(settings.enabled_modules) + desired = saved_desired_enabled_modules(session, configured) + plan = saved_module_install_plan(session) + if args.supervise: + return _supervise_install_command(args=args, runtime_dir=runtime_dir, session=session, available=available, desired=desired, plan=plan) + if args.apply or args.dry_run: + return _apply_install_command(args=args, runtime_dir=runtime_dir, session=session, available=available, desired=desired, plan=plan) + return _preflight_install_command(args=args, runtime_dir=runtime_dir, session=session, available=available, desired=desired, plan=plan) + + +def _supervise_install_command( + *, + args: argparse.Namespace, + runtime_dir: Path, + session: object, + available: object, + desired: object, + plan: object, +) -> int: + result = supervise_module_install_plan( + session=session, + plan=plan, + available=available, + current_enabled=desired, + desired_enabled=desired, + database_url=str(args.database_url), + runtime_dir=runtime_dir, + webui_root=args.webui_root, + npm_bin=args.npm_bin, + build_webui=args.build_webui, + migrate_database=args.migrate, + database_backup_command=args.database_backup_command, + database_restore_command=args.database_restore_command, + database_restore_check_command=args.database_restore_check_command, + activate_installed_modules=not args.no_activate_installed_modules, + remove_uninstalled_modules_from_desired=not args.keep_uninstalled_modules_in_desired, + restart_command=args.restart_command, + health_url=args.health_url, + health_timeout_seconds=args.health_timeout_seconds, + health_interval_seconds=args.health_interval_seconds, + ) + _print_result(result.as_dict(), output_format=args.format) + return 0 if result.return_code == 0 else 1 + + +def _apply_install_command( + *, + args: argparse.Namespace, + runtime_dir: Path, + session: object, + available: object, + desired: object, + plan: object, +) -> int: + result = run_module_install_plan( + session=session, + plan=plan, + available=available, + current_enabled=desired, + desired_enabled=desired, + database_url=str(args.database_url), + runtime_dir=runtime_dir, + webui_root=args.webui_root, + npm_bin=args.npm_bin, + build_webui=args.build_webui, + migrate_database=args.migrate, + database_backup_command=args.database_backup_command, + database_restore_command=args.database_restore_command, + database_restore_check_command=args.database_restore_check_command, + activate_installed_modules=not args.no_activate_installed_modules, + remove_uninstalled_modules_from_desired=not args.keep_uninstalled_modules_in_desired, + dry_run=args.dry_run, + ) + _print_result(result.as_dict(), output_format=args.format) + return 0 if result.return_code == 0 else 1 + + +def _preflight_install_command( + *, + args: argparse.Namespace, + runtime_dir: Path, + session: object, + available: object, + desired: object, + plan: object, +) -> int: + from govoplan_core.core.maintenance import saved_maintenance_mode + + maintenance = saved_maintenance_mode(session) + preflight = module_install_preflight( + plan=plan, + available=available, + current_enabled=desired, + desired_enabled=desired, + maintenance_mode=maintenance.enabled, + session=session, + webui_root=args.webui_root, + runtime_dir=runtime_dir, + ) + _print_preflight(preflight.as_dict(), output_format=args.format) + return 0 if preflight.allowed else 1 + + def _default_webui_root() -> Path: return Path(__file__).resolve().parents[3] / "webui" @@ -355,6 +454,7 @@ def _process_request(*, request: dict[str, object], args: argparse.Namespace, ru request_id = str(request["request_id"]) options = request.get("options") if isinstance(request.get("options"), dict) else {} configure_database(str(args.database_url)) + _emit_installer_daemon_notification(request, event_kind="module_installer.request.running") try: available = available_module_manifests(ignore_load_errors=True) with get_database().session() as session: @@ -384,7 +484,7 @@ def _process_request(*, request: dict[str, object], args: argparse.Namespace, ru health_interval_seconds=_float_option(options, "health_interval_seconds", args.health_interval_seconds), request_context=_request_context(request), ) - update_module_installer_request( + updated_request = update_module_installer_request( runtime_dir=runtime_dir, request_id=request_id, patch={ @@ -393,8 +493,12 @@ def _process_request(*, request: dict[str, object], args: argparse.Namespace, ru "result": result.as_dict(), }, ) + _emit_installer_daemon_notification( + updated_request, + event_kind="module_installer.request.completed" if result.return_code == 0 else "module_installer.request.failed", + ) except Exception as exc: - update_module_installer_request( + updated_request = update_module_installer_request( runtime_dir=runtime_dir, request_id=request_id, patch={ @@ -403,6 +507,34 @@ def _process_request(*, request: dict[str, object], args: argparse.Namespace, ru "error": str(exc), }, ) + _emit_installer_daemon_notification(updated_request, event_kind="module_installer.request.failed") + + +def _emit_installer_daemon_notification(request: dict[str, object], *, event_kind: str) -> None: + tenant_id = str(request.get("tenant_id") or "") + if not tenant_id: + return + registry = build_runtime_notification_registry(settings) + if registry is None: + return + status_value = str(request.get("status") or event_kind.rsplit(".", 1)[-1]) + try: + with get_database().SessionLocal() as session: + emitted = emit_module_installer_notification( + session=session, + registry=registry, + tenant_id=tenant_id, + request=request, + event_kind=event_kind, + subject=installer_notification_subject(event_kind, request), + body_text=installer_notification_body(event_kind, request), + recipient_id=str(request.get("requested_by") or "") or None, + priority=installer_notification_priority(status_value), + ) + if emitted: + session.commit() + except Exception: # noqa: BLE001 - notification bridge must not block installer work. + return def _request_options_from_args(args: argparse.Namespace) -> dict[str, object]: @@ -427,6 +559,7 @@ def _request_context(request: dict[str, object]) -> dict[str, object]: context: dict[str, object] = { "request_id": request.get("request_id"), "requested_by": request.get("requested_by"), + "tenant_id": request.get("tenant_id"), } if request.get("retry_of"): context["retry_of"] = request["retry_of"] diff --git a/src/govoplan_core/core/configuration_packages.py b/src/govoplan_core/core/configuration_packages.py index ba514b0..5995894 100644 --- a/src/govoplan_core/core/configuration_packages.py +++ b/src/govoplan_core/core/configuration_packages.py @@ -8,8 +8,6 @@ from pathlib import Path import json import os from typing import Any, Literal, Protocol, runtime_checkable -import urllib.error -import urllib.request from govoplan_core.core.module_package_catalog import ( _canonical_catalog_bytes, @@ -23,6 +21,7 @@ from govoplan_core.core.module_package_catalog import ( _load_private_key, _parse_trusted_keys, ) +from govoplan_core.security.http_fetch import fetch_http_text CONFIGURATION_PROVIDER_CAPABILITY = "configuration.provider" @@ -31,6 +30,20 @@ DiagnosticSeverity = Literal["blocker", "warning", "info"] PlanAction = Literal["create", "update", "bind", "skip", "blocked", "noop"] +@dataclass(frozen=True, slots=True) +class _ConfigurationCatalogValidationState: + packages: tuple[dict[str, object], ...] + channel: str | None + sequence: int | None + generated_at: str | None + not_before: str | None + expires_at: str | None + signature_state: dict[str, object] + freshness: dict[str, object] + replay: dict[str, object] + read_state: dict[str, object] + + @dataclass(frozen=True, slots=True) class ConfigurationModuleRequirement: module_id: str @@ -461,55 +474,148 @@ def validate_configuration_package_catalog( configured = source is not None and _catalog_source_exists(source) if source is not None and not _catalog_source_exists(source): return _validation_result(source, configured=True, packages=(), error=f"Configuration package catalog does not exist: {source}") - read_state = {"cache_used": False, "cache_path": str(_configured_catalog_cache_path()) if _configured_catalog_cache_path() else None} + read_state = _configuration_catalog_default_read_state() try: - payload, read_state = _read_catalog_payload_with_metadata(source) - packages = _normalize_catalog_packages(payload) - channel = _catalog_channel(payload) - sequence = _catalog_sequence(payload) - generated_at = _catalog_optional_text(payload, "generated_at") - not_before = _catalog_optional_text(payload, "not_before") - expires_at = _catalog_optional_text(payload, "expires_at") effective_require_trusted = _configured_require_signature() if require_trusted is None else require_trusted effective_approved_channels = _configured_approved_channels() if approved_channels is None else approved_channels effective_trusted_keys = trusted_keys if trusted_keys is not None else _configured_trusted_keys() - signature_state = _catalog_signature_state(payload, trusted_keys=effective_trusted_keys) - freshness = _catalog_freshness_state(payload) - replay = _catalog_replay_state(channel=channel, sequence=sequence) + state = _configuration_catalog_validation_state(source, trusted_keys=effective_trusted_keys) + read_state = state.read_state except Exception as exc: return _validation_result(source, configured=configured, read_state=read_state, packages=(), error=str(exc)) - if signature_state.get("fatal"): - return _validation_result(source, configured=configured, read_state=read_state, packages=(), channel=channel, sequence=sequence, generated_at=generated_at, not_before=not_before, expires_at=expires_at, signature_state=signature_state, error=str(signature_state["error"])) - if effective_approved_channels and channel not in effective_approved_channels: - return _validation_result(source, configured=configured, read_state=read_state, packages=(), channel=channel, sequence=sequence, generated_at=generated_at, not_before=not_before, expires_at=expires_at, signature_state=signature_state, error=f"Configuration package catalog channel {channel!r} is not approved. Approved channels: {', '.join(effective_approved_channels)}.") - if effective_require_trusted and not signature_state["trusted"]: - return _validation_result(source, configured=configured, read_state=read_state, packages=(), channel=channel, sequence=sequence, generated_at=generated_at, not_before=not_before, expires_at=expires_at, signature_state=signature_state, error=str(signature_state["error"] or "Configuration package catalog must be signed by a trusted key.")) - if not freshness["valid"]: - return _validation_result(source, configured=configured, read_state=read_state, packages=(), channel=channel, sequence=sequence, generated_at=generated_at, not_before=not_before, expires_at=expires_at, signature_state=signature_state, error=str(freshness["error"])) - if not replay["valid"]: - return _validation_result(source, configured=configured, read_state=read_state, packages=(), channel=channel, sequence=sequence, generated_at=generated_at, not_before=not_before, expires_at=expires_at, signature_state=signature_state, error=str(replay["error"])) - warnings = [str(item) for item in freshness.get("warnings", ()) if item] - warnings.extend(str(item) for item in replay.get("warnings", ()) if item) - if not signature_state["signed"]: - warnings.append("Configuration package catalog is unsigned; use only for local development unless signature enforcement is disabled intentionally.") - elif not signature_state["trusted"]: - warnings.append(str(signature_state["error"] or "Catalog signature could not be verified against a trusted key.")) + policy_error = _configuration_catalog_policy_error( + source, + configured=configured, + state=state, + require_trusted=effective_require_trusted, + approved_channels=effective_approved_channels, + ) + if policy_error is not None: + return policy_error return _validation_result( source, valid=True, configured=configured, - read_state=read_state, - packages=packages, + read_state=state.read_state, + packages=state.packages, + channel=state.channel, + sequence=state.sequence, + generated_at=state.generated_at, + not_before=state.not_before, + expires_at=state.expires_at, + signature_state=state.signature_state, + warnings=_configuration_catalog_warnings(state), + ) + + +def _configuration_catalog_validation_state( + source: Path | str | None, + *, + trusted_keys: dict[str, str], +) -> _ConfigurationCatalogValidationState: + payload, read_state = _read_catalog_payload_with_metadata(source) + channel = _catalog_channel(payload) + sequence = _catalog_sequence(payload) + return _ConfigurationCatalogValidationState( + packages=_normalize_catalog_packages(payload), channel=channel, sequence=sequence, - generated_at=generated_at, - not_before=not_before, - expires_at=expires_at, - signature_state=signature_state, - warnings=warnings, + generated_at=_catalog_optional_text(payload, "generated_at"), + not_before=_catalog_optional_text(payload, "not_before"), + expires_at=_catalog_optional_text(payload, "expires_at"), + signature_state=_catalog_signature_state(payload, trusted_keys=trusted_keys), + freshness=_catalog_freshness_state(payload), + replay=_catalog_replay_state(channel=channel, sequence=sequence), + read_state=read_state, ) +def _configuration_catalog_policy_error( + source: Path | str | None, + *, + configured: bool, + state: _ConfigurationCatalogValidationState, + require_trusted: bool, + approved_channels: tuple[str, ...], +) -> dict[str, object] | None: + if state.signature_state.get("fatal"): + return _configuration_catalog_invalid_result( + source, + configured=configured, + state=state, + error=str(state.signature_state["error"]), + ) + if approved_channels and state.channel not in approved_channels: + return _configuration_catalog_invalid_result( + source, + configured=configured, + state=state, + error=( + f"Configuration package catalog channel {state.channel!r} is not approved. " + f"Approved channels: {', '.join(approved_channels)}." + ), + ) + if require_trusted and not state.signature_state["trusted"]: + return _configuration_catalog_invalid_result( + source, + configured=configured, + state=state, + error=str(state.signature_state["error"] or "Configuration package catalog must be signed by a trusted key."), + ) + if not state.freshness["valid"]: + return _configuration_catalog_invalid_result( + source, + configured=configured, + state=state, + error=str(state.freshness["error"]), + ) + if not state.replay["valid"]: + return _configuration_catalog_invalid_result( + source, + configured=configured, + state=state, + error=str(state.replay["error"]), + ) + return None + + +def _configuration_catalog_invalid_result( + source: Path | str | None, + *, + configured: bool, + state: _ConfigurationCatalogValidationState, + error: str, +) -> dict[str, object]: + return _validation_result( + source, + configured=configured, + read_state=state.read_state, + packages=(), + channel=state.channel, + sequence=state.sequence, + generated_at=state.generated_at, + not_before=state.not_before, + expires_at=state.expires_at, + signature_state=state.signature_state, + error=error, + ) + + +def _configuration_catalog_warnings(state: _ConfigurationCatalogValidationState) -> list[str]: + warnings = [str(item) for item in state.freshness.get("warnings", ()) if item] + warnings.extend(str(item) for item in state.replay.get("warnings", ()) if item) + if not state.signature_state["signed"]: + warnings.append("Configuration package catalog is unsigned; use only for local development unless signature enforcement is disabled intentionally.") + elif not state.signature_state["trusted"]: + warnings.append(str(state.signature_state["error"] or "Catalog signature could not be verified against a trusted key.")) + return warnings + + +def _configuration_catalog_default_read_state() -> dict[str, object]: + cache_path = _configured_catalog_cache_path() + return {"cache_used": False, "cache_path": str(cache_path) if cache_path else None} + + def sign_configuration_package_catalog(*, path: Path, key_id: str, private_key_path: Path, output_path: Path | None = None) -> Path: payload = _read_catalog_payload(path) if not isinstance(payload, dict): @@ -686,17 +792,14 @@ def _configured_trusted_keys_cache_path() -> Path | None: def _read_trusted_keys_url(url: str) -> str: - if not _is_http_url(url): - raise ValueError("Trusted configuration catalog key URL must use http:// or https://.") cache_path = _configured_trusted_keys_cache_path() try: - with urllib.request.urlopen(url, timeout=15) as response: # noqa: S310 - validated configuration key HTTP(S) URL. # nosemgrep: python.lang.security.audit.dynamic-urllib-use-detected.dynamic-urllib-use-detected - body = response.read().decode("utf-8") + body = fetch_http_text(url, timeout=15, label="Trusted configuration catalog key URL") if cache_path is not None: cache_path.parent.mkdir(parents=True, exist_ok=True) cache_path.write_text(body, encoding="utf-8") return body - except (OSError, urllib.error.URLError): + except OSError: if cache_path is not None and cache_path.exists(): return cache_path.read_text(encoding="utf-8") raise @@ -714,13 +817,12 @@ def _read_catalog_payload_with_metadata(source: Path | str | None) -> tuple[obje return {"packages": []}, metadata if isinstance(source, str) and _is_http_url(source): try: - with urllib.request.urlopen(source, timeout=15) as response: # noqa: S310 - validated configuration catalog HTTP(S) URL. # nosemgrep: python.lang.security.audit.dynamic-urllib-use-detected.dynamic-urllib-use-detected - body = response.read().decode("utf-8") + body = fetch_http_text(source, timeout=15, label="Configuration package catalog URL") if cache_path is not None: cache_path.parent.mkdir(parents=True, exist_ok=True) cache_path.write_text(body, encoding="utf-8") return json.loads(body), metadata - except (OSError, urllib.error.URLError): + except OSError: if cache_path is not None and cache_path.exists(): metadata["cache_used"] = True return json.loads(cache_path.read_text(encoding="utf-8")), metadata diff --git a/src/govoplan_core/core/configuration_safety.py b/src/govoplan_core/core/configuration_safety.py index a7f42d9..bfd54d8 100644 --- a/src/govoplan_core/core/configuration_safety.py +++ b/src/govoplan_core/core/configuration_safety.py @@ -97,6 +97,16 @@ class ConfigurationChangeSafetyPlan: } +@dataclass(frozen=True, slots=True) +class _ConfigurationChangeSafetyState: + field: ConfigurationFieldSafety + missing_scopes: tuple[str, ...] + blockers: tuple[str, ...] + warnings: tuple[str, ...] + approval_required: bool + approval_satisfied: bool + + _CONFIGURATION_FIELD_SAFETY: tuple[ConfigurationFieldSafety, ...] = ( ConfigurationFieldSafety( key="module_management.desired_enabled", @@ -339,23 +349,56 @@ def plan_configuration_change( ) -> ConfigurationChangeSafetyPlan: field = classify_configuration_field(key) if field is None: - return ConfigurationChangeSafetyPlan( - key=key, - allowed=False, - field=None, - blockers=("unknown_configuration_field",), - policy_explanation="This setting is not in the configuration safety catalog.", - ) + return _unknown_configuration_plan(key) if not include_env_only and not field.ui_managed: - return ConfigurationChangeSafetyPlan( - key=key, - allowed=False, - field=field, - risk=field.risk, - secret_handling=field.secret_handling, - blockers=("deployment_managed",), - policy_explanation="This setting is deployment-managed and cannot be edited through the UI.", - ) + return _deployment_managed_configuration_plan(key, field) + state = _configuration_change_safety_state( + field, + actor_scopes=actor_scopes, + value=value, + dry_run=dry_run, + maintenance_mode=maintenance_mode, + approval_count=approval_count, + ) + return _configuration_change_safety_plan( + field, + state=state, + dry_run=dry_run, + maintenance_mode=maintenance_mode, + ) + + +def _unknown_configuration_plan(key: str) -> ConfigurationChangeSafetyPlan: + return ConfigurationChangeSafetyPlan( + key=key, + allowed=False, + field=None, + blockers=("unknown_configuration_field",), + policy_explanation="This setting is not in the configuration safety catalog.", + ) + + +def _deployment_managed_configuration_plan(key: str, field: ConfigurationFieldSafety) -> ConfigurationChangeSafetyPlan: + return ConfigurationChangeSafetyPlan( + key=key, + allowed=False, + field=field, + risk=field.risk, + secret_handling=field.secret_handling, + blockers=("deployment_managed",), + policy_explanation="This setting is deployment-managed and cannot be edited through the UI.", + ) + + +def _configuration_change_safety_state( + field: ConfigurationFieldSafety, + *, + actor_scopes: tuple[str, ...] | list[str], + value: object, + dry_run: bool, + maintenance_mode: bool, + approval_count: int, +) -> _ConfigurationChangeSafetyState: blockers: list[str] = [] warnings: list[str] = [] missing_scopes = tuple(scope for scope in field.required_scopes if not scopes_grant(actor_scopes, scope)) @@ -377,24 +420,41 @@ def plan_configuration_change( blockers.append("env_only_secret") if field.rollback_history_required: warnings.append("rollback_history_required") - return ConfigurationChangeSafetyPlan( - key=field.key, - allowed=not blockers, + return _ConfigurationChangeSafetyState( field=field, - risk=field.risk, missing_scopes=missing_scopes, - dry_run_required=field.dry_run_required, - dry_run_satisfied=not field.dry_run_required or dry_run, + blockers=tuple(dict.fromkeys(blockers)), + warnings=tuple(dict.fromkeys(warnings)), approval_required=approval_required, approval_satisfied=approval_satisfied, + ) + + +def _configuration_change_safety_plan( + field: ConfigurationFieldSafety, + *, + state: _ConfigurationChangeSafetyState, + dry_run: bool, + maintenance_mode: bool, +) -> ConfigurationChangeSafetyPlan: + return ConfigurationChangeSafetyPlan( + key=field.key, + allowed=not state.blockers, + field=field, + risk=field.risk, + missing_scopes=state.missing_scopes, + dry_run_required=field.dry_run_required, + dry_run_satisfied=not field.dry_run_required or dry_run, + approval_required=state.approval_required, + approval_satisfied=state.approval_satisfied, maintenance_required=field.maintenance_required, maintenance_satisfied=not field.maintenance_required or maintenance_mode, rollback_history_required=field.rollback_history_required, secret_handling=field.secret_handling, audit_event=field.audit_event, policy_explanation=_policy_explanation(field), - blockers=tuple(dict.fromkeys(blockers)), - warnings=tuple(dict.fromkeys(warnings)), + blockers=state.blockers, + warnings=state.warnings, ) diff --git a/src/govoplan_core/core/install_config.py b/src/govoplan_core/core/install_config.py index 40de016..784cf40 100644 --- a/src/govoplan_core/core/install_config.py +++ b/src/govoplan_core/core/install_config.py @@ -72,6 +72,25 @@ class ConfigValidationResult: return "\n".join(lines) +@dataclass(frozen=True, slots=True) +class _RuntimeProfile: + name: str + production: bool + production_like: bool + local: bool + + +@dataclass(slots=True) +class _ConfigIssueCollector: + strict: bool + issues: list[ConfigIssue] + + def add(self, level: ConfigIssueLevel, key: str, message: str, action: str) -> None: + if self.strict and level == "warning": + level = "error" + self.issues.append(ConfigIssue(level=level, key=key, message=message, action=action)) + + _LOCAL_PROFILES = {"dev", "local", "local-dev", "test"} _PRODUCTION_PROFILES = {"prod", "production", "self-hosted"} _PRODUCTION_LIKE_PROFILES = {"staging", "production-like", "production-like-dev"} @@ -114,95 +133,130 @@ def validate_runtime_configuration( strict: bool = False, ) -> ConfigValidationResult: env = dict(os.environ if environ is None else environ) - clean_profile = normalize_install_profile(profile or env.get("GOVOPLAN_INSTALL_PROFILE") or env.get("APP_ENV")) - issues: list[ConfigIssue] = [] + runtime = _runtime_profile(env, profile=profile) + collector = _ConfigIssueCollector(strict=strict, issues=[]) + _validate_app_env(env, runtime, collector) + _validate_database_settings(env, runtime, collector) + _validate_master_key(env, runtime, collector) + _validate_enabled_modules(env, runtime, collector) + _validate_async_and_auth_settings(env, runtime, collector) + _validate_cors_settings(env, runtime, collector) + _validate_file_storage_settings(env, runtime, collector) + _validate_module_catalog_trust(env, runtime, collector) + return ConfigValidationResult(profile=runtime.name, issues=tuple(collector.issues)) + +def _runtime_profile(env: Mapping[str, str], *, profile: str | None) -> _RuntimeProfile: + clean_profile = normalize_install_profile(profile or env.get("GOVOPLAN_INSTALL_PROFILE") or env.get("APP_ENV")) production = clean_profile in _PRODUCTION_PROFILES or env.get("APP_ENV", "").strip().lower() in {"prod", "production"} production_like = production or clean_profile in _PRODUCTION_LIKE_PROFILES - local = clean_profile in _LOCAL_PROFILES and not production_like + return _RuntimeProfile( + name=clean_profile, + production=production, + production_like=production_like, + local=clean_profile in _LOCAL_PROFILES and not production_like, + ) - def issue(level: ConfigIssueLevel, key: str, message: str, action: str) -> None: - if strict and level == "warning": - level = "error" - issues.append(ConfigIssue(level=level, key=key, message=message, action=action)) +def _validate_app_env(env: Mapping[str, str], runtime: _RuntimeProfile, collector: _ConfigIssueCollector) -> None: app_env = _clean(env.get("APP_ENV")) - if not app_env and production_like: - issue("error", "APP_ENV", "APP_ENV is missing for a production-like install.", "Set APP_ENV=staging, APP_ENV=production, or another explicit deployment profile.") - elif app_env.lower() in {"dev", "test", "local"} and production_like: - issue("error", "APP_ENV", f"APP_ENV={app_env!r} is not valid for profile {clean_profile!r}.", "Use APP_ENV=staging for production-like testing or APP_ENV=production for a real deployment.") + if not app_env and runtime.production_like: + collector.add("error", "APP_ENV", "APP_ENV is missing for a production-like install.", "Set APP_ENV=staging, APP_ENV=production, or another explicit deployment profile.") + elif app_env.lower() in {"dev", "test", "local"} and runtime.production_like: + collector.add("error", "APP_ENV", f"APP_ENV={app_env!r} is not valid for profile {runtime.name!r}.", "Use APP_ENV=staging for production-like testing or APP_ENV=production for a real deployment.") + +def _validate_database_settings(env: Mapping[str, str], runtime: _RuntimeProfile, collector: _ConfigIssueCollector) -> None: database_url = _clean(env.get("DATABASE_URL")) if not database_url: - issue("error", "DATABASE_URL", "DATABASE_URL is missing.", "Set DATABASE_URL to the PostgreSQL SQLAlchemy URL used by the API and modules.") - else: - backend = _database_backend(database_url) - if backend is None: - issue("error", "DATABASE_URL", "DATABASE_URL is not a valid SQLAlchemy URL.", "Use a value like postgresql+psycopg://user:password@host:5432/database.") - elif backend == "sqlite" and production_like: - issue("error", "DATABASE_URL", "SQLite is only supported for disposable local development.", "Use PostgreSQL for production-like and self-hosted installs.") - elif backend != "postgresql" and production: - issue("warning", "DATABASE_URL", f"Database backend {backend!r} is not the preferred production target.", "Use PostgreSQL unless this deployment has an explicit support decision.") - if backend == "postgresql" and not _clean(env.get("GOVOPLAN_DATABASE_URL_PGTOOLS")): - issue("warning", "GOVOPLAN_DATABASE_URL_PGTOOLS", "PostgreSQL backup/restore tools URL is missing.", "Set GOVOPLAN_DATABASE_URL_PGTOOLS to the same database without the SQLAlchemy driver marker, for example postgresql://user:password@host:5432/database.") + collector.add("error", "DATABASE_URL", "DATABASE_URL is missing.", "Set DATABASE_URL to the PostgreSQL SQLAlchemy URL used by the API and modules.") + return + backend = _database_backend(database_url) + if backend is None: + collector.add("error", "DATABASE_URL", "DATABASE_URL is not a valid SQLAlchemy URL.", "Use a value like postgresql+psycopg://user:password@host:5432/database.") + return + if backend == "sqlite" and runtime.production_like: + collector.add("error", "DATABASE_URL", "SQLite is only supported for disposable local development.", "Use PostgreSQL for production-like and self-hosted installs.") + elif backend != "postgresql" and runtime.production: + collector.add("warning", "DATABASE_URL", f"Database backend {backend!r} is not the preferred production target.", "Use PostgreSQL unless this deployment has an explicit support decision.") + if backend == "postgresql" and not _clean(env.get("GOVOPLAN_DATABASE_URL_PGTOOLS")): + collector.add("warning", "GOVOPLAN_DATABASE_URL_PGTOOLS", "PostgreSQL backup/restore tools URL is missing.", "Set GOVOPLAN_DATABASE_URL_PGTOOLS to the same database without the SQLAlchemy driver marker, for example postgresql://user:password@host:5432/database.") + +def _validate_master_key(env: Mapping[str, str], runtime: _RuntimeProfile, collector: _ConfigIssueCollector) -> None: master_key = _clean(env.get("MASTER_KEY_B64")) - if not master_key and not local: - issue("error", "MASTER_KEY_B64", "MASTER_KEY_B64 is required outside local dev/test.", "Generate a Fernet key with `govoplan-config env-template --generate-secrets` or `python -c 'from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())'` and store it in deployment secrets.") - elif master_key: - error = _master_key_error(master_key) - if error: - issue("error", "MASTER_KEY_B64", error, "Replace MASTER_KEY_B64 with a Fernet key or base64-encoded 32-byte key.") - elif "change-me" in master_key.lower() or "generate" in master_key.lower(): - issue("error", "MASTER_KEY_B64", "MASTER_KEY_B64 still looks like a placeholder.", "Generate a real deployment key and store it outside git.") + if not master_key and not runtime.local: + collector.add("error", "MASTER_KEY_B64", "MASTER_KEY_B64 is required outside local dev/test.", "Generate a Fernet key with `govoplan-config env-template --generate-secrets` or `python -c 'from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())'` and store it in deployment secrets.") + return + if not master_key: + return + error = _master_key_error(master_key) + if error: + collector.add("error", "MASTER_KEY_B64", error, "Replace MASTER_KEY_B64 with a Fernet key or base64-encoded 32-byte key.") + elif "change-me" in master_key.lower() or "generate" in master_key.lower(): + collector.add("error", "MASTER_KEY_B64", "MASTER_KEY_B64 still looks like a placeholder.", "Generate a real deployment key and store it outside git.") + +def _validate_enabled_modules(env: Mapping[str, str], runtime: _RuntimeProfile, collector: _ConfigIssueCollector) -> None: enabled_modules = _csv(env.get("ENABLED_MODULES")) - if not enabled_modules and production_like: - issue("error", "ENABLED_MODULES", "ENABLED_MODULES is missing.", "Set ENABLED_MODULES explicitly so startup module composition is intentional.") - elif "access" not in enabled_modules and production_like: - issue("error", "ENABLED_MODULES", "The access module is not enabled.", "Include `access` unless this deployment has a replacement auth/principal provider.") - elif enabled_modules and "admin" not in enabled_modules and production_like: - issue("warning", "ENABLED_MODULES", "The admin module is not enabled.", "Keep `admin` enabled for operator UI unless this is a deliberately headless install.") + if not enabled_modules and runtime.production_like: + collector.add("error", "ENABLED_MODULES", "ENABLED_MODULES is missing.", "Set ENABLED_MODULES explicitly so startup module composition is intentional.") + elif "access" not in enabled_modules and runtime.production_like: + collector.add("error", "ENABLED_MODULES", "The access module is not enabled.", "Include `access` unless this deployment has a replacement auth/principal provider.") + elif enabled_modules and "admin" not in enabled_modules and runtime.production_like: + collector.add("warning", "ENABLED_MODULES", "The admin module is not enabled.", "Keep `admin` enabled for operator UI unless this is a deliberately headless install.") - celery_enabled = _truthy(env.get("CELERY_ENABLED")) - if celery_enabled and not _clean(env.get("REDIS_URL")): - issue("error", "REDIS_URL", "CELERY_ENABLED=true but REDIS_URL is missing.", "Set REDIS_URL to the Redis broker/result backend used by workers.") - if production and _truthy(env.get("DEV_BOOTSTRAP_ENABLED")): - issue("error", "DEV_BOOTSTRAP_ENABLED", "Development bootstrap is enabled in production.", "Set DEV_BOOTSTRAP_ENABLED=false and create first administrators through the controlled bootstrap path.") +def _validate_async_and_auth_settings(env: Mapping[str, str], runtime: _RuntimeProfile, collector: _ConfigIssueCollector) -> None: + if _truthy(env.get("CELERY_ENABLED")) and not _clean(env.get("REDIS_URL")): + collector.add("error", "REDIS_URL", "CELERY_ENABLED=true but REDIS_URL is missing.", "Set REDIS_URL to the Redis broker/result backend used by workers.") + if runtime.production and _truthy(env.get("DEV_BOOTSTRAP_ENABLED")): + collector.add("error", "DEV_BOOTSTRAP_ENABLED", "Development bootstrap is enabled in production.", "Set DEV_BOOTSTRAP_ENABLED=false and create first administrators through the controlled bootstrap path.") + if runtime.production and not _truthy(env.get("AUTH_COOKIE_SECURE")): + collector.add("error", "AUTH_COOKIE_SECURE", "Secure auth cookies are disabled for production.", "Set AUTH_COOKIE_SECURE=true behind HTTPS.") - if production and not _truthy(env.get("AUTH_COOKIE_SECURE")): - issue("error", "AUTH_COOKIE_SECURE", "Secure auth cookies are disabled for production.", "Set AUTH_COOKIE_SECURE=true behind HTTPS.") +def _validate_cors_settings(env: Mapping[str, str], runtime: _RuntimeProfile, collector: _ConfigIssueCollector) -> None: cors_origins = _csv(env.get("CORS_ORIGINS")) - if production_like and not cors_origins: - issue("error", "CORS_ORIGINS", "CORS_ORIGINS is missing.", "Set CORS_ORIGINS to the exact WebUI origin or origins.") - elif "*" in cors_origins and production_like: - issue("error", "CORS_ORIGINS", "Wildcard CORS is not allowed for production-like installs.", "Replace `*` with exact HTTPS/WebUI origins.") - elif production and set(cors_origins) <= _DEFAULT_LOCAL_CORS: - issue("warning", "CORS_ORIGINS", "CORS_ORIGINS still contains only local development origins.", "Set CORS_ORIGINS to the deployed WebUI origin.") + if runtime.production_like and not cors_origins: + collector.add("error", "CORS_ORIGINS", "CORS_ORIGINS is missing.", "Set CORS_ORIGINS to the exact WebUI origin or origins.") + elif "*" in cors_origins and runtime.production_like: + collector.add("error", "CORS_ORIGINS", "Wildcard CORS is not allowed for production-like installs.", "Replace `*` with exact HTTPS/WebUI origins.") + elif runtime.production and set(cors_origins) <= _DEFAULT_LOCAL_CORS: + collector.add("warning", "CORS_ORIGINS", "CORS_ORIGINS still contains only local development origins.", "Set CORS_ORIGINS to the deployed WebUI origin.") + +def _validate_file_storage_settings(env: Mapping[str, str], runtime: _RuntimeProfile, collector: _ConfigIssueCollector) -> None: storage_backend = _clean(env.get("FILE_STORAGE_BACKEND")) or "local" if storage_backend == "local": - if not _clean(env.get("FILE_STORAGE_LOCAL_ROOT")) and production_like: - issue("error", "FILE_STORAGE_LOCAL_ROOT", "Local file storage root is missing.", "Set FILE_STORAGE_LOCAL_ROOT to a durable, backed-up path.") - elif production: - issue("warning", "FILE_STORAGE_BACKEND", "Production is configured for local file storage.", "Confirm the path is durable and backed up, or use object storage once the deployment needs independent file scaling.") + _validate_local_file_storage(env, runtime, collector) elif storage_backend == "s3": - for key in ("FILE_STORAGE_S3_ENDPOINT_URL", "FILE_STORAGE_S3_REGION", "FILE_STORAGE_S3_ACCESS_KEY_ID", "FILE_STORAGE_S3_SECRET_ACCESS_KEY", "FILE_STORAGE_S3_BUCKET"): - if not _clean(env.get(key)): - issue("error", key, f"{key} is required when FILE_STORAGE_BACKEND=s3.", "Configure all FILE_STORAGE_S3_* settings through deployment secrets.") + _validate_s3_file_storage(env, collector) else: - issue("error", "FILE_STORAGE_BACKEND", f"Unsupported FILE_STORAGE_BACKEND={storage_backend!r}.", "Use `local` or `s3`.") + collector.add("error", "FILE_STORAGE_BACKEND", f"Unsupported FILE_STORAGE_BACKEND={storage_backend!r}.", "Use `local` or `s3`.") + +def _validate_local_file_storage(env: Mapping[str, str], runtime: _RuntimeProfile, collector: _ConfigIssueCollector) -> None: + if not _clean(env.get("FILE_STORAGE_LOCAL_ROOT")) and runtime.production_like: + collector.add("error", "FILE_STORAGE_LOCAL_ROOT", "Local file storage root is missing.", "Set FILE_STORAGE_LOCAL_ROOT to a durable, backed-up path.") + elif runtime.production: + collector.add("warning", "FILE_STORAGE_BACKEND", "Production is configured for local file storage.", "Confirm the path is durable and backed up, or use object storage once the deployment needs independent file scaling.") + + +def _validate_s3_file_storage(env: Mapping[str, str], collector: _ConfigIssueCollector) -> None: + for key in ("FILE_STORAGE_S3_ENDPOINT_URL", "FILE_STORAGE_S3_REGION", "FILE_STORAGE_S3_ACCESS_KEY_ID", "FILE_STORAGE_S3_SECRET_ACCESS_KEY", "FILE_STORAGE_S3_BUCKET"): + if not _clean(env.get(key)): + collector.add("error", key, f"{key} is required when FILE_STORAGE_BACKEND=s3.", "Configure all FILE_STORAGE_S3_* settings through deployment secrets.") + + +def _validate_module_catalog_trust(env: Mapping[str, str], runtime: _RuntimeProfile, collector: _ConfigIssueCollector) -> None: catalog_source = _clean(env.get("GOVOPLAN_MODULE_PACKAGE_CATALOG_URL")) or _clean(env.get("GOVOPLAN_MODULE_PACKAGE_CATALOG")) - if production and catalog_source: - if not _clean(env.get("GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE")): - issue("error", "GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE", "A module catalog source is configured without a trusted keyring file.", "Pin the published GovOPlaN catalog keyring locally and set GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE.") - if not _clean(env.get("GOVOPLAN_MODULE_PACKAGE_CATALOG_APPROVED_CHANNEL")): - issue("error", "GOVOPLAN_MODULE_PACKAGE_CATALOG_APPROVED_CHANNEL", "A module catalog source is configured without an approved release channel.", "Set GOVOPLAN_MODULE_PACKAGE_CATALOG_APPROVED_CHANNEL=stable or another approved deployment channel.") - - return ConfigValidationResult(profile=clean_profile, issues=tuple(issues)) + if not runtime.production or not catalog_source: + return + if not _clean(env.get("GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE")): + collector.add("error", "GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE", "A module catalog source is configured without a trusted keyring file.", "Pin the published GovOPlaN catalog keyring locally and set GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE.") + if not _clean(env.get("GOVOPLAN_MODULE_PACKAGE_CATALOG_APPROVED_CHANNEL")): + collector.add("error", "GOVOPLAN_MODULE_PACKAGE_CATALOG_APPROVED_CHANNEL", "A module catalog source is configured without an approved release channel.", "Set GOVOPLAN_MODULE_PACKAGE_CATALOG_APPROVED_CHANNEL=stable or another approved deployment channel.") def _self_hosted_env_template(master_key: str) -> str: @@ -299,8 +353,8 @@ def _master_key_error(value: str) -> str | None: try: Fernet(candidate) return None - except Exception: - pass + except (TypeError, ValueError): + raw = None try: raw = base64.b64decode(candidate) except Exception: @@ -309,6 +363,6 @@ def _master_key_error(value: str) -> str | None: return "MASTER_KEY_B64 must decode to exactly 32 bytes." try: Fernet(base64.urlsafe_b64encode(raw)) - except Exception: + except (TypeError, ValueError): return "MASTER_KEY_B64 is not usable as a Fernet key." return None diff --git a/src/govoplan_core/core/module_installer.py b/src/govoplan_core/core/module_installer.py index 33ae86e..590c661 100644 --- a/src/govoplan_core/core/module_installer.py +++ b/src/govoplan_core/core/module_installer.py @@ -8,6 +8,7 @@ from datetime import UTC, datetime from importlib import metadata import hashlib import json +import logging import os from pathlib import Path import re @@ -19,8 +20,6 @@ import sys import tomllib from typing import Any, Literal import time -import urllib.error -import urllib.request from sqlalchemy.engine import make_url from sqlalchemy.orm import Session @@ -39,6 +38,9 @@ from govoplan_core.core.module_management import ( ) from govoplan_core.core.modules import ModuleManifest, MigrationRetirementPlan from govoplan_core.core.versioning import compare_versions, format_version_range, version_satisfies_range +from govoplan_core.security.http_fetch import fetch_http, validate_http_url + +logger = logging.getLogger(__name__) IssueSeverity = Literal["blocker", "warning", "info"] ChecklistStatus = Literal["done", "pending", "blocked", "warning"] @@ -281,6 +283,17 @@ class ModuleInstallerRunResult: return payload +@dataclass(slots=True) +class _ModuleInstallRunState: + run_id: str + run_dir: Path + record_path: Path + commands: tuple[dict[str, Any], ...] + result_commands: tuple[str, ...] + record_redactions: tuple[str, ...] + record: dict[str, Any] + + def default_installer_runtime_dir(database_url: str | None = None, *, cwd: Path | None = None) -> Path: root = cwd or Path.cwd() if database_url and database_url.startswith("sqlite:///"): @@ -329,53 +342,16 @@ def module_install_preflight( frontend_rebuild_required = False for item in planned_items: - if item.action == "uninstall": - issues.extend(_uninstall_guard_issues( - item, - current=current, - desired=desired, - dependents=dependents, - available=available, - session=session, - )) - if item.action in PACKAGE_TARGET_ACTIONS: - if item.action == "update" and item.module_id not in available: - issues.append(ModuleInstallerIssue( - "blocker", - "update_module_missing", - "Package updates require the module to be installed; use install for new modules.", - item.module_id, - )) - if item.module_id not in available: - issues.append(ModuleInstallerIssue( - "info", - "compatibility_checked_after_install", - "Module manifest compatibility can be fully checked after the package is installed and discovered.", - item.module_id, - )) - if item.python_ref and not item.python_package: - issues.append(ModuleInstallerIssue( - "blocker", - "python_package_required", - "Python installs must include the distribution package name so rollback can uninstall newly added packages.", - item.module_id, - )) - if item.python_ref and not _looks_pinned_dependency_ref(item.python_ref): - issues.append(ModuleInstallerIssue( - "blocker", - "unpinned_python_ref", - "Python install refs must be pinned to an exact version or tagged git ref.", - item.module_id, - )) - if item.webui_ref and not _looks_pinned_dependency_ref(item.webui_ref): - issues.append(ModuleInstallerIssue( - "blocker", - "unpinned_webui_ref", - "WebUI install refs must be pinned to an exact version or tagged git ref.", - item.module_id, - )) - if item.webui_package or item.webui_ref: - frontend_rebuild_required = True + item_issues, item_frontend_rebuild_required = _planned_item_preflight_issues( + item, + current=current, + desired=desired, + dependents=dependents, + available=available, + session=session, + ) + issues.extend(item_issues) + frontend_rebuild_required = frontend_rebuild_required or item_frontend_rebuild_required artifact_integrity, artifact_integrity_issues = _verify_artifact_integrity( planned_items, @@ -383,11 +359,7 @@ def module_install_preflight( ) issues.extend(artifact_integrity_issues) - if frontend_rebuild_required: - if webui_root is None: - issues.append(ModuleInstallerIssue("blocker", "webui_root_missing", "WebUI package changes need a webui root for npm install/build.")) - elif not (webui_root / "package.json").exists(): - issues.append(ModuleInstallerIssue("blocker", "webui_package_missing", f"WebUI package.json does not exist at {webui_root}.")) + issues.extend(_frontend_rebuild_preflight_issues(frontend_rebuild_required=frontend_rebuild_required, webui_root=webui_root)) record_dir = runtime_dir or default_installer_runtime_dir() rollback_commands = ( @@ -416,6 +388,88 @@ def module_install_preflight( ) +def _planned_item_preflight_issues( + item: ModuleInstallPlanItem, + *, + current: set[str], + desired: set[str], + dependents: Mapping[str, tuple[str, ...]], + available: Mapping[str, ModuleManifest], + session: object | None, +) -> tuple[tuple[ModuleInstallerIssue, ...], bool]: + issues: list[ModuleInstallerIssue] = [] + if item.action == "uninstall": + issues.extend(_uninstall_guard_issues( + item, + current=current, + desired=desired, + dependents=dependents, + available=available, + session=session, + )) + if item.action in PACKAGE_TARGET_ACTIONS: + issues.extend(_package_target_action_preflight_issues(item, available=available)) + return tuple(issues), bool(item.webui_package or item.webui_ref) + + +def _package_target_action_preflight_issues( + item: ModuleInstallPlanItem, + *, + available: Mapping[str, ModuleManifest], +) -> tuple[ModuleInstallerIssue, ...]: + issues: list[ModuleInstallerIssue] = [] + if item.action == "update" and item.module_id not in available: + issues.append(ModuleInstallerIssue( + "blocker", + "update_module_missing", + "Package updates require the module to be installed; use install for new modules.", + item.module_id, + )) + if item.module_id not in available: + issues.append(ModuleInstallerIssue( + "info", + "compatibility_checked_after_install", + "Module manifest compatibility can be fully checked after the package is installed and discovered.", + item.module_id, + )) + if item.python_ref and not item.python_package: + issues.append(ModuleInstallerIssue( + "blocker", + "python_package_required", + "Python installs must include the distribution package name so rollback can uninstall newly added packages.", + item.module_id, + )) + if item.python_ref and not _looks_pinned_dependency_ref(item.python_ref): + issues.append(ModuleInstallerIssue( + "blocker", + "unpinned_python_ref", + "Python install refs must be pinned to an exact version or tagged git ref.", + item.module_id, + )) + if item.webui_ref and not _looks_pinned_dependency_ref(item.webui_ref): + issues.append(ModuleInstallerIssue( + "blocker", + "unpinned_webui_ref", + "WebUI install refs must be pinned to an exact version or tagged git ref.", + item.module_id, + )) + return tuple(issues) + + +def _frontend_rebuild_preflight_issues( + *, + frontend_rebuild_required: bool, + webui_root: Path | None, +) -> tuple[ModuleInstallerIssue, ...]: + if not frontend_rebuild_required: + return () + if webui_root is None: + return (ModuleInstallerIssue("blocker", "webui_root_missing", "WebUI package changes need a webui root for npm install/build."),) + if not (webui_root / "package.json").exists(): + return (ModuleInstallerIssue("blocker", "webui_package_missing", f"WebUI package.json does not exist at {webui_root}."),) + return () + + def run_module_install_plan( *, session: Session, @@ -452,10 +506,80 @@ def run_module_install_plan( if not preflight.allowed: raise ModuleInstallerError("Install preflight is blocked: " + "; ".join(issue.message for issue in preflight.issues if issue.severity == "blocker")) + state = _prepare_module_install_run( + plan=plan, + preflight=preflight, + database_url=database_url, + effective_runtime_dir=effective_runtime_dir, + webui_root=webui_root, + npm_bin=npm_bin, + build_webui=build_webui, + migrate_database=migrate_database, + database_backup_command=database_backup_command, + database_restore_command=database_restore_command, + database_restore_check_command=database_restore_check_command, + activate_installed_modules=activate_installed_modules, + remove_uninstalled_modules_from_desired=remove_uninstalled_modules_from_desired, + dry_run=dry_run, + request_context=request_context, + ) + + if dry_run: + return ModuleInstallerRunResult(run_id=state.run_id, status="dry-run", record_path=state.record_path, commands=state.result_commands) + + executed, failed_error = _execute_module_install_run( + session=session, + plan=plan, + available=available, + effective_runtime_dir=effective_runtime_dir, + state=state, + ) + + if failed_error is not None: + return _failed_module_install_run_result( + state=state, + plan=plan, + executed=executed, + failed_error=failed_error, + effective_runtime_dir=effective_runtime_dir, + npm_bin=npm_bin, + build_webui=build_webui, + database_restore_command=database_restore_command, + database_url=database_url, + ) + + return _applied_module_install_run_result( + session=session, + plan=plan, + desired_enabled=desired_enabled, + activate_installed_modules=activate_installed_modules, + remove_uninstalled_modules_from_desired=remove_uninstalled_modules_from_desired, + executed=executed, + state=state, + ) + + +def _prepare_module_install_run( + *, + plan: ModuleInstallPlan, + preflight: ModuleInstallerPreflight, + database_url: str, + effective_runtime_dir: Path, + webui_root: Path | None, + npm_bin: str, + build_webui: bool, + migrate_database: bool, + database_backup_command: str | None, + database_restore_command: str | None, + database_restore_check_command: str | None, + activate_installed_modules: bool, + remove_uninstalled_modules_from_desired: bool, + dry_run: bool, + request_context: Mapping[str, object] | None, +) -> _ModuleInstallRunState: run_id = _run_id() run_dir = effective_runtime_dir / "runs" / run_id run_dir.mkdir(parents=True, exist_ok=False) - record_path = run_dir / "record.json" commands = structured_install_commands( plan, webui_root=webui_root, @@ -467,18 +591,57 @@ def run_module_install_plan( migration_task_record_path=run_dir / "migration-tasks.json" if migrate_database and preflight.migration_plan.tasks else None, verify_modules=True, ) - snapshot = _snapshot_environment( - run_dir, - webui_root=webui_root, - database_url=database_url, - include_database=migrate_database or _destructive_retirement_requested(plan), - database_backup_command=database_backup_command, - database_restore_command=database_restore_command, - database_restore_check_command=database_restore_check_command, - ) record_redactions = _installer_secret_redactions(database_url) - result_commands = _command_displays(commands, redactions=record_redactions) + record = _initial_module_install_record( + run_id=run_id, + plan=plan, + preflight=preflight, + commands=commands, + record_redactions=record_redactions, + snapshot=_snapshot_environment( + run_dir, + webui_root=webui_root, + database_url=database_url, + include_database=migrate_database or _destructive_retirement_requested(plan), + database_backup_command=database_backup_command, + database_restore_command=database_restore_command, + database_restore_check_command=database_restore_check_command, + ), + build_webui=build_webui, + migrate_database=migrate_database, + activate_installed_modules=activate_installed_modules, + remove_uninstalled_modules_from_desired=remove_uninstalled_modules_from_desired, + dry_run=dry_run, + request_context=request_context, + ) + record_path = run_dir / "record.json" + _write_json(record_path, record) + return _ModuleInstallRunState( + run_id=run_id, + run_dir=run_dir, + record_path=record_path, + commands=commands, + result_commands=_command_displays(commands, redactions=record_redactions), + record_redactions=record_redactions, + record=record, + ) + +def _initial_module_install_record( + *, + run_id: str, + plan: ModuleInstallPlan, + preflight: ModuleInstallerPreflight, + commands: tuple[dict[str, Any], ...], + record_redactions: tuple[str, ...], + snapshot: dict[str, object], + build_webui: bool, + migrate_database: bool, + activate_installed_modules: bool, + remove_uninstalled_modules_from_desired: bool, + dry_run: bool, + request_context: Mapping[str, object] | None, +) -> dict[str, Any]: record: dict[str, Any] = { "run_id": run_id, "started_at": datetime.now(tz=UTC).isoformat(), @@ -494,76 +657,133 @@ def run_module_install_plan( } if request_context: record["request"] = dict(request_context) - _write_json(record_path, record) + return record - if dry_run: - return ModuleInstallerRunResult(run_id=run_id, status="dry-run", record_path=record_path, commands=result_commands) +def _execute_module_install_run( + *, + session: Session, + plan: ModuleInstallPlan, + available: Mapping[str, ModuleManifest], + effective_runtime_dir: Path, + state: _ModuleInstallRunState, +) -> tuple[list[dict[str, object]], str | None]: executed: list[dict[str, object]] = [] failed_error: str | None = None with _installer_lock(effective_runtime_dir): try: - retirements = _execute_destructive_retirements(session=session, plan=plan, available=available) - if retirements: - session.commit() - record["retirements"] = retirements - _write_json(record_path, record) - for command in commands: - result = subprocess.run(command["argv"], cwd=command.get("cwd"), text=True, capture_output=True, check=False) - executed.append({ - **_command_record(command, redactions=record_redactions), - "return_code": result.returncode, - "stdout": _redact_installer_text(result.stdout[-8000:], redactions=record_redactions), - "stderr": _redact_installer_text(result.stderr[-8000:], redactions=record_redactions), - }) - migration_task_record_path = command.get("migration_task_record_path") - if migration_task_record_path: - task_records = _read_json_file(Path(str(migration_task_record_path))) - if isinstance(task_records, list): - executed[-1]["migration_tasks"] = task_records - record["migration_tasks"] = task_records - record["commands"] = executed - _write_json(record_path, record) - if result.returncode != 0: - raise ModuleInstallerError(f"Command failed ({result.returncode}): {_redact_installer_text(str(command['display']), redactions=record_redactions)}") + _execute_module_install_retirements(session=session, plan=plan, available=available, state=state) + for command in state.commands: + executed.append(_run_module_install_command(command, state=state)) + state.record["commands"] = executed + _write_json(state.record_path, state.record) except Exception as exc: - failed_error = _redact_installer_text(str(exc), redactions=record_redactions) - try: - session.rollback() - except Exception: - pass + failed_error = _redact_installer_text(str(exc), redactions=state.record_redactions) + _rollback_session_after_module_install_error(session, exc) + return executed, failed_error - if failed_error is not None: - record.update({ - "status": "failed", - "finished_at": datetime.now(tz=UTC).isoformat(), - "error": failed_error, - "commands": executed, - }) - _write_json(record_path, record) - if _destructive_retirement_requested(plan): - rollback = rollback_module_install_run( - run_id=run_id, - runtime_dir=effective_runtime_dir, - npm_bin=npm_bin, - build_webui=build_webui, - database_restore_command=database_restore_command, - database_url=database_url, - ) - _update_run_record(record_path, {"destructive_retirement_rollback": rollback.as_dict()}) - return ModuleInstallerRunResult( - run_id=run_id, - status="rolled-back" if rollback.return_code == 0 else "failed", - record_path=record_path, - commands=result_commands, - return_code=1, - error=failed_error, - rollback=rollback.as_dict(), - ) - return ModuleInstallerRunResult(run_id=run_id, status="failed", record_path=record_path, commands=result_commands, return_code=1, error=failed_error) - applied_items = tuple(_mark_applied(item) for item in plan.items) - save_module_install_plan(session, applied_items) +def _execute_module_install_retirements( + *, + session: Session, + plan: ModuleInstallPlan, + available: Mapping[str, ModuleManifest], + state: _ModuleInstallRunState, +) -> None: + retirements = _execute_destructive_retirements(session=session, plan=plan, available=available) + if not retirements: + return + session.commit() + state.record["retirements"] = retirements + _write_json(state.record_path, state.record) + + +def _run_module_install_command(command: Mapping[str, Any], *, state: _ModuleInstallRunState) -> dict[str, object]: + result = _run_structured_installer_command(command) + command_record: dict[str, object] = { + **_command_record(command, redactions=state.record_redactions), + "return_code": result.returncode, + "stdout": _redact_installer_text(result.stdout[-8000:], redactions=state.record_redactions), + "stderr": _redact_installer_text(result.stderr[-8000:], redactions=state.record_redactions), + } + migration_task_record_path = command.get("migration_task_record_path") + if migration_task_record_path: + task_records = _read_json_file(Path(str(migration_task_record_path))) + if isinstance(task_records, list): + command_record["migration_tasks"] = task_records + state.record["migration_tasks"] = task_records + if result.returncode != 0: + display = _redact_installer_text(str(command["display"]), redactions=state.record_redactions) + raise ModuleInstallerError(f"Command failed ({result.returncode}): {display}") + return command_record + + +def _rollback_session_after_module_install_error(session: Session, exc: Exception) -> None: + try: + session.rollback() + except Exception as rollback_exc: + logger.debug("Module installer rollback failed after run error: %s", rollback_exc, exc_info=True) + + +def _failed_module_install_run_result( + *, + state: _ModuleInstallRunState, + plan: ModuleInstallPlan, + executed: list[dict[str, object]], + failed_error: str, + effective_runtime_dir: Path, + npm_bin: str, + build_webui: bool, + database_restore_command: str | None, + database_url: str, +) -> ModuleInstallerRunResult: + state.record.update({ + "status": "failed", + "finished_at": datetime.now(tz=UTC).isoformat(), + "error": failed_error, + "commands": executed, + }) + _write_json(state.record_path, state.record) + if not _destructive_retirement_requested(plan): + return ModuleInstallerRunResult( + run_id=state.run_id, + status="failed", + record_path=state.record_path, + commands=state.result_commands, + return_code=1, + error=failed_error, + ) + rollback = rollback_module_install_run( + run_id=state.run_id, + runtime_dir=effective_runtime_dir, + npm_bin=npm_bin, + build_webui=build_webui, + database_restore_command=database_restore_command, + database_url=database_url, + ) + _update_run_record(state.record_path, {"destructive_retirement_rollback": rollback.as_dict()}) + return ModuleInstallerRunResult( + run_id=state.run_id, + status="rolled-back" if rollback.return_code == 0 else "failed", + record_path=state.record_path, + commands=state.result_commands, + return_code=1, + error=failed_error, + rollback=rollback.as_dict(), + ) + + +def _applied_module_install_run_result( + *, + session: Session, + plan: ModuleInstallPlan, + desired_enabled: Iterable[str], + activate_installed_modules: bool, + remove_uninstalled_modules_from_desired: bool, + executed: list[dict[str, object]], + state: _ModuleInstallRunState, +) -> ModuleInstallerRunResult: + save_module_install_plan(session, tuple(_mark_applied(item) for item in plan.items)) if activate_installed_modules or remove_uninstalled_modules_from_desired: next_desired = desired_modules_after_package_plan( desired_enabled, @@ -572,15 +792,15 @@ def run_module_install_plan( remove_uninstalled_modules_from_desired=remove_uninstalled_modules_from_desired, ) save_desired_enabled_modules(session, next_desired) - record["desired_enabled_after"] = list(next_desired) + state.record["desired_enabled_after"] = list(next_desired) session.commit() - record.update({ + state.record.update({ "status": "applied", "finished_at": datetime.now(tz=UTC).isoformat(), "commands": executed, }) - _write_json(record_path, record) - return ModuleInstallerRunResult(run_id=run_id, status="applied", record_path=record_path, commands=result_commands) + _write_json(state.record_path, state.record) + return ModuleInstallerRunResult(run_id=state.run_id, status="applied", record_path=state.record_path, commands=state.result_commands) def supervise_module_install_plan( @@ -726,36 +946,14 @@ def rollback_module_install_run( snapshot = record.get("snapshot") if isinstance(record.get("snapshot"), dict) else {} raw_database_backup = snapshot.get("database_backup") if isinstance(snapshot.get("database_backup"), Mapping) else {} rollback_redactions = _installer_secret_redactions(database_url or _database_url_from_external_snapshot(run_dir, raw_database_backup)) - commands: list[dict[str, Any]] = [] - for package in _planned_python_install_packages(record): - commands.append(_structured_command([sys.executable, "-m", "pip", "uninstall", "-y", package])) - pip_freeze = snapshot.get("pip_freeze") - if isinstance(pip_freeze, str) and (run_dir / pip_freeze).exists(): - commands.append(_structured_command([sys.executable, "-m", "pip", "install", "-r", str(run_dir / pip_freeze)])) - webui_root = snapshot.get("webui_root") - if isinstance(webui_root, str): - root = Path(webui_root) - _restore_snapshot_file(run_dir, snapshot, "package_json", root / "package.json") - _restore_snapshot_file(run_dir, snapshot, "package_lock", root / "package-lock.json") - if (root / "package.json").exists(): - commands.append(_structured_command([npm_bin, "install"], cwd=root)) - if build_webui or bool(record.get("build_webui")): - commands.append(_structured_command([npm_bin, "run", "build"], cwd=root)) + commands = _rollback_commands_from_snapshot(run_dir=run_dir, record=record, snapshot=snapshot, npm_bin=npm_bin, build_webui=build_webui) with _installer_lock(runtime_dir): - executed: list[dict[str, object]] = [] - for command in commands: - result = subprocess.run(command["argv"], cwd=command.get("cwd"), text=True, capture_output=True, check=False) - executed.append({ - **_command_record(command, redactions=rollback_redactions), - "return_code": result.returncode, - "stdout": _redact_installer_text(result.stdout[-8000:], redactions=rollback_redactions), - "stderr": _redact_installer_text(result.stderr[-8000:], redactions=rollback_redactions), - }) - if result.returncode != 0: - record.update({"rollback_status": "failed", "rollback_commands": executed}) - _write_json(record_path, record) - return ModuleInstallerRunResult(run_id=run_id, status="failed", record_path=record_path, commands=_command_displays(commands, redactions=rollback_redactions), return_code=1) + executed, command_failed = _execute_rollback_commands(commands=commands, redactions=rollback_redactions) + if command_failed: + record.update({"rollback_status": "failed", "rollback_commands": executed}) + _write_json(record_path, record) + return ModuleInstallerRunResult(run_id=run_id, status="failed", record_path=record_path, commands=_command_displays(commands, redactions=rollback_redactions), return_code=1) database_restore = _restore_database_snapshot( run_dir, snapshot, @@ -782,6 +980,76 @@ def rollback_module_install_run( return ModuleInstallerRunResult(run_id=run_id, status="rolled-back", record_path=record_path, commands=_command_displays(commands, redactions=rollback_redactions)) +def _rollback_commands_from_snapshot( + *, + run_dir: Path, + record: Mapping[str, object], + snapshot: Mapping[str, object], + npm_bin: str, + build_webui: bool, +) -> list[dict[str, Any]]: + commands = _rollback_python_commands(run_dir=run_dir, record=record, snapshot=snapshot) + commands.extend(_rollback_webui_commands(run_dir=run_dir, record=record, snapshot=snapshot, npm_bin=npm_bin, build_webui=build_webui)) + return commands + + +def _rollback_python_commands( + *, + run_dir: Path, + record: Mapping[str, object], + snapshot: Mapping[str, object], +) -> list[dict[str, Any]]: + commands = [ + _structured_command([sys.executable, "-m", "pip", "uninstall", "-y", package], source="rollback.python") + for package in _planned_python_install_packages(record) + ] + pip_freeze = snapshot.get("pip_freeze") + if isinstance(pip_freeze, str) and (run_dir / pip_freeze).exists(): + commands.append(_structured_command([sys.executable, "-m", "pip", "install", "-r", str(run_dir / pip_freeze)], source="rollback.python")) + return commands + + +def _rollback_webui_commands( + *, + run_dir: Path, + record: Mapping[str, object], + snapshot: Mapping[str, object], + npm_bin: str, + build_webui: bool, +) -> list[dict[str, Any]]: + webui_root = snapshot.get("webui_root") + if not isinstance(webui_root, str): + return [] + root = Path(webui_root) + _restore_snapshot_file(run_dir, snapshot, "package_json", root / "package.json") + _restore_snapshot_file(run_dir, snapshot, "package_lock", root / "package-lock.json") + if not (root / "package.json").exists(): + return [] + commands = [_structured_command([npm_bin, "install"], cwd=root, source="rollback.webui")] + if build_webui or bool(record.get("build_webui")): + commands.append(_structured_command([npm_bin, "run", "build"], cwd=root, source="rollback.webui")) + return commands + + +def _execute_rollback_commands( + *, + commands: list[dict[str, Any]], + redactions: tuple[str, ...], +) -> tuple[list[dict[str, object]], bool]: + executed: list[dict[str, object]] = [] + for command in commands: + result = _run_structured_installer_command(command) + executed.append({ + **_command_record(command, redactions=redactions), + "return_code": result.returncode, + "stdout": _redact_installer_text(result.stdout[-8000:], redactions=redactions), + "stderr": _redact_installer_text(result.stderr[-8000:], redactions=redactions), + }) + if result.returncode != 0: + return executed, True + return executed, False + + def list_module_installer_runs(*, runtime_dir: Path, limit: int = 25) -> tuple[dict[str, object], ...]: runs_dir = runtime_dir / "runs" if not runs_dir.exists(): @@ -848,6 +1116,7 @@ def queue_module_installer_request( runtime_dir: Path, options: Mapping[str, object] | None = None, requested_by: str | None = None, + tenant_id: str | None = None, retry_of: str | None = None, trace: Mapping[str, object] | None = None, ) -> dict[str, object]: @@ -858,6 +1127,7 @@ def queue_module_installer_request( "status": "queued", "created_at": datetime.now(tz=UTC).isoformat(), "requested_by": requested_by, + "tenant_id": tenant_id, "options": dict(options or {}), } if effective_trace: @@ -932,6 +1202,7 @@ def retry_module_installer_request( return queue_module_installer_request( runtime_dir=runtime_dir, requested_by=requested_by, + tenant_id=str(record.get("tenant_id")) if record.get("tenant_id") else None, options=options, retry_of=request_id, ) @@ -975,45 +1246,95 @@ def structured_install_commands( migration_task_record_path: Path | None = None, verify_modules: bool = False, ) -> tuple[dict[str, Any], ...]: + """Build the only structured commands the installer may execute directly. + + Command provenance is intentionally narrow: Python package refs and WebUI refs + come from the reviewed module install plan, WebUI install/build commands are + fixed internal templates, and migration/verification commands call core-owned + modules with argv lists. Operator-provided database backup/restore hooks use + `_operator_command_argv` instead and are rejected when they contain shell + operators, substitutions, redirects, or inline environment assignments. + """ commands: list[dict[str, Any]] = [] webui_changed = False for item in plan.items: if item.status != "planned": continue - if item.action in PACKAGE_TARGET_ACTIONS: - if item.python_ref: - commands.append(_structured_command([sys.executable, "-m", "pip", "install", item.python_ref])) - if item.webui_package and item.webui_ref and webui_root is not None: - commands.append(_structured_command([npm_bin, "pkg", "set", f"dependencies.{item.webui_package}={item.webui_ref}"], cwd=webui_root)) - webui_changed = True - elif item.action == "uninstall": - if item.python_package: - commands.append(_structured_command([sys.executable, "-m", "pip", "uninstall", "-y", item.python_package])) - if item.webui_package and webui_root is not None: - commands.append(_structured_command([npm_bin, "pkg", "delete", f"dependencies.{item.webui_package}"], cwd=webui_root)) - webui_changed = True - if webui_changed and webui_root is not None: - commands.append(_structured_command([npm_bin, "install"], cwd=webui_root)) - if build_webui: - commands.append(_structured_command([npm_bin, "run", "build"], cwd=webui_root)) + item_commands, item_webui_changed = _structured_item_commands(item, webui_root=webui_root, npm_bin=npm_bin) + commands.extend(item_commands) + webui_changed = webui_changed or item_webui_changed + commands.extend(_structured_webui_followup_commands(webui_changed=webui_changed, webui_root=webui_root, npm_bin=npm_bin, build_webui=build_webui)) verify_command = _module_verify_command(plan) if verify_modules else None if verify_command is not None: commands.append(verify_command) if migrate_database_url is not None: - migrate_command = [sys.executable, "-m", "govoplan_core.commands.init_db", "--database-url", migrate_database_url] - for module_id in dict.fromkeys(str(item).strip() for item in migration_enabled_modules if str(item).strip()): - migrate_command.extend(["--enabled-module", module_id]) - for module_id in dict.fromkeys(str(item).strip() for item in migration_module_order if str(item).strip()): - migrate_command.extend(["--migration-module", module_id]) - command = _structured_command(migrate_command) - if migration_task_record_path is not None: - migrate_command.extend(["--migration-task-record-output", str(migration_task_record_path)]) - command = _structured_command(migrate_command) - command["migration_task_record_path"] = str(migration_task_record_path) - commands.append(command) + commands.append(_structured_migration_command( + migrate_database_url=migrate_database_url, + migration_enabled_modules=migration_enabled_modules, + migration_module_order=migration_module_order, + migration_task_record_path=migration_task_record_path, + )) return tuple(commands) +def _structured_item_commands( + item: ModuleInstallPlanItem, + *, + webui_root: Path | None, + npm_bin: str, +) -> tuple[tuple[dict[str, Any], ...], bool]: + commands: list[dict[str, Any]] = [] + webui_changed = False + if item.action in PACKAGE_TARGET_ACTIONS: + if item.python_ref: + commands.append(_structured_command([sys.executable, "-m", "pip", "install", item.python_ref], source="module-plan.python")) + if item.webui_package and item.webui_ref and webui_root is not None: + commands.append(_structured_command([npm_bin, "pkg", "set", f"dependencies.{item.webui_package}={item.webui_ref}"], cwd=webui_root, source="module-plan.webui")) + webui_changed = True + elif item.action == "uninstall": + if item.python_package: + commands.append(_structured_command([sys.executable, "-m", "pip", "uninstall", "-y", item.python_package], source="module-plan.python")) + if item.webui_package and webui_root is not None: + commands.append(_structured_command([npm_bin, "pkg", "delete", f"dependencies.{item.webui_package}"], cwd=webui_root, source="module-plan.webui")) + webui_changed = True + return tuple(commands), webui_changed + + +def _structured_webui_followup_commands( + *, + webui_changed: bool, + webui_root: Path | None, + npm_bin: str, + build_webui: bool, +) -> tuple[dict[str, Any], ...]: + if not webui_changed or webui_root is None: + return () + commands = [_structured_command([npm_bin, "install"], cwd=webui_root, source="webui-dependencies")] + if build_webui: + commands.append(_structured_command([npm_bin, "run", "build"], cwd=webui_root, source="webui-build")) + return tuple(commands) + + +def _structured_migration_command( + *, + migrate_database_url: str, + migration_enabled_modules: Iterable[str], + migration_module_order: Iterable[str], + migration_task_record_path: Path | None, +) -> dict[str, Any]: + migrate_command = [sys.executable, "-m", "govoplan_core.commands.init_db", "--database-url", migrate_database_url] + for module_id in dict.fromkeys(str(item).strip() for item in migration_enabled_modules if str(item).strip()): + migrate_command.extend(["--enabled-module", module_id]) + for module_id in dict.fromkeys(str(item).strip() for item in migration_module_order if str(item).strip()): + migrate_command.extend(["--migration-module", module_id]) + if migration_task_record_path is None: + return _structured_command(migrate_command, source="database-migration") + migrate_command.extend(["--migration-task-record-output", str(migration_task_record_path)]) + command = _structured_command(migrate_command, source="database-migration") + command["migration_task_record_path"] = str(migration_task_record_path) + return command + + def _destructive_retirement_requested(plan: ModuleInstallPlan) -> bool: return any(item.status == "planned" and item.action == "uninstall" and item.destroy_data for item in plan.items) @@ -1024,27 +1345,36 @@ def _execute_destructive_retirements( plan: ModuleInstallPlan, available: Mapping[str, ModuleManifest], ) -> list[dict[str, object]]: - retirements: list[dict[str, object]] = [] - for item in plan.items: - if item.status != "planned" or item.action != "uninstall" or not item.destroy_data: - continue - manifest = available.get(item.module_id) - migration = manifest.migration_spec if manifest is not None else None - provider = migration.retirement_provider if migration is not None else None - if provider is None: - raise ModuleInstallerError(f"Module {item.module_id!r} does not provide a destructive retirement provider.") - retirement_plan = provider(session, item.module_id) - if not retirement_plan.supported: - raise ModuleInstallerError(retirement_plan.blocking_reason or retirement_plan.summary or f"Module {item.module_id!r} cannot be retired.") - if not retirement_plan.destroy_data_supported or retirement_plan.destroy_data_executor is None: - raise ModuleInstallerError(f"Module {item.module_id!r} does not support destructive data retirement.") - retirement_plan.destroy_data_executor(session, item.module_id) - retirements.append({ - "module_id": item.module_id, - "summary": retirement_plan.destroy_data_summary or retirement_plan.summary, - "warnings": list(retirement_plan.destroy_data_warnings), - }) - return retirements + return [ + _execute_destructive_retirement_item(session=session, item=item, available=available) + for item in plan.items + if item.status == "planned" and item.action == "uninstall" and item.destroy_data + ] + + +def _execute_destructive_retirement_item( + *, + session: Session, + item: ModuleInstallPlanItem, + available: Mapping[str, ModuleManifest], +) -> dict[str, object]: + manifest = available.get(item.module_id) + migration = manifest.migration_spec if manifest is not None else None + provider = migration.retirement_provider if migration is not None else None + if provider is None: + raise ModuleInstallerError(f"Module {item.module_id!r} does not provide a destructive retirement provider.") + retirement_plan = provider(session, item.module_id) + if not retirement_plan.supported: + raise ModuleInstallerError(retirement_plan.blocking_reason or retirement_plan.summary or f"Module {item.module_id!r} cannot be retired.") + if not retirement_plan.destroy_data_supported or retirement_plan.destroy_data_executor is None: + raise ModuleInstallerError(f"Module {item.module_id!r} does not support destructive data retirement.") + retirement_plan.destroy_data_executor(session, item.module_id) + return { + "module_id": item.module_id, + "summary": retirement_plan.destroy_data_summary or retirement_plan.summary, + "warnings": list(retirement_plan.destroy_data_warnings), + } + class ModuleInstallerError(RuntimeError): @@ -1071,43 +1401,75 @@ def _uninstall_guard_issues( if active_dependents: issues.append(ModuleInstallerIssue("blocker", "active_dependents", "Disable dependent modules first: " + ", ".join(active_dependents), item.module_id)) manifest = available.get(item.module_id) - destroy_supported = False if manifest is not None: - migration = manifest.migration_spec - if migration is not None: - if not migration.retirement_supported: - issues.append(ModuleInstallerIssue( - "blocker" if item.destroy_data else "warning", - "migration_retirement_not_requested", - "This module owns migrations. Default package uninstall is non-destructive: schema/data remain dormant until reinstall or explicit retirement support is added.", - item.module_id, - )) - elif migration.retirement_provider is None: - detail = f" {migration.retirement_notes}" if migration.retirement_notes else "" - issues.append(ModuleInstallerIssue( - "blocker" if item.destroy_data else "warning", - "migration_retirement_manual", - "This module declares migration retirement support but no automated retirement provider is registered." + detail, - item.module_id, - )) - else: - try: - retirement_plan = migration.retirement_provider(session, item.module_id) - except Exception as exc: - issues.append(ModuleInstallerIssue("blocker", "migration_retirement_check_failed", f"Migration retirement check failed: {exc}", item.module_id)) - else: - destroy_supported = retirement_plan.destroy_data_supported and retirement_plan.destroy_data_executor is not None - issues.extend(_migration_retirement_issues(retirement_plan, item.module_id, destroy_data_requested=item.destroy_data)) - elif item.destroy_data: - issues.append(ModuleInstallerIssue("blocker", "migration_retirement_missing", "This module does not declare a data retirement provider.", item.module_id)) - for provider in manifest.uninstall_guard_providers: - try: - for result in provider(session, item.module_id): - if item.destroy_data and destroy_supported and result.code == "persistent_data_present": - continue - issues.append(ModuleInstallerIssue(result.severity, result.code, result.message, item.module_id)) - except Exception as exc: - issues.append(ModuleInstallerIssue("blocker", "uninstall_guard_failed", f"Uninstall guard failed: {exc}", item.module_id)) + migration_issues, destroy_supported = _migration_uninstall_guard_issues(item, manifest=manifest, session=session) + issues.extend(migration_issues) + issues.extend(_provider_uninstall_guard_issues(item, manifest=manifest, session=session, destroy_supported=destroy_supported)) + return tuple(issues) + + +def _migration_uninstall_guard_issues( + item: ModuleInstallPlanItem, + *, + manifest: ModuleManifest, + session: object | None, +) -> tuple[tuple[ModuleInstallerIssue, ...], bool]: + migration = manifest.migration_spec + if migration is None: + if item.destroy_data: + return ( + (ModuleInstallerIssue("blocker", "migration_retirement_missing", "This module does not declare a data retirement provider.", item.module_id),), + False, + ) + return (), False + if not migration.retirement_supported: + return ( + (ModuleInstallerIssue( + "blocker" if item.destroy_data else "warning", + "migration_retirement_not_requested", + "This module owns migrations. Default package uninstall is non-destructive: schema/data remain dormant until reinstall or explicit retirement support is added.", + item.module_id, + ),), + False, + ) + if migration.retirement_provider is None: + detail = f" {migration.retirement_notes}" if migration.retirement_notes else "" + return ( + (ModuleInstallerIssue( + "blocker" if item.destroy_data else "warning", + "migration_retirement_manual", + "This module declares migration retirement support but no automated retirement provider is registered." + detail, + item.module_id, + ),), + False, + ) + try: + retirement_plan = migration.retirement_provider(session, item.module_id) + except Exception as exc: + return ( + (ModuleInstallerIssue("blocker", "migration_retirement_check_failed", f"Migration retirement check failed: {exc}", item.module_id),), + False, + ) + destroy_supported = retirement_plan.destroy_data_supported and retirement_plan.destroy_data_executor is not None + return _migration_retirement_issues(retirement_plan, item.module_id, destroy_data_requested=item.destroy_data), destroy_supported + + +def _provider_uninstall_guard_issues( + item: ModuleInstallPlanItem, + *, + manifest: ModuleManifest, + session: object | None, + destroy_supported: bool, +) -> tuple[ModuleInstallerIssue, ...]: + issues: list[ModuleInstallerIssue] = [] + for provider in manifest.uninstall_guard_providers: + try: + for result in provider(session, item.module_id): + if item.destroy_data and destroy_supported and result.code == "persistent_data_present": + continue + issues.append(ModuleInstallerIssue(result.severity, result.code, result.message, item.module_id)) + except Exception as exc: + issues.append(ModuleInstallerIssue("blocker", "uninstall_guard_failed", f"Uninstall guard failed: {exc}", item.module_id)) return tuple(issues) @@ -1161,7 +1523,7 @@ def _module_verify_command(plan: ModuleInstallPlan) -> dict[str, Any] | None: argv.extend(["--installed", module_id]) for module_id in dict.fromkeys(absent): argv.extend(["--absent", module_id]) - return _structured_command(argv) + return _structured_command(argv, source="module-verification") def module_manifest_compatibility_issues( @@ -1214,7 +1576,7 @@ def _current_core_version() -> str: if isinstance(version, str) and version: return version except (OSError, tomllib.TOMLDecodeError): - pass + return "0.0.0" return "0.0.0" @@ -1293,12 +1655,30 @@ def _package_catalog_preflight_issues( result = validate_module_package_catalog() except Exception as exc: - severity: IssueSeverity = "blocker" if catalog_items else "warning" - return (ModuleInstallerIssue( - severity, - "catalog_validation_failed", - f"Package catalog could not be validated for this install plan: {exc}", - ),) + return _catalog_validation_exception_issues(exc, catalog_items=bool(catalog_items)) + issues = list(_catalog_validation_result_issues(result, catalog_items=bool(catalog_items))) + if issues or not result.get("configured"): + return tuple(issues) + issues.extend(_catalog_warning_issues(result)) + if catalog_items: + issues.extend(_selected_catalog_interface_issues(catalog_items, result, available)) + return tuple(issues) + + +def _catalog_validation_exception_issues(exc: Exception, *, catalog_items: bool) -> tuple[ModuleInstallerIssue, ...]: + severity: IssueSeverity = "blocker" if catalog_items else "warning" + return (ModuleInstallerIssue( + severity, + "catalog_validation_failed", + f"Package catalog could not be validated for this install plan: {exc}", + ),) + + +def _catalog_validation_result_issues( + result: Mapping[str, object], + *, + catalog_items: bool, +) -> tuple[ModuleInstallerIssue, ...]: if not result.get("configured"): if catalog_items: return (ModuleInstallerIssue( @@ -1307,24 +1687,21 @@ def _package_catalog_preflight_issues( "Catalog-sourced package installs require a configured package catalog before activation.", ),) return () - issues: list[ModuleInstallerIssue] = [] - if result.get("valid") is not True: - severity = "blocker" if catalog_items else "warning" - issues.append(ModuleInstallerIssue( - severity, - "catalog_validation_failed", - str(result.get("error") or "Module package catalog is invalid."), - )) - return tuple(issues) + if result.get("valid") is True: + return () + severity: IssueSeverity = "blocker" if catalog_items else "warning" + return (ModuleInstallerIssue( + severity, + "catalog_validation_failed", + str(result.get("error") or "Module package catalog is invalid."), + ),) + + +def _catalog_warning_issues(result: Mapping[str, object]) -> tuple[ModuleInstallerIssue, ...]: warnings = result.get("warnings") - if isinstance(warnings, list): - issues.extend( - ModuleInstallerIssue("warning", "catalog_warning", str(warning)) - for warning in warnings - ) - if catalog_items: - issues.extend(_selected_catalog_interface_issues(catalog_items, result, available)) - return tuple(issues) + if not isinstance(warnings, list): + return () + return tuple(ModuleInstallerIssue("warning", "catalog_warning", str(warning)) for warning in warnings) def _module_install_target_plan( @@ -1335,55 +1712,74 @@ def _module_install_target_plan( if not planned_items: return () catalog_modules = _catalog_modules_for_target_plan(planned_items) - targets: list[ModuleInstallTargetItem] = [] - for item in planned_items: - manifest = available.get(item.module_id) - catalog_module = catalog_modules.get(item.module_id) if item.source == "catalog" else None - target_version = None if item.action == "uninstall" else _catalog_optional_string(catalog_module, "version") - migration_safety = "automatic" - migration_notes = None - current_version_min = None - current_version_max_exclusive = None - bridge_release = False - bridge_notes = None - allow_downgrade = False - allow_same_version = False - recovery_tested = False - recovery_notes = None - if catalog_module is not None and item.action in PACKAGE_TARGET_ACTIONS: - migration_safety = _catalog_optional_string(catalog_module, "migration_safety") or "automatic" - migration_notes = _catalog_optional_string(catalog_module, "migration_notes") - current_version_min = _catalog_optional_string(catalog_module, "current_version_min") - current_version_max_exclusive = _catalog_optional_string(catalog_module, "current_version_max_exclusive") - bridge_release = _catalog_optional_bool(catalog_module, "bridge_release") - bridge_notes = _catalog_optional_string(catalog_module, "bridge_notes") - allow_downgrade = _catalog_optional_bool(catalog_module, "allow_downgrade") - allow_same_version = _catalog_optional_bool(catalog_module, "allow_same_version") - recovery_tested = _catalog_optional_bool(catalog_module, "recovery_tested") - recovery_notes = _catalog_optional_string(catalog_module, "recovery_notes") - targets.append(ModuleInstallTargetItem( - module_id=item.module_id, - action=item.action, - source=item.source, - current_version=manifest.version if manifest is not None else None, - target_version=target_version, - python_package=item.python_package or _catalog_optional_string(catalog_module, "python_package"), - python_ref=item.python_ref or _catalog_optional_string(catalog_module, "python_ref"), - webui_package=item.webui_package or _catalog_optional_string(catalog_module, "webui_package"), - webui_ref=item.webui_ref or _catalog_optional_string(catalog_module, "webui_ref"), - migration_safety=migration_safety, - migration_notes=migration_notes, - current_version_min=current_version_min, - current_version_max_exclusive=current_version_max_exclusive, - bridge_release=bridge_release, - bridge_notes=bridge_notes, - allow_downgrade=allow_downgrade, - allow_same_version=allow_same_version, - recovery_tested=recovery_tested, - recovery_notes=recovery_notes, - data_safety_acknowledged=item.data_safety_acknowledged, - )) - return tuple(targets) + return tuple( + _module_install_target_item(item, available=available, catalog_modules=catalog_modules) + for item in planned_items + ) + + +def _module_install_target_item( + item: ModuleInstallPlanItem, + *, + available: Mapping[str, ModuleManifest], + catalog_modules: Mapping[str, Mapping[str, object]], +) -> ModuleInstallTargetItem: + manifest = available.get(item.module_id) + catalog_module = catalog_modules.get(item.module_id) if item.source == "catalog" else None + policy = _catalog_target_policy(item, catalog_module) + return ModuleInstallTargetItem( + module_id=item.module_id, + action=item.action, + source=item.source, + current_version=manifest.version if manifest is not None else None, + target_version=None if item.action == "uninstall" else _catalog_optional_string(catalog_module, "version"), + python_package=item.python_package or _catalog_optional_string(catalog_module, "python_package"), + python_ref=item.python_ref or _catalog_optional_string(catalog_module, "python_ref"), + webui_package=item.webui_package or _catalog_optional_string(catalog_module, "webui_package"), + webui_ref=item.webui_ref or _catalog_optional_string(catalog_module, "webui_ref"), + migration_safety=str(policy["migration_safety"]), + migration_notes=policy["migration_notes"] if isinstance(policy["migration_notes"], str) else None, + current_version_min=policy["current_version_min"] if isinstance(policy["current_version_min"], str) else None, + current_version_max_exclusive=policy["current_version_max_exclusive"] if isinstance(policy["current_version_max_exclusive"], str) else None, + bridge_release=policy["bridge_release"] is True, + bridge_notes=policy["bridge_notes"] if isinstance(policy["bridge_notes"], str) else None, + allow_downgrade=policy["allow_downgrade"] is True, + allow_same_version=policy["allow_same_version"] is True, + recovery_tested=policy["recovery_tested"] is True, + recovery_notes=policy["recovery_notes"] if isinstance(policy["recovery_notes"], str) else None, + data_safety_acknowledged=item.data_safety_acknowledged, + ) + + +def _catalog_target_policy( + item: ModuleInstallPlanItem, + catalog_module: Mapping[str, object] | None, +) -> dict[str, object]: + if catalog_module is None or item.action not in PACKAGE_TARGET_ACTIONS: + return { + "migration_safety": "automatic", + "migration_notes": None, + "current_version_min": None, + "current_version_max_exclusive": None, + "bridge_release": False, + "bridge_notes": None, + "allow_downgrade": False, + "allow_same_version": False, + "recovery_tested": False, + "recovery_notes": None, + } + return { + "migration_safety": _catalog_optional_string(catalog_module, "migration_safety") or "automatic", + "migration_notes": _catalog_optional_string(catalog_module, "migration_notes"), + "current_version_min": _catalog_optional_string(catalog_module, "current_version_min"), + "current_version_max_exclusive": _catalog_optional_string(catalog_module, "current_version_max_exclusive"), + "bridge_release": _catalog_optional_bool(catalog_module, "bridge_release"), + "bridge_notes": _catalog_optional_string(catalog_module, "bridge_notes"), + "allow_downgrade": _catalog_optional_bool(catalog_module, "allow_downgrade"), + "allow_same_version": _catalog_optional_bool(catalog_module, "allow_same_version"), + "recovery_tested": _catalog_optional_bool(catalog_module, "recovery_tested"), + "recovery_notes": _catalog_optional_string(catalog_module, "recovery_notes"), + } def module_install_catalog_companion_module_ids( @@ -1392,80 +1788,18 @@ def module_install_catalog_companion_module_ids( *, validation: Mapping[str, object] | None = None, ) -> tuple[str, ...]: - planned_items = tuple( - item - for item in plan.items - if item.status == "planned" and item.source == "catalog" and item.action in PACKAGE_TARGET_ACTIONS - ) + planned_items = _catalog_companion_planned_items(plan) if not planned_items: return () + validation = _catalog_companion_validation(validation) if validation is None: - try: - from govoplan_core.core.module_package_catalog import validate_module_package_catalog - - validation = validate_module_package_catalog() - except Exception: - return () - if validation.get("valid") is not True: return () modules = _catalog_modules_by_id(validation) original_ids = {item.module_id for item in planned_items} planned_ids = set(original_ids) companions: list[str] = [] for _round in range(20): - target_provider_versions = _catalog_target_provider_versions_for_ids( - planned_ids=planned_ids, - modules=modules, - available=available, - ) - missing: set[str] = set() - for module_id in sorted(planned_ids): - module = modules.get(module_id) - if module is None: - continue - for dependency_id in _catalog_string_list(module.get("dependencies")): - if dependency_id in planned_ids or dependency_id in available: - continue - if dependency_id in modules: - missing.add(dependency_id) - for requirement in _catalog_interface_items(module.get("requires_interfaces")): - if _catalog_requirement_satisfied(requirement, target_provider_versions): - continue - name = requirement.get("name") if isinstance(requirement.get("name"), str) else None - if not name: - continue - missing.update(_catalog_provider_suggestions( - name=name, - version_min=requirement.get("version_min") if isinstance(requirement.get("version_min"), str) else None, - version_max_exclusive=( - requirement.get("version_max_exclusive") - if isinstance(requirement.get("version_max_exclusive"), str) - else None - ), - catalog_modules=modules, - planned_ids=planned_ids, - )) - for manifest in available.values(): - if manifest.id in planned_ids: - continue - requirements = tuple( - { - "name": requirement.name, - "version_min": requirement.version_min, - "version_max_exclusive": requirement.version_max_exclusive, - "optional": requirement.optional, - } - for requirement in manifest.requires_interfaces - ) - if all(_catalog_requirement_satisfied(requirement, target_provider_versions) for requirement in requirements): - continue - candidate_update = modules.get(manifest.id) - if candidate_update is None: - continue - candidate_requirements = _catalog_interface_items(candidate_update.get("requires_interfaces")) - if _catalog_requirements_satisfied(candidate_requirements, target_provider_versions): - missing.add(manifest.id) - missing = {module_id for module_id in missing if module_id not in planned_ids and module_id in modules} + missing = _catalog_companion_missing_ids(planned_ids=planned_ids, modules=modules, available=available) if not missing: break for module_id in sorted(missing): @@ -1474,6 +1808,149 @@ def module_install_catalog_companion_module_ids( return tuple(module_id for module_id in companions if module_id not in original_ids) +def _catalog_companion_planned_items(plan: ModuleInstallPlan) -> tuple[ModuleInstallPlanItem, ...]: + return tuple( + item + for item in plan.items + if item.status == "planned" and item.source == "catalog" and item.action in PACKAGE_TARGET_ACTIONS + ) + + +def _catalog_companion_validation(validation: Mapping[str, object] | None) -> Mapping[str, object] | None: + if validation is None: + try: + from govoplan_core.core.module_package_catalog import validate_module_package_catalog + + validation = validate_module_package_catalog() + except Exception: + return None + return validation if validation.get("valid") is True else None + + +def _catalog_companion_missing_ids( + *, + planned_ids: set[str], + modules: Mapping[str, Mapping[str, object]], + available: Mapping[str, ModuleManifest], +) -> set[str]: + target_provider_versions = _catalog_target_provider_versions_for_ids( + planned_ids=planned_ids, + modules=modules, + available=available, + ) + missing = set() + missing.update(_missing_catalog_companions_for_planned_ids( + planned_ids=planned_ids, + modules=modules, + available=available, + target_provider_versions=target_provider_versions, + )) + missing.update(_installed_modules_needing_catalog_companion_updates( + planned_ids=planned_ids, + modules=modules, + available=available, + target_provider_versions=target_provider_versions, + )) + return {module_id for module_id in missing if module_id not in planned_ids and module_id in modules} + + +def _missing_catalog_companions_for_planned_ids( + *, + planned_ids: set[str], + modules: Mapping[str, Mapping[str, object]], + available: Mapping[str, ModuleManifest], + target_provider_versions: Mapping[str, tuple[tuple[str, str], ...]], +) -> set[str]: + missing: set[str] = set() + for module_id in sorted(planned_ids): + module = modules.get(module_id) + if module is None: + continue + missing.update(_missing_catalog_dependency_ids(module, planned_ids=planned_ids, modules=modules, available=available)) + missing.update(_missing_catalog_requirement_provider_ids(module, planned_ids=planned_ids, modules=modules, target_provider_versions=target_provider_versions)) + return missing + + +def _missing_catalog_dependency_ids( + module: Mapping[str, object], + *, + planned_ids: set[str], + modules: Mapping[str, Mapping[str, object]], + available: Mapping[str, ModuleManifest], +) -> set[str]: + return { + dependency_id + for dependency_id in _catalog_string_list(module.get("dependencies")) + if dependency_id not in planned_ids and dependency_id not in available and dependency_id in modules + } + + +def _missing_catalog_requirement_provider_ids( + module: Mapping[str, object], + *, + planned_ids: set[str], + modules: Mapping[str, Mapping[str, object]], + target_provider_versions: Mapping[str, tuple[tuple[str, str], ...]], +) -> set[str]: + missing: set[str] = set() + for requirement in _catalog_interface_items(module.get("requires_interfaces")): + if _catalog_requirement_satisfied(requirement, target_provider_versions): + continue + name = requirement.get("name") if isinstance(requirement.get("name"), str) else None + if not name: + continue + missing.update(_catalog_provider_suggestions( + name=name, + version_min=requirement.get("version_min") if isinstance(requirement.get("version_min"), str) else None, + version_max_exclusive=( + requirement.get("version_max_exclusive") + if isinstance(requirement.get("version_max_exclusive"), str) + else None + ), + catalog_modules=modules, + planned_ids=planned_ids, + )) + return missing + + +def _installed_modules_needing_catalog_companion_updates( + *, + planned_ids: set[str], + modules: Mapping[str, Mapping[str, object]], + available: Mapping[str, ModuleManifest], + target_provider_versions: Mapping[str, tuple[tuple[str, str], ...]], +) -> set[str]: + missing: set[str] = set() + for manifest in available.values(): + if manifest.id in planned_ids: + continue + if _manifest_requirements_satisfied(manifest, target_provider_versions): + continue + candidate_update = modules.get(manifest.id) + if candidate_update is None: + continue + candidate_requirements = _catalog_interface_items(candidate_update.get("requires_interfaces")) + if _catalog_requirements_satisfied(candidate_requirements, target_provider_versions): + missing.add(manifest.id) + return missing + + +def _manifest_requirements_satisfied( + manifest: ModuleManifest, + target_provider_versions: Mapping[str, tuple[tuple[str, str], ...]], +) -> bool: + requirements = tuple( + { + "name": requirement.name, + "version_min": requirement.version_min, + "version_max_exclusive": requirement.version_max_exclusive, + "optional": requirement.optional, + } + for requirement in manifest.requires_interfaces + ) + return all(_catalog_requirement_satisfied(requirement, target_provider_versions) for requirement in requirements) + + def _module_migration_execution_plan( *, plan: ModuleInstallPlan, @@ -1483,7 +1960,34 @@ def _module_migration_execution_plan( planned_items = tuple(item for item in plan.items if item.status == "planned") enabled_modules = desired_modules_after_package_plan(desired_enabled, plan) catalog_modules = _catalog_modules_for_target_plan(planned_items) - nodes = { + nodes = _migration_plan_nodes(planned_items=planned_items, available=available, catalog_modules=catalog_modules) + if not nodes: + return ModuleMigrationExecutionPlan(enabled_modules=enabled_modules), () + + issues: list[ModuleInstallerIssue] = [] + target_metadata = _migration_target_metadata_by_id(enabled_modules=enabled_modules, nodes=nodes, available=available, catalog_modules=catalog_modules) + edges = _migration_execution_edges( + nodes=nodes, + target_metadata=target_metadata, + enabled_modules=enabled_modules, + available=available, + issues=issues, + ) + ordered_ids = _ordered_migration_execution_ids(edges=edges, nodes=nodes, issues=issues) + + steps = tuple(_migration_step_from_node(nodes[module_id]) for module_id in ordered_ids if module_id in nodes) + tasks = _migration_task_plan_items(ordered_ids, target_metadata, nodes) + issues.extend(_migration_execution_plan_issues(planned_items=planned_items, steps=steps, tasks=tasks)) + return ModuleMigrationExecutionPlan(enabled_modules=enabled_modules, steps=steps, tasks=tasks), tuple(issues) + + +def _migration_plan_nodes( + *, + planned_items: tuple[ModuleInstallPlanItem, ...], + available: Mapping[str, ModuleManifest], + catalog_modules: Mapping[str, Mapping[str, object]], +) -> dict[str, Mapping[str, object]]: + return { item.module_id: _migration_node( item=item, manifest=available.get(item.module_id), @@ -1492,12 +1996,16 @@ def _module_migration_execution_plan( for item in planned_items if _migration_step_relevant(item, available.get(item.module_id), catalog_modules.get(item.module_id)) } - if not nodes: - return ModuleMigrationExecutionPlan(enabled_modules=enabled_modules), () - edges: dict[str, set[str]] = {module_id: set() for module_id in nodes} - issues: list[ModuleInstallerIssue] = [] - target_metadata = { + +def _migration_target_metadata_by_id( + *, + enabled_modules: tuple[str, ...], + nodes: Mapping[str, Mapping[str, object]], + available: Mapping[str, ModuleManifest], + catalog_modules: Mapping[str, Mapping[str, object]], +) -> dict[str, Mapping[str, object]]: + return { module_id: _migration_target_metadata( module_id=module_id, manifest=available.get(module_id), @@ -1505,74 +2013,132 @@ def _module_migration_execution_plan( ) for module_id in set(enabled_modules) | set(nodes) } - provider_modules = _migration_provider_modules(target_metadata) + +def _migration_execution_edges( + *, + nodes: Mapping[str, Mapping[str, object]], + target_metadata: Mapping[str, Mapping[str, object]], + enabled_modules: tuple[str, ...], + available: Mapping[str, ModuleManifest], + issues: list[ModuleInstallerIssue], +) -> dict[str, set[str]]: + edges: dict[str, set[str]] = {module_id: set() for module_id in nodes} + provider_modules = _migration_provider_modules(target_metadata) + target_enabled = set(enabled_modules) for module_id, node in nodes.items(): metadata = target_metadata.get(module_id) if metadata is None: continue - for before_id in metadata["migration_before"]: - _add_migration_edge( - edges, - issues, - before=module_id, - after=before_id, - source=module_id, - target_enabled=set(enabled_modules), - available=available, - ) - for after_id in metadata["migration_after"]: - _add_migration_edge( - edges, - issues, - before=after_id, - after=module_id, - source=module_id, - target_enabled=set(enabled_modules), - available=available, - ) - dependency_ids = [*metadata["dependencies"], *metadata["optional_dependencies"]] - for dependency_id in dependency_ids: - if dependency_id not in nodes: - continue - if node["phase"] == "retirement" and nodes[dependency_id]["phase"] == "retirement": - edges[module_id].add(dependency_id) - else: - edges[dependency_id].add(module_id) - for requirement in metadata["requires_interfaces"]: - name = requirement.get("name") if isinstance(requirement.get("name"), str) else None - if not name: - continue - version_min = requirement.get("version_min") if isinstance(requirement.get("version_min"), str) else None - version_max_exclusive = ( - requirement.get("version_max_exclusive") - if isinstance(requirement.get("version_max_exclusive"), str) - else None - ) - for provider_id, version in provider_modules.get(name, ()): - if provider_id not in nodes or provider_id == module_id: - continue - if not version_satisfies_range(version, version_min=version_min, version_max_exclusive=version_max_exclusive): - continue - if node["phase"] == "retirement" and nodes[provider_id]["phase"] == "retirement": - edges[module_id].add(provider_id) - else: - edges[provider_id].add(module_id) + _add_explicit_migration_edges(edges, issues, module_id=module_id, metadata=metadata, target_enabled=target_enabled, available=available) + _add_dependency_migration_edges(edges, module_id=module_id, node=node, metadata=metadata, nodes=nodes) + _add_interface_migration_edges(edges, module_id=module_id, node=node, metadata=metadata, nodes=nodes, provider_modules=provider_modules) + return edges + +def _add_explicit_migration_edges( + edges: dict[str, set[str]], + issues: list[ModuleInstallerIssue], + *, + module_id: str, + metadata: Mapping[str, object], + target_enabled: set[str], + available: Mapping[str, ModuleManifest], +) -> None: + for before_id in metadata["migration_before"]: + _add_migration_edge(edges, issues, before=module_id, after=before_id, source=module_id, target_enabled=target_enabled, available=available) + for after_id in metadata["migration_after"]: + _add_migration_edge(edges, issues, before=after_id, after=module_id, source=module_id, target_enabled=target_enabled, available=available) + + +def _add_dependency_migration_edges( + edges: dict[str, set[str]], + *, + module_id: str, + node: Mapping[str, object], + metadata: Mapping[str, object], + nodes: Mapping[str, Mapping[str, object]], +) -> None: + for dependency_id in [*metadata["dependencies"], *metadata["optional_dependencies"]]: + if dependency_id not in nodes: + continue + _add_related_migration_edge(edges, module_id=module_id, related_id=dependency_id, node=node, related_node=nodes[dependency_id]) + + +def _add_interface_migration_edges( + edges: dict[str, set[str]], + *, + module_id: str, + node: Mapping[str, object], + metadata: Mapping[str, object], + nodes: Mapping[str, Mapping[str, object]], + provider_modules: Mapping[str, tuple[tuple[str, str], ...]], +) -> None: + for requirement in metadata["requires_interfaces"]: + for provider_id in _matching_migration_provider_ids(module_id=module_id, requirement=requirement, nodes=nodes, provider_modules=provider_modules): + _add_related_migration_edge(edges, module_id=module_id, related_id=provider_id, node=node, related_node=nodes[provider_id]) + + +def _matching_migration_provider_ids( + *, + module_id: str, + requirement: Mapping[str, object], + nodes: Mapping[str, Mapping[str, object]], + provider_modules: Mapping[str, tuple[tuple[str, str], ...]], +) -> tuple[str, ...]: + name = requirement.get("name") if isinstance(requirement.get("name"), str) else None + if not name: + return () + version_min = requirement.get("version_min") if isinstance(requirement.get("version_min"), str) else None + version_max_exclusive = requirement.get("version_max_exclusive") if isinstance(requirement.get("version_max_exclusive"), str) else None + return tuple( + provider_id + for provider_id, version in provider_modules.get(name, ()) + if provider_id in nodes + and provider_id != module_id + and version_satisfies_range(version, version_min=version_min, version_max_exclusive=version_max_exclusive) + ) + + +def _add_related_migration_edge( + edges: dict[str, set[str]], + *, + module_id: str, + related_id: str, + node: Mapping[str, object], + related_node: Mapping[str, object], +) -> None: + if node["phase"] == "retirement" and related_node["phase"] == "retirement": + edges[module_id].add(related_id) + else: + edges[related_id].add(module_id) + + +def _ordered_migration_execution_ids( + *, + edges: dict[str, set[str]], + nodes: Mapping[str, Mapping[str, object]], + issues: list[ModuleInstallerIssue], +) -> tuple[str, ...]: ordered_ids, cycle_ids = _topological_module_order(edges) - if cycle_ids: - cycle_text = ", ".join(cycle_ids) - issues.append(ModuleInstallerIssue( - "blocker", - "migration_graph_cycle", - f"Module migration ordering has a cycle involving: {cycle_text}. Adjust migration_after/migration_before metadata or publish a bridge release.", - )) - ordered_ids = tuple(sorted(nodes)) + if not cycle_ids: + return ordered_ids + cycle_text = ", ".join(cycle_ids) + issues.append(ModuleInstallerIssue( + "blocker", + "migration_graph_cycle", + f"Module migration ordering has a cycle involving: {cycle_text}. Adjust migration_after/migration_before metadata or publish a bridge release.", + )) + return tuple(sorted(nodes)) - steps = tuple(_migration_step_from_node(nodes[module_id]) for module_id in ordered_ids if module_id in nodes) - tasks = _migration_task_plan_items(ordered_ids, target_metadata, nodes) - planned_by_id = {item.module_id: item for item in planned_items} - issues.extend(_migration_task_preflight_issues(tasks, planned_by_id)) + +def _migration_execution_plan_issues( + *, + planned_items: tuple[ModuleInstallPlanItem, ...], + steps: tuple[ModuleMigrationPlanStep, ...], + tasks: tuple[ModuleMigrationTaskPlanItem, ...], +) -> tuple[ModuleInstallerIssue, ...]: + issues = list(_migration_task_preflight_issues(tasks, {item.module_id: item for item in planned_items})) pending = [step.module_id for step in steps if step.metadata_pending] if pending: issues.append(ModuleInstallerIssue( @@ -1580,7 +2146,7 @@ def _module_migration_execution_plan( "migration_metadata_pending", "Migration metadata for newly installed manual package(s) can only be confirmed after package verification: " + ", ".join(pending) + ".", )) - return ModuleMigrationExecutionPlan(enabled_modules=enabled_modules, steps=steps, tasks=tasks), tuple(issues) + return tuple(issues) def _migration_step_relevant( @@ -1612,23 +2178,9 @@ def _migration_node( catalog_module: Mapping[str, object] | None, ) -> dict[str, object]: has_manifest_metadata = manifest is not None and manifest.migration_spec is not None - has_catalog_metadata = catalog_module is not None and ( - (_catalog_optional_string(catalog_module, "migration_safety") or "automatic") != "automatic" - or bool(_catalog_string_list(catalog_module.get("migration_after"))) - or bool(_catalog_string_list(catalog_module.get("migration_before"))) - or bool(_catalog_migration_task_items(catalog_module.get("migration_tasks"))) - ) + has_catalog_metadata = _catalog_module_has_migration_metadata(catalog_module) phase = "retirement" if item.action == "uninstall" else "upgrade" - source = "manifest" if has_manifest_metadata else "catalog" if has_catalog_metadata else "pending" - reason = None - if item.action == "uninstall": - reason = "Module owns migration metadata and may leave dormant schema behind." - elif has_manifest_metadata: - reason = "Installed module owns migration metadata." - elif has_catalog_metadata: - reason = "Catalog target declares migration metadata." - else: - reason = "Manual backend package may add migration metadata after installation." + source = _migration_node_source(has_manifest_metadata=has_manifest_metadata, has_catalog_metadata=has_catalog_metadata) return { "module_id": item.module_id, "action": item.action, @@ -1639,10 +2191,44 @@ def _migration_node( "migration_safety": _catalog_optional_string(catalog_module, "migration_safety") or "automatic", "current_version": manifest.version if manifest is not None else None, "target_version": _catalog_optional_string(catalog_module, "version"), - "reason": reason, + "reason": _migration_node_reason(item, has_manifest_metadata=has_manifest_metadata, has_catalog_metadata=has_catalog_metadata), } +def _catalog_module_has_migration_metadata(catalog_module: Mapping[str, object] | None) -> bool: + if catalog_module is None: + return False + return ( + (_catalog_optional_string(catalog_module, "migration_safety") or "automatic") != "automatic" + or bool(_catalog_string_list(catalog_module.get("migration_after"))) + or bool(_catalog_string_list(catalog_module.get("migration_before"))) + or bool(_catalog_migration_task_items(catalog_module.get("migration_tasks"))) + ) + + +def _migration_node_source(*, has_manifest_metadata: bool, has_catalog_metadata: bool) -> str: + if has_manifest_metadata: + return "manifest" + if has_catalog_metadata: + return "catalog" + return "pending" + + +def _migration_node_reason( + item: ModuleInstallPlanItem, + *, + has_manifest_metadata: bool, + has_catalog_metadata: bool, +) -> str: + if item.action == "uninstall": + return "Module owns migration metadata and may leave dormant schema behind." + if has_manifest_metadata: + return "Installed module owns migration metadata." + if has_catalog_metadata: + return "Catalog target declares migration metadata." + return "Manual backend package may add migration metadata after installation." + + def _migration_step_from_node(node: Mapping[str, object]) -> ModuleMigrationPlanStep: return ModuleMigrationPlanStep( module_id=str(node["module_id"]), @@ -1664,18 +2250,33 @@ def _migration_target_metadata( manifest: ModuleManifest | None, catalog_module: Mapping[str, object] | None, ) -> dict[str, object]: - dependencies: tuple[str, ...] = () - optional_dependencies: tuple[str, ...] = () - provides_interfaces: tuple[Mapping[str, object], ...] = () - requires_interfaces: tuple[Mapping[str, object], ...] = () - migration_after: tuple[str, ...] = () - migration_before: tuple[str, ...] = () - migration_tasks: tuple[Mapping[str, object], ...] = () + metadata = _empty_migration_target_metadata(module_id) if manifest is not None: - dependencies = tuple(manifest.dependencies) - optional_dependencies = tuple(manifest.optional_dependencies) - provides_interfaces = tuple({"name": item.name, "version": item.version} for item in manifest.provides_interfaces) - requires_interfaces = tuple( + metadata.update(_manifest_migration_target_metadata(manifest)) + if catalog_module is not None: + _apply_catalog_migration_target_metadata(metadata, catalog_module) + return metadata + + +def _empty_migration_target_metadata(module_id: str) -> dict[str, object]: + return { + "module_id": module_id, + "dependencies": (), + "optional_dependencies": (), + "provides_interfaces": (), + "requires_interfaces": (), + "migration_after": (), + "migration_before": (), + "migration_tasks": (), + } + + +def _manifest_migration_target_metadata(manifest: ModuleManifest) -> dict[str, object]: + metadata: dict[str, object] = { + "dependencies": tuple(manifest.dependencies), + "optional_dependencies": tuple(manifest.optional_dependencies), + "provides_interfaces": tuple({"name": item.name, "version": item.version} for item in manifest.provides_interfaces), + "requires_interfaces": tuple( { "name": item.name, "version_min": item.version_min, @@ -1683,44 +2284,42 @@ def _migration_target_metadata( "optional": item.optional, } for item in manifest.requires_interfaces - ) - if manifest.migration_spec is not None: - migration_after = tuple(manifest.migration_spec.migration_after) - migration_before = tuple(manifest.migration_spec.migration_before) - migration_tasks = tuple( - { - "task_id": task.task_id, - "phase": task.phase, - "summary": task.summary, - "task_version": task.task_version, - "safety": task.safety, - "idempotent": task.idempotent, - "timeout_seconds": task.timeout_seconds, - "source": "manifest", - "has_executor": task.executor is not None, - } - for task in manifest.migration_spec.migration_tasks - ) - if catalog_module is not None: - dependencies = _catalog_string_list(catalog_module.get("dependencies")) or dependencies - optional_dependencies = _catalog_string_list(catalog_module.get("optional_dependencies")) or optional_dependencies - provides_interfaces = _catalog_interface_items(catalog_module.get("provides_interfaces")) or provides_interfaces - requires_interfaces = _catalog_interface_items(catalog_module.get("requires_interfaces")) or requires_interfaces - migration_after = _catalog_string_list(catalog_module.get("migration_after")) or migration_after - migration_before = _catalog_string_list(catalog_module.get("migration_before")) or migration_before - catalog_tasks = _catalog_migration_task_items(catalog_module.get("migration_tasks")) - if catalog_tasks: - migration_tasks = catalog_tasks - return { - "module_id": module_id, - "dependencies": dependencies, - "optional_dependencies": optional_dependencies, - "provides_interfaces": provides_interfaces, - "requires_interfaces": requires_interfaces, - "migration_after": migration_after, - "migration_before": migration_before, - "migration_tasks": migration_tasks, + ), } + if manifest.migration_spec is not None: + metadata["migration_after"] = tuple(manifest.migration_spec.migration_after) + metadata["migration_before"] = tuple(manifest.migration_spec.migration_before) + metadata["migration_tasks"] = tuple(_manifest_migration_task_metadata(task) for task in manifest.migration_spec.migration_tasks) + return metadata + + +def _manifest_migration_task_metadata(task: object) -> dict[str, object]: + return { + "task_id": task.task_id, + "phase": task.phase, + "summary": task.summary, + "task_version": task.task_version, + "safety": task.safety, + "idempotent": task.idempotent, + "timeout_seconds": task.timeout_seconds, + "source": "manifest", + "has_executor": task.executor is not None, + } + + +def _apply_catalog_migration_target_metadata(metadata: dict[str, object], catalog_module: Mapping[str, object]) -> None: + _apply_catalog_metadata_value(metadata, "dependencies", _catalog_string_list(catalog_module.get("dependencies"))) + _apply_catalog_metadata_value(metadata, "optional_dependencies", _catalog_string_list(catalog_module.get("optional_dependencies"))) + _apply_catalog_metadata_value(metadata, "provides_interfaces", _catalog_interface_items(catalog_module.get("provides_interfaces"))) + _apply_catalog_metadata_value(metadata, "requires_interfaces", _catalog_interface_items(catalog_module.get("requires_interfaces"))) + _apply_catalog_metadata_value(metadata, "migration_after", _catalog_string_list(catalog_module.get("migration_after"))) + _apply_catalog_metadata_value(metadata, "migration_before", _catalog_string_list(catalog_module.get("migration_before"))) + _apply_catalog_metadata_value(metadata, "migration_tasks", _catalog_migration_task_items(catalog_module.get("migration_tasks"))) + + +def _apply_catalog_metadata_value(metadata: dict[str, object], key: str, value: tuple[object, ...]) -> None: + if value: + metadata[key] = value def _migration_task_plan_items( @@ -1730,36 +2329,52 @@ def _migration_task_plan_items( ) -> tuple[ModuleMigrationTaskPlanItem, ...]: tasks: list[ModuleMigrationTaskPlanItem] = [] for module_id in ordered_ids: - node = nodes.get(module_id) - metadata = target_metadata.get(module_id) - if node is None or metadata is None: - continue - metadata_pending = node.get("metadata_pending") is True - migration_tasks = tuple( - task for task in metadata.get("migration_tasks", ()) - if isinstance(task, Mapping) - ) - for phase in MIGRATION_TASK_PHASES: - for task in migration_tasks: - if str(task.get("phase") or "") != phase: - continue - source = str(task.get("source") or node.get("source") or "manifest") - tasks.append(ModuleMigrationTaskPlanItem( - module_id=module_id, - task_id=str(task.get("task_id") or ""), - phase=phase, - summary=str(task.get("summary") or ""), - task_version=str(task.get("task_version") or "1"), - safety=str(task.get("safety") or "automatic"), - idempotent=task.get("idempotent") is not False, - timeout_seconds=task.get("timeout_seconds") if isinstance(task.get("timeout_seconds"), int) else None, - source=source, - has_executor=task.get("has_executor") is True, - metadata_pending=metadata_pending or source != "manifest", - )) + tasks.extend(_migration_task_plan_items_for_module(module_id=module_id, target_metadata=target_metadata, nodes=nodes)) return tuple(tasks) +def _migration_task_plan_items_for_module( + *, + module_id: str, + target_metadata: Mapping[str, Mapping[str, object]], + nodes: Mapping[str, Mapping[str, object]], +) -> tuple[ModuleMigrationTaskPlanItem, ...]: + node = nodes.get(module_id) + metadata = target_metadata.get(module_id) + if node is None or metadata is None: + return () + tasks: list[ModuleMigrationTaskPlanItem] = [] + migration_tasks = tuple(task for task in metadata.get("migration_tasks", ()) if isinstance(task, Mapping)) + for phase in MIGRATION_TASK_PHASES: + for task in migration_tasks: + if str(task.get("phase") or "") == phase: + tasks.append(_migration_task_plan_item(module_id=module_id, node=node, task=task, phase=phase)) + return tuple(tasks) + + +def _migration_task_plan_item( + *, + module_id: str, + node: Mapping[str, object], + task: Mapping[str, object], + phase: str, +) -> ModuleMigrationTaskPlanItem: + source = str(task.get("source") or node.get("source") or "manifest") + return ModuleMigrationTaskPlanItem( + module_id=module_id, + task_id=str(task.get("task_id") or ""), + phase=phase, + summary=str(task.get("summary") or ""), + task_version=str(task.get("task_version") or "1"), + safety=str(task.get("safety") or "automatic"), + idempotent=task.get("idempotent") is not False, + timeout_seconds=task.get("timeout_seconds") if isinstance(task.get("timeout_seconds"), int) else None, + source=source, + has_executor=task.get("has_executor") is True, + metadata_pending=node.get("metadata_pending") is True or source != "manifest", + ) + + def _migration_task_preflight_issues( tasks: tuple[ModuleMigrationTaskPlanItem, ...], planned_by_id: Mapping[str, ModuleInstallPlanItem], @@ -1873,26 +2488,49 @@ def _add_migration_edge( def _topological_module_order(edges: Mapping[str, set[str]]) -> tuple[tuple[str, ...], tuple[str, ...]]: + incoming, outgoing = _topological_graph_indexes(edges) + ready = _topological_ready_nodes(incoming) + ordered: list[str] = [] + while ready: + module_id = ready.pop(0) + ordered.append(module_id) + _release_topological_node(module_id, incoming=incoming, outgoing=outgoing, ordered=ordered, ready=ready) + if len(ordered) == len(incoming): + return tuple(ordered), () + return tuple(ordered), _topological_cycle_ids(incoming) + + +def _topological_graph_indexes(edges: Mapping[str, set[str]]) -> tuple[dict[str, set[str]], dict[str, set[str]]]: incoming: dict[str, set[str]] = {module_id: set() for module_id in edges} outgoing: dict[str, set[str]] = {module_id: set(values) for module_id, values in edges.items()} for before, after_values in edges.items(): for after in after_values: incoming.setdefault(after, set()).add(before) outgoing.setdefault(before, set()).add(after) - ready = sorted(module_id for module_id, values in incoming.items() if not values) - ordered: list[str] = [] - while ready: - module_id = ready.pop(0) - ordered.append(module_id) - for after in sorted(outgoing.get(module_id, ())): - incoming[after].discard(module_id) - if not incoming[after] and after not in ordered and after not in ready: - ready.append(after) - ready.sort() - if len(ordered) == len(incoming): - return tuple(ordered), () - cycle_ids = tuple(sorted(module_id for module_id, values in incoming.items() if values)) - return tuple(ordered), cycle_ids + return incoming, outgoing + + +def _topological_ready_nodes(incoming: Mapping[str, set[str]]) -> list[str]: + return sorted(module_id for module_id, values in incoming.items() if not values) + + +def _release_topological_node( + module_id: str, + *, + incoming: dict[str, set[str]], + outgoing: Mapping[str, set[str]], + ordered: list[str], + ready: list[str], +) -> None: + for after in sorted(outgoing.get(module_id, ())): + incoming[after].discard(module_id) + if not incoming[after] and after not in ordered and after not in ready: + ready.append(after) + ready.sort() + + +def _topological_cycle_ids(incoming: Mapping[str, set[str]]) -> tuple[str, ...]: + return tuple(sorted(module_id for module_id, values in incoming.items() if values)) def _catalog_modules_for_target_plan( @@ -2025,81 +2663,11 @@ def _catalog_data_safety_issues( if safety == "automatic": return () if safety == "requires_review": - severity: IssueSeverity = "info" if item.data_safety_acknowledged else "warning" - code = "migration_review_acknowledged" if item.data_safety_acknowledged else "migration_review_required" - message = "Catalog marks this package change as requiring migration review before activation." + detail - return (ModuleInstallerIssue(severity, code, message, item.module_id),) + return _catalog_review_safety_issue(item=item, detail=detail) if safety == "forward_only": - issues: list[ModuleInstallerIssue] = [] - recovery_notes = _catalog_optional_string(module, "recovery_notes") - if not _catalog_optional_bool(module, "recovery_tested"): - issues.append(ModuleInstallerIssue( - "blocker", - "recovery_validation_required", - "Forward-only catalog updates must declare recovery_tested=true after a verified restore or forward-recovery rehearsal.", - item.module_id, - )) - if not recovery_notes and not item.notes: - issues.append(ModuleInstallerIssue( - "blocker", - "recovery_notes_required", - "Forward-only catalog updates require recovery notes or operator notes describing the tested recovery path.", - item.module_id, - )) - if not item.data_safety_acknowledged: - issues.append(ModuleInstallerIssue( - "blocker", - "forward_only_migration_acknowledgement_required", - "Catalog marks this package change as forward-only; acknowledge the data safety review before activation." + detail, - item.module_id, - )) - if issues: - return tuple(issues) - return (ModuleInstallerIssue( - "warning", - "forward_only_migration_acknowledged", - "Forward-only package change has been acknowledged by the operator." + detail, - item.module_id, - ),) + return _catalog_forward_only_safety_issues(item=item, module=module, detail=detail) if safety == "destructive": - issues: list[ModuleInstallerIssue] = [] - recovery_notes = _catalog_optional_string(module, "recovery_notes") - if not _catalog_optional_bool(module, "recovery_tested"): - issues.append(ModuleInstallerIssue( - "blocker", - "recovery_validation_required", - "Destructive catalog updates must declare recovery_tested=true after a verified restore or forward-recovery rehearsal.", - item.module_id, - )) - if not recovery_notes and not item.notes: - issues.append(ModuleInstallerIssue( - "blocker", - "recovery_notes_required", - "Destructive catalog updates require recovery notes or operator notes describing the tested recovery path.", - item.module_id, - )) - if not notes and not item.notes: - issues.append(ModuleInstallerIssue( - "blocker", - "destructive_migration_plan_required", - "Catalog marks this package change as destructive; add catalog migration notes or operator notes describing the cleanup/retirement plan.", - item.module_id, - )) - if not item.data_safety_acknowledged: - issues.append(ModuleInstallerIssue( - "blocker", - "destructive_migration_acknowledgement_required", - "Catalog marks this package change as destructive; acknowledge the data safety review before activation." + detail, - item.module_id, - )) - if not issues: - issues.append(ModuleInstallerIssue( - "warning", - "destructive_migration_acknowledged", - "Destructive package change has been acknowledged by the operator." + detail, - item.module_id, - )) - return tuple(issues) + return _catalog_destructive_safety_issues(item=item, module=module, notes=notes, detail=detail) return (ModuleInstallerIssue( "blocker", "catalog_migration_safety_invalid", @@ -2108,6 +2676,99 @@ def _catalog_data_safety_issues( ),) +def _catalog_review_safety_issue(*, item: ModuleInstallPlanItem, detail: str) -> tuple[ModuleInstallerIssue, ...]: + severity: IssueSeverity = "info" if item.data_safety_acknowledged else "warning" + code = "migration_review_acknowledged" if item.data_safety_acknowledged else "migration_review_required" + message = "Catalog marks this package change as requiring migration review before activation." + detail + return (ModuleInstallerIssue(severity, code, message, item.module_id),) + + +def _catalog_forward_only_safety_issues( + *, + item: ModuleInstallPlanItem, + module: Mapping[str, object], + detail: str, +) -> tuple[ModuleInstallerIssue, ...]: + issues: list[ModuleInstallerIssue] = [] + recovery_notes = _catalog_optional_string(module, "recovery_notes") + if not _catalog_optional_bool(module, "recovery_tested"): + issues.append(ModuleInstallerIssue( + "blocker", + "recovery_validation_required", + "Forward-only catalog updates must declare recovery_tested=true after a verified restore or forward-recovery rehearsal.", + item.module_id, + )) + if not recovery_notes and not item.notes: + issues.append(ModuleInstallerIssue( + "blocker", + "recovery_notes_required", + "Forward-only catalog updates require recovery notes or operator notes describing the tested recovery path.", + item.module_id, + )) + if not item.data_safety_acknowledged: + issues.append(ModuleInstallerIssue( + "blocker", + "forward_only_migration_acknowledgement_required", + "Catalog marks this package change as forward-only; acknowledge the data safety review before activation." + detail, + item.module_id, + )) + if issues: + return tuple(issues) + return (ModuleInstallerIssue( + "warning", + "forward_only_migration_acknowledged", + "Forward-only package change has been acknowledged by the operator." + detail, + item.module_id, + ),) + + +def _catalog_destructive_safety_issues( + *, + item: ModuleInstallPlanItem, + module: Mapping[str, object], + notes: str | None, + detail: str, +) -> tuple[ModuleInstallerIssue, ...]: + issues: list[ModuleInstallerIssue] = [] + recovery_notes = _catalog_optional_string(module, "recovery_notes") + if not _catalog_optional_bool(module, "recovery_tested"): + issues.append(ModuleInstallerIssue( + "blocker", + "recovery_validation_required", + "Destructive catalog updates must declare recovery_tested=true after a verified restore or forward-recovery rehearsal.", + item.module_id, + )) + if not recovery_notes and not item.notes: + issues.append(ModuleInstallerIssue( + "blocker", + "recovery_notes_required", + "Destructive catalog updates require recovery notes or operator notes describing the tested recovery path.", + item.module_id, + )) + if not notes and not item.notes: + issues.append(ModuleInstallerIssue( + "blocker", + "destructive_migration_plan_required", + "Catalog marks this package change as destructive; add catalog migration notes or operator notes describing the cleanup/retirement plan.", + item.module_id, + )) + if not item.data_safety_acknowledged: + issues.append(ModuleInstallerIssue( + "blocker", + "destructive_migration_acknowledgement_required", + "Catalog marks this package change as destructive; acknowledge the data safety review before activation." + detail, + item.module_id, + )) + if not issues: + issues.append(ModuleInstallerIssue( + "warning", + "destructive_migration_acknowledged", + "Destructive package change has been acknowledged by the operator." + detail, + item.module_id, + )) + return tuple(issues) + + def _catalog_update_policy_issues( *, item: ModuleInstallPlanItem, @@ -2121,64 +2782,88 @@ def _catalog_update_policy_issues( return () target_version = _catalog_optional_string(module, "version") issues: list[ModuleInstallerIssue] = [] - current_version_min = _catalog_optional_string(module, "current_version_min") - current_version_max_exclusive = _catalog_optional_string(module, "current_version_max_exclusive") - if target_version: - comparison = compare_versions(target_version, manifest.version) - if comparison < 0 and not _catalog_optional_bool(module, "allow_downgrade"): - issues.append(ModuleInstallerIssue( - "blocker", - "catalog_downgrade_blocked", - ( - f"Catalog target version {target_version} is older than installed version " - f"{manifest.version}; set allow_downgrade only for a reviewed rollback path." - ), - item.module_id, - )) - elif comparison == 0 and not _catalog_optional_bool(module, "allow_same_version"): - issues.append(ModuleInstallerIssue( - "blocker", - "catalog_noop_update_blocked", - ( - f"Catalog target version {target_version} matches the installed version; " - "use an install plan only when package refs or metadata intentionally need refreshing." - ), - item.module_id, - )) - if ( - current_version_min is not None - or current_version_max_exclusive is not None - ) and not version_satisfies_range( - manifest.version, - version_min=current_version_min, - version_max_exclusive=current_version_max_exclusive, - ): - version_range = format_version_range( - version_min=current_version_min, - version_max_exclusive=current_version_max_exclusive, - ) - bridge_note = _catalog_optional_string(module, "bridge_notes") - bridge_detail = f" Bridge note: {bridge_note}" if bridge_note else "" - issues.append(ModuleInstallerIssue( + issues.extend(_catalog_target_version_policy_issues(item=item, module=module, manifest=manifest, target_version=target_version)) + issues.extend(_catalog_update_window_issues(item=item, module=module, manifest=manifest, target_version=target_version)) + issues.extend(_catalog_bridge_release_issues(item=item, module=module)) + return tuple(issues) + + +def _catalog_target_version_policy_issues( + *, + item: ModuleInstallPlanItem, + module: Mapping[str, object], + manifest: ModuleManifest, + target_version: str | None, +) -> tuple[ModuleInstallerIssue, ...]: + if not target_version: + return () + comparison = compare_versions(target_version, manifest.version) + if comparison < 0 and not _catalog_optional_bool(module, "allow_downgrade"): + return (ModuleInstallerIssue( "blocker", - "catalog_update_bridge_required", + "catalog_downgrade_blocked", ( - f"Installed version {manifest.version} is outside the supported update window " - f"for catalog target {target_version or item.module_id} ({version_range})." - + bridge_detail + f"Catalog target version {target_version} is older than installed version " + f"{manifest.version}; set allow_downgrade only for a reviewed rollback path." ), item.module_id, - )) - if _catalog_optional_bool(module, "bridge_release"): - bridge_note = _catalog_optional_string(module, "bridge_notes") - detail = f" Bridge note: {bridge_note}" if bridge_note else "" - issues.append(ModuleInstallerIssue( - "info", - "catalog_bridge_release", - "Catalog marks this target as a bridge release for staged update paths." + detail, + ),) + if comparison == 0 and not _catalog_optional_bool(module, "allow_same_version"): + return (ModuleInstallerIssue( + "blocker", + "catalog_noop_update_blocked", + ( + f"Catalog target version {target_version} matches the installed version; " + "use an install plan only when package refs or metadata intentionally need refreshing." + ), item.module_id, - )) - return tuple(issues) + ),) + return () + + +def _catalog_update_window_issues( + *, + item: ModuleInstallPlanItem, + module: Mapping[str, object], + manifest: ModuleManifest, + target_version: str | None, +) -> tuple[ModuleInstallerIssue, ...]: + current_version_min = _catalog_optional_string(module, "current_version_min") + current_version_max_exclusive = _catalog_optional_string(module, "current_version_max_exclusive") + if current_version_min is None and current_version_max_exclusive is None: + return () + if version_satisfies_range(manifest.version, version_min=current_version_min, version_max_exclusive=current_version_max_exclusive): + return () + version_range = format_version_range(version_min=current_version_min, version_max_exclusive=current_version_max_exclusive) + bridge_note = _catalog_optional_string(module, "bridge_notes") + bridge_detail = f" Bridge note: {bridge_note}" if bridge_note else "" + return (ModuleInstallerIssue( + "blocker", + "catalog_update_bridge_required", + ( + f"Installed version {manifest.version} is outside the supported update window " + f"for catalog target {target_version or item.module_id} ({version_range})." + + bridge_detail + ), + item.module_id, + ),) + + +def _catalog_bridge_release_issues( + *, + item: ModuleInstallPlanItem, + module: Mapping[str, object], +) -> tuple[ModuleInstallerIssue, ...]: + if not _catalog_optional_bool(module, "bridge_release"): + return () + bridge_note = _catalog_optional_string(module, "bridge_notes") + detail = f" Bridge note: {bridge_note}" if bridge_note else "" + return (ModuleInstallerIssue( + "info", + "catalog_bridge_release", + "Catalog marks this target as a bridge release for staged update paths." + detail, + item.module_id, + ),) def _catalog_modules_by_id(validation: Mapping[str, object]) -> dict[str, Mapping[str, object]]: @@ -2556,8 +3241,8 @@ def _rollback_after_supervisor_failure( } try: session.rollback() - except Exception: - pass + except Exception as rollback_exc: + logger.debug("Module installer rollback failed before rollback plan: %s", rollback_exc, exc_info=True) rollback = rollback_module_install_run(run_id=result.run_id, runtime_dir=runtime_dir, npm_bin=npm_bin, build_webui=build_webui) rollback_record = _read_run_record(result.record_path) or {} if not _database_restore_was_applied(rollback_record): @@ -2686,21 +3371,21 @@ def _python_package_name_from_ref(value: str) -> str | None: def _wait_for_health_url(url: str, *, timeout_seconds: float, interval_seconds: float) -> dict[str, object]: - parsed = urllib.parse.urlparse(url) - if parsed.scheme not in {"http", "https"} or not parsed.netloc or parsed.username or parsed.password: - return {"ok": False, "attempts": 0, "error": "health URL must be an absolute HTTP(S) URL without embedded credentials"} + try: + health_url = validate_http_url(url, label="health URL") + except ValueError as exc: + return {"ok": False, "attempts": 0, "error": str(exc)} deadline = time.monotonic() + max(timeout_seconds, 0.1) attempts = 0 last_error = "" while time.monotonic() < deadline: attempts += 1 try: - with urllib.request.urlopen(url, timeout=min(max(interval_seconds, 0.5), 10.0)) as response: # noqa: S310 - validated module health HTTP(S) URL. # nosemgrep: python.lang.security.audit.dynamic-urllib-use-detected.dynamic-urllib-use-detected - status = int(getattr(response, "status", 0)) - if 200 <= status < 400: - return {"ok": True, "status": status, "attempts": attempts} - last_error = f"HTTP {status}" - except (OSError, urllib.error.URLError) as exc: + response = fetch_http(health_url, timeout=min(max(interval_seconds, 0.5), 10.0), label="health URL") + if 200 <= response.status < 400: + return {"ok": True, "status": response.status, "attempts": attempts} + last_error = f"HTTP {response.status}" + except OSError as exc: last_error = str(exc) time.sleep(max(interval_seconds, 0.1)) return {"ok": False, "attempts": attempts, "error": last_error or "health timeout"} @@ -2943,7 +3628,7 @@ def _snapshot_environment( database_restore_check_command: str | None, ) -> dict[str, object]: snapshot: dict[str, object] = {} - freeze_result = subprocess.run([sys.executable, "-m", "pip", "freeze"], text=True, capture_output=True, check=False) + freeze_result = _run_installer_subprocess([sys.executable, "-m", "pip", "freeze"]) if freeze_result.returncode == 0: freeze_path = run_dir / "pip-freeze.txt" freeze_path.write_text(freeze_result.stdout, encoding="utf-8") @@ -3146,8 +3831,8 @@ def _write_installer_secret(path: Path, value: str | None) -> str | None: path.write_text(value, encoding="utf-8") try: path.chmod(0o600) - except OSError: - pass + except OSError as exc: + logger.debug("Could not restrict installer secret file permissions for %s: %s", path, exc, exc_info=True) return path.name @@ -3201,6 +3886,65 @@ def _redact_url_credentials(value: str) -> str: return re.sub(r"([A-Za-z][A-Za-z0-9+.-]*://[^\s/@:]+:)([^@\s]+)(@)", r"\1***\3", value) +def _run_structured_installer_command(command: Mapping[str, Any]) -> subprocess.CompletedProcess[str]: + try: + argv = _validated_installer_argv(command.get("argv")) + cwd = _validated_installer_cwd(command.get("cwd")) + except ValueError as exc: + return subprocess.CompletedProcess(args=command.get("argv"), returncode=2, stdout="", stderr=str(exc)) + return _run_installer_subprocess(argv, cwd=cwd) + + +def _run_installer_subprocess( + argv: Iterable[str], + *, + cwd: Path | str | None = None, + env: Mapping[str, str] | None = None, +) -> subprocess.CompletedProcess[str]: + command = _validated_installer_argv(argv) + effective_cwd = _validated_installer_cwd(cwd) + try: + # Audited subprocess choke point for the installer: callers must provide + # a validated argv sequence and `shell=True` is never used here. + return subprocess.run( # noqa: S603 # nosec B603 # nosemgrep: python.lang.security.audit.dangerous-subprocess-use.dangerous-subprocess-use + command, + cwd=effective_cwd, + env=dict(env) if env is not None else None, + text=True, + capture_output=True, + check=False, + ) + except OSError as exc: + return subprocess.CompletedProcess(args=command, returncode=127, stdout="", stderr=str(exc)) + + +def _validated_installer_argv(value: object) -> list[str]: + if not isinstance(value, (list, tuple)): + raise ValueError("Installer command argv must be a list or tuple") + if not value: + raise ValueError("Installer command argv must not be empty") + argv: list[str] = [] + for raw in value: + if not isinstance(raw, str): + raise ValueError("Installer command argv entries must be strings") + if "\x00" in raw: + raise ValueError("Installer command argv entries must not contain NUL bytes") + argv.append(raw) + return argv + + +def _validated_installer_cwd(value: object) -> str | None: + if value is None: + return None + if isinstance(value, Path): + return str(value) + if isinstance(value, str): + if "\x00" in value: + raise ValueError("Installer command cwd must not contain NUL bytes") + return value + raise ValueError("Installer command cwd must be a path string") + + def _run_database_hook( command: str, *, @@ -3234,17 +3978,7 @@ def _run_operator_command( argv = _operator_command_argv(command) except ValueError as exc: return subprocess.CompletedProcess(args=command, returncode=2, stdout="", stderr=str(exc)) - try: - return subprocess.run( - argv, - cwd=cwd, - env=dict(env) if env is not None else None, - text=True, - capture_output=True, - check=False, - ) - except OSError as exc: - return subprocess.CompletedProcess(args=argv, returncode=127, stdout="", stderr=str(exc)) + return _run_installer_subprocess(argv, cwd=cwd, env=env) def _operator_command_argv(command: str) -> tuple[str, ...]: @@ -3322,15 +4056,20 @@ def _mark_applied(item: ModuleInstallPlanItem) -> ModuleInstallPlanItem: ) -def _structured_command(argv: list[str], *, cwd: Path | None = None) -> dict[str, Any]: +def _structured_command(argv: list[str], *, cwd: Path | None = None, source: str) -> dict[str, Any]: + command = _validated_installer_argv(argv) + effective_cwd = _validated_installer_cwd(cwd) + if not source or "\x00" in source: + raise ValueError("Installer command source must be a non-empty string") return { - "argv": argv, - "cwd": str(cwd) if cwd is not None else None, - "display": _display_command(argv, cwd=cwd), + "argv": command, + "cwd": effective_cwd, + "display": _display_command(command, cwd=effective_cwd), + "source": source, } -def _display_command(argv: list[str], *, cwd: Path | None = None) -> str: +def _display_command(argv: list[str], *, cwd: Path | str | None = None) -> str: command = " ".join(_quote_arg(arg) for arg in argv) if cwd is not None: return f"cd {_quote_arg(str(cwd))} && {command}" @@ -3352,6 +4091,7 @@ def _command_record(command: Mapping[str, Any], *, redactions: Mapping[str, str] payload: dict[str, object] = { "display": _redact_installer_text(str(command["display"]), redactions=effective_redactions), "argv": [_redact_installer_text(str(item), redactions=effective_redactions) for item in command["argv"]], + "source": _redact_installer_text(str(command.get("source") or "unknown"), redactions=effective_redactions), } if command.get("cwd"): payload["cwd"] = _redact_installer_text(str(command["cwd"]), redactions=effective_redactions) diff --git a/src/govoplan_core/core/module_installer_notifications.py b/src/govoplan_core/core/module_installer_notifications.py new file mode 100644 index 0000000..db7bfff --- /dev/null +++ b/src/govoplan_core/core/module_installer_notifications.py @@ -0,0 +1,125 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +from govoplan_core.core.module_management import load_startup_enabled_modules, startup_candidate_module_ids +from govoplan_core.core.modules import ModuleContext +from govoplan_core.core.notifications import NotificationDispatchRequest, notification_dispatch_provider +from govoplan_core.server.registry import available_module_manifests, build_platform_registry + + +INSTALLER_NOTIFICATION_ACTION_URL = "/admin?section=modules" + + +def emit_module_installer_notification( + *, + session: object, + registry: object | None, + tenant_id: str | None, + request: Mapping[str, object], + event_kind: str, + subject: str, + body_text: str, + recipient_id: str | None = None, + priority: int = 5, +) -> bool: + if not tenant_id: + return False + provider = notification_dispatch_provider(registry) + if provider is None: + return False + request_id = str(request.get("request_id") or "") + if not request_id: + return False + provider.enqueue_notification( + session, + NotificationDispatchRequest( + tenant_id=tenant_id, + source_module="core", + source_resource_type="module_install_request", + source_resource_id=request_id, + event_kind=event_kind, + channel="inbox", + recipient_type="user" if recipient_id else None, + recipient_id=recipient_id, + subject=subject, + body_text=body_text, + action_url=INSTALLER_NOTIFICATION_ACTION_URL, + priority=priority, + payload={ + "request_id": request_id, + "status": request.get("status"), + "run_id": _result_run_id(request), + }, + metadata={ + "trace": request.get("trace") if isinstance(request.get("trace"), Mapping) else None, + "retry_of": request.get("retry_of"), + }, + ), + enqueue_delivery=False, + ) + return True + + +def build_runtime_notification_registry(settings: object) -> object | None: + try: + raw_enabled_modules = load_startup_enabled_modules(str(getattr(settings, "enabled_modules", "") or "")) + candidate_modules = startup_candidate_module_ids(str(getattr(settings, "enabled_modules", "") or ""), raw_enabled_modules) + available_modules = available_module_manifests(enabled_modules=candidate_modules, ignore_load_errors=True) + enabled_modules = load_startup_enabled_modules(str(getattr(settings, "enabled_modules", "") or ""), available=available_modules) + if "notifications" not in enabled_modules: + return None + registry = build_platform_registry(enabled_modules) + registry.configure_capability_context(ModuleContext(registry=registry, settings=settings)) + return registry + except Exception: # noqa: BLE001 - notification bridge must not block installer work. + return None + + +def installer_notification_subject(event_kind: str, request: Mapping[str, object]) -> str: + request_id = str(request.get("request_id") or "unknown") + status = str(request.get("status") or event_kind.rsplit(".", 1)[-1]) + prefixes = { + "queued": "Module installer request queued", + "running": "Module installer request started", + "completed": "Module installer request completed", + "failed": "Module installer request failed", + "cancelled": "Module installer request cancelled", + } + prefix = prefixes.get(status, "Module installer request updated") + return ": ".join((prefix, request_id)) + + +def _installer_status_sentence(request_id: str, status: str) -> str: + return " ".join(("Installer request", request_id, "is", status)) + + +def installer_notification_body(event_kind: str, request: Mapping[str, object]) -> str: + status = str(request.get("status") or event_kind.rsplit(".", 1)[-1]) + request_id = str(request.get("request_id") or "unknown") + result = request.get("result") if isinstance(request.get("result"), Mapping) else {} + error = str(request.get("error") or result.get("error") or "").strip() + sentence = _installer_status_sentence(request_id, status) + if error: + return ". Error: ".join((sentence, error)) + run_id = _result_run_id(request) + if run_id: + return ". Run: ".join((sentence, run_id)) + return sentence + "." + + +def installer_notification_priority(status: str) -> int: + if status == "failed": + return 20 + if status in {"completed", "cancelled"}: + return 8 + if status == "running": + return 4 + return 5 + + +def _result_run_id(request: Mapping[str, object]) -> str | None: + result = request.get("result") if isinstance(request.get("result"), Mapping) else {} + run_id: Any = result.get("run_id") if isinstance(result, Mapping) else None + return str(run_id) if run_id else None diff --git a/src/govoplan_core/core/module_license.py b/src/govoplan_core/core/module_license.py index 7f277f1..13ab67e 100644 --- a/src/govoplan_core/core/module_license.py +++ b/src/govoplan_core/core/module_license.py @@ -7,13 +7,11 @@ import json import os from pathlib import Path from typing import Any -import urllib.error -import urllib.parse -import urllib.request from cryptography.exceptions import InvalidSignature from cryptography.hazmat.primitives import serialization from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey, Ed25519PublicKey +from govoplan_core.security.http_fetch import fetch_http_text def module_license_decision(required_features: list[str] | tuple[str, ...]) -> dict[str, object]: @@ -258,18 +256,14 @@ def _configured_trusted_keys_cache_path() -> Path | None: def _read_trusted_keys_url(url: str) -> str: - parsed = urllib.parse.urlparse(url) - if parsed.scheme not in {"http", "https"} or not parsed.netloc or parsed.username or parsed.password: - raise ValueError("Trusted license key URL must be an absolute HTTP(S) URL without embedded credentials.") cache_path = _configured_trusted_keys_cache_path() try: - with urllib.request.urlopen(url, timeout=15) as response: # noqa: S310 - validated license key HTTP(S) URL. # nosemgrep: python.lang.security.audit.dynamic-urllib-use-detected.dynamic-urllib-use-detected - body = response.read().decode("utf-8") + body = fetch_http_text(url, timeout=15, label="Trusted license key URL") if cache_path is not None: cache_path.parent.mkdir(parents=True, exist_ok=True) cache_path.write_text(body, encoding="utf-8") return body - except (OSError, urllib.error.URLError): + except OSError: if cache_path is not None and cache_path.exists(): return cache_path.read_text(encoding="utf-8") raise diff --git a/src/govoplan_core/core/module_management.py b/src/govoplan_core/core/module_management.py index 13ea7a7..e7728cb 100644 --- a/src/govoplan_core/core/module_management.py +++ b/src/govoplan_core/core/module_management.py @@ -73,6 +73,23 @@ class ModuleInstallPlanItem: return payload +@dataclass(frozen=True, slots=True) +class _NormalizedModuleInstallPlanItem: + module_id: str + action: str + source: str + catalog: Mapping[str, object] | None + status: str + python_package: str | None + python_ref: str | None + webui_package: str | None + webui_ref: str | None + artifact_integrity: Mapping[str, object] | None + data_safety_acknowledged: bool + destroy_data: bool + notes: str | None + + @dataclass(frozen=True, slots=True) class ModuleInstallPlan: items: tuple[ModuleInstallPlanItem, ...] = () @@ -291,6 +308,28 @@ def desired_modules_after_package_plan( def normalize_module_install_plan_item( item: Mapping[str, object] | ModuleInstallPlanItem, ) -> ModuleInstallPlanItem: + normalized = _normalized_module_install_plan_item(item) + _validate_module_install_plan_item(normalized) + return ModuleInstallPlanItem( + module_id=normalized.module_id, + action=normalized.action, + source=normalized.source, + catalog=normalized.catalog, + python_package=normalized.python_package, + python_ref=normalized.python_ref, + webui_package=normalized.webui_package, + webui_ref=normalized.webui_ref, + artifact_integrity=normalized.artifact_integrity, + data_safety_acknowledged=normalized.data_safety_acknowledged, + destroy_data=normalized.destroy_data, + status=normalized.status, + notes=normalized.notes, + ) + + +def _normalized_module_install_plan_item( + item: Mapping[str, object] | ModuleInstallPlanItem, +) -> _NormalizedModuleInstallPlanItem: if isinstance(item, ModuleInstallPlanItem): raw = item.as_dict() elif isinstance(item, Mapping): @@ -311,33 +350,12 @@ def normalize_module_install_plan_item( data_safety_acknowledged = _clean_bool(raw.get("data_safety_acknowledged")) destroy_data = _clean_bool(raw.get("destroy_data")) notes = _clean_optional_string(raw.get("notes")) - - if action not in INSTALL_PLAN_ACTIONS: - raise ModuleManagementError(f"Unsupported install plan action for {module_id!r}: {action!r}.") - if status not in INSTALL_PLAN_STATUSES: - raise ModuleManagementError(f"Unsupported install plan status for {module_id!r}: {status!r}.") - if source not in INSTALL_PLAN_SOURCES: - raise ModuleManagementError(f"Unsupported install plan source for {module_id!r}: {source!r}.") - if action in {"install", "update"} and not python_ref: - raise ModuleManagementError(f"Install plan item {module_id!r} needs a Python package reference.") - if action == "uninstall" and not python_package: - raise ModuleManagementError(f"Uninstall plan item {module_id!r} needs a Python package name.") - if action != "uninstall" and destroy_data: - raise ModuleManagementError(f"Install plan item {module_id!r} can only destroy data during uninstall.") - if action in {"install", "update"} and bool(webui_package) != bool(webui_ref): - raise ModuleManagementError(f"Install plan item {module_id!r} needs both WebUI package and WebUI reference, or neither.") - if action == "uninstall" and webui_ref and not webui_package: - raise ModuleManagementError(f"Uninstall plan item {module_id!r} has a WebUI reference but no WebUI package.") - if python_ref: - _validate_dependency_ref(python_ref, field="python_ref", module_id=module_id) - if webui_ref: - _validate_dependency_ref(webui_ref, field="webui_ref", module_id=module_id) - - return ModuleInstallPlanItem( + return _NormalizedModuleInstallPlanItem( module_id=module_id, action=action, source=source, catalog=catalog, + status=status, python_package=python_package, python_ref=python_ref, webui_package=webui_package, @@ -345,11 +363,41 @@ def normalize_module_install_plan_item( artifact_integrity=artifact_integrity, data_safety_acknowledged=data_safety_acknowledged, destroy_data=destroy_data, - status=status, notes=notes, ) +def _validate_module_install_plan_item(item: _NormalizedModuleInstallPlanItem) -> None: + if item.action not in INSTALL_PLAN_ACTIONS: + raise ModuleManagementError(f"Unsupported install plan action for {item.module_id!r}: {item.action!r}.") + if item.status not in INSTALL_PLAN_STATUSES: + raise ModuleManagementError(f"Unsupported install plan status for {item.module_id!r}: {item.status!r}.") + if item.source not in INSTALL_PLAN_SOURCES: + raise ModuleManagementError(f"Unsupported install plan source for {item.module_id!r}: {item.source!r}.") + _validate_module_install_plan_item_requirements(item) + _validate_module_install_plan_item_refs(item) + + +def _validate_module_install_plan_item_requirements(item: _NormalizedModuleInstallPlanItem) -> None: + if item.action in {"install", "update"} and not item.python_ref: + raise ModuleManagementError(f"Install plan item {item.module_id!r} needs a Python package reference.") + if item.action == "uninstall" and not item.python_package: + raise ModuleManagementError(f"Uninstall plan item {item.module_id!r} needs a Python package name.") + if item.action != "uninstall" and item.destroy_data: + raise ModuleManagementError(f"Install plan item {item.module_id!r} can only destroy data during uninstall.") + if item.action in {"install", "update"} and bool(item.webui_package) != bool(item.webui_ref): + raise ModuleManagementError(f"Install plan item {item.module_id!r} needs both WebUI package and WebUI reference, or neither.") + if item.action == "uninstall" and item.webui_ref and not item.webui_package: + raise ModuleManagementError(f"Uninstall plan item {item.module_id!r} has a WebUI reference but no WebUI package.") + + +def _validate_module_install_plan_item_refs(item: _NormalizedModuleInstallPlanItem) -> None: + if item.python_ref: + _validate_dependency_ref(item.python_ref, field="python_ref", module_id=item.module_id) + if item.webui_ref: + _validate_dependency_ref(item.webui_ref, field="webui_ref", module_id=item.module_id) + + def plan_desired_enabled_modules( requested_enabled: Iterable[str], available: Mapping[str, ModuleManifest], diff --git a/src/govoplan_core/core/module_package_catalog.py b/src/govoplan_core/core/module_package_catalog.py index 8f1b743..637b022 100644 --- a/src/govoplan_core/core/module_package_catalog.py +++ b/src/govoplan_core/core/module_package_catalog.py @@ -3,20 +3,20 @@ from __future__ import annotations import base64 import binascii from collections import defaultdict +from dataclasses import dataclass from datetime import UTC, datetime from pathlib import Path import json import os import re from typing import Any -import urllib.error -import urllib.request from cryptography.exceptions import InvalidSignature from cryptography.hazmat.primitives import serialization from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey, Ed25519PublicKey from govoplan_core.core.versioning import format_version_range, version_range_is_valid, version_satisfies_range +from govoplan_core.security.http_fetch import fetch_http_text, is_http_url _INTERFACE_NAME_RE = re.compile(r"^[a-z][a-z0-9_]*(?:\.[a-z][a-z0-9_]*)+$") CATALOG_MIGRATION_SAFETY = ("automatic", "requires_review", "forward_only", "destructive") @@ -28,6 +28,20 @@ CATALOG_MIGRATION_TASK_PHASES = ( ) +@dataclass(frozen=True, slots=True) +class _CatalogValidationState: + modules: tuple[dict[str, object], ...] + channel: str | None + sequence: int | None + generated_at: str | None + not_before: str | None + expires_at: str | None + signature_state: dict[str, object] + freshness: dict[str, object] + replay: dict[str, object] + read_state: dict[str, object] + + def module_package_catalog( path: Path | str | None = None, *, @@ -61,178 +75,143 @@ def validate_module_package_catalog( effective_approved_channels = _configured_approved_channels() if approved_channels is None else approved_channels effective_trusted_keys = trusted_keys if trusted_keys is not None else _configured_trusted_keys() if catalog_source is not None and not _catalog_source_exists(catalog_source): - return { - "valid": False, - "configured": True, - "path": str(catalog_source), - "source": str(catalog_source), - "source_type": _catalog_source_type(catalog_source), - "cache_used": False, - "cache_path": str(_configured_catalog_cache_path()) if _configured_catalog_cache_path() is not None else None, - "modules": [], - "channel": None, - "sequence": None, - "generated_at": None, - "not_before": None, - "expires_at": None, - "signed": False, - "trusted": False, - "key_id": None, - "warnings": [], - "error": f"Module package catalog does not exist: {catalog_source}", - } - warnings: list[str] = [] - read_state = {"cache_used": False, "cache_path": str(_configured_catalog_cache_path()) if _configured_catalog_cache_path() is not None else None} + return _catalog_error_result(catalog_source, error=f"Module package catalog does not exist: {catalog_source}") try: - payload, read_state = _read_catalog_payload_with_metadata(catalog_source) - modules = _normalize_catalog_modules(payload) - channel = _catalog_channel(payload) - sequence = _catalog_sequence(payload) - generated_at = _catalog_optional_text(payload, "generated_at") - not_before = _catalog_optional_text(payload, "not_before") - expires_at = _catalog_optional_text(payload, "expires_at") - signature_state = _catalog_signature_state(payload, trusted_keys=effective_trusted_keys) - freshness = _catalog_freshness_state(payload) - replay = _catalog_replay_state(channel=channel, sequence=sequence) + state = _catalog_validation_state(catalog_source, trusted_keys=effective_trusted_keys) except Exception as exc: - return { - "valid": False, - "configured": catalog_source is not None, - "path": str(catalog_source) if catalog_source is not None else None, - "source": str(catalog_source) if catalog_source is not None else None, - "source_type": _catalog_source_type(catalog_source), - "cache_used": bool(read_state.get("cache_used")), - "cache_path": read_state.get("cache_path") if isinstance(read_state.get("cache_path"), str) else None, - "modules": [], - "channel": None, - "sequence": None, - "generated_at": None, - "not_before": None, - "expires_at": None, - "signed": False, - "trusted": False, - "key_id": None, - "warnings": [], - "error": str(exc), - } - if signature_state.get("fatal"): - return { - "valid": False, - "configured": catalog_source is not None, - "path": str(catalog_source) if catalog_source is not None else None, - "source": str(catalog_source) if catalog_source is not None else None, - "source_type": _catalog_source_type(catalog_source), - "cache_used": bool(read_state.get("cache_used")), - "cache_path": read_state.get("cache_path") if isinstance(read_state.get("cache_path"), str) else None, - "modules": [], - "channel": channel, - "sequence": sequence, - "generated_at": generated_at, - "not_before": not_before, - "expires_at": expires_at, - "signed": signature_state["signed"], - "trusted": signature_state["trusted"], - "key_id": signature_state["key_id"], - "warnings": [], - "error": str(signature_state["error"] or "Module package catalog signature is invalid."), - } - if effective_approved_channels and channel not in effective_approved_channels: - return { - "valid": False, - "configured": catalog_source is not None, - "path": str(catalog_source) if catalog_source is not None else None, - "source": str(catalog_source) if catalog_source is not None else None, - "source_type": _catalog_source_type(catalog_source), - "cache_used": bool(read_state.get("cache_used")), - "cache_path": read_state.get("cache_path") if isinstance(read_state.get("cache_path"), str) else None, - "modules": [], - "channel": channel, - "sequence": sequence, - "generated_at": generated_at, - "not_before": not_before, - "expires_at": expires_at, - "signed": signature_state["signed"], - "trusted": signature_state["trusted"], - "key_id": signature_state["key_id"], - "warnings": [], - "error": f"Module package catalog channel {channel!r} is not approved. Approved channels: {', '.join(effective_approved_channels)}.", - } - if effective_require_trusted and not signature_state["trusted"]: - return { - "valid": False, - "configured": catalog_source is not None, - "path": str(catalog_source) if catalog_source is not None else None, - "source": str(catalog_source) if catalog_source is not None else None, - "source_type": _catalog_source_type(catalog_source), - "cache_used": bool(read_state.get("cache_used")), - "cache_path": read_state.get("cache_path") if isinstance(read_state.get("cache_path"), str) else None, - "modules": [], - "channel": channel, - "sequence": sequence, - "generated_at": generated_at, - "not_before": not_before, - "expires_at": expires_at, - "signed": signature_state["signed"], - "trusted": False, - "key_id": signature_state["key_id"], - "warnings": [], - "error": str(signature_state["error"] or "Module package catalog must be signed by a trusted key."), - } - if not freshness["valid"]: - return _invalid_catalog_result( - catalog_source, - modules=(), - channel=channel, - sequence=sequence, - generated_at=generated_at, - not_before=not_before, - expires_at=expires_at, - signature_state=signature_state, - read_state=read_state, - error=str(freshness["error"]), + return _catalog_error_result(catalog_source, error=str(exc)) + policy_error = _catalog_policy_error( + catalog_source, + state, + require_trusted=effective_require_trusted, + approved_channels=effective_approved_channels, + ) + if policy_error is not None: + return policy_error + return _valid_catalog_result(catalog_source, state) + + +def _catalog_validation_state( + source: Path | str | None, + *, + trusted_keys: dict[str, str], +) -> _CatalogValidationState: + payload, read_state = _read_catalog_payload_with_metadata(source) + modules = _normalize_catalog_modules(payload) + channel = _catalog_channel(payload) + sequence = _catalog_sequence(payload) + return _CatalogValidationState( + modules=modules, + channel=channel, + sequence=sequence, + generated_at=_catalog_optional_text(payload, "generated_at"), + not_before=_catalog_optional_text(payload, "not_before"), + expires_at=_catalog_optional_text(payload, "expires_at"), + signature_state=_catalog_signature_state(payload, trusted_keys=trusted_keys), + freshness=_catalog_freshness_state(payload), + replay=_catalog_replay_state(channel=channel, sequence=sequence), + read_state=read_state, + ) + + +def _catalog_policy_error( + source: Path | str | None, + state: _CatalogValidationState, + *, + require_trusted: bool, + approved_channels: tuple[str, ...], +) -> dict[str, object] | None: + if state.signature_state.get("fatal"): + return _invalid_catalog_state_result(source, state, error=str(state.signature_state["error"] or "Module package catalog signature is invalid.")) + if approved_channels and state.channel not in approved_channels: + return _invalid_catalog_state_result( + source, + state, + error=f"Module package catalog channel {state.channel!r} is not approved. Approved channels: {', '.join(approved_channels)}.", ) - if not replay["valid"]: - return _invalid_catalog_result( - catalog_source, - modules=(), - channel=channel, - sequence=sequence, - generated_at=generated_at, - not_before=not_before, - expires_at=expires_at, - signature_state=signature_state, - read_state=read_state, - error=str(replay["error"]), - ) - warnings.extend(str(item) for item in freshness.get("warnings", ()) if item) - warnings.extend(str(item) for item in replay.get("warnings", ()) if item) - warnings.extend(_catalog_interface_warnings(modules)) - if not signature_state["signed"]: - warnings.append("Catalog is unsigned; use only for local development unless signature enforcement is disabled intentionally.") - elif not signature_state["trusted"]: - warnings.append(str(signature_state["error"] or "Catalog signature could not be verified against a trusted key.")) + if require_trusted and not state.signature_state["trusted"]: + return _invalid_catalog_state_result(source, state, error=str(state.signature_state["error"] or "Module package catalog must be signed by a trusted key.")) + if not state.freshness["valid"]: + return _invalid_catalog_state_result(source, state, error=str(state.freshness["error"])) + if not state.replay["valid"]: + return _invalid_catalog_state_result(source, state, error=str(state.replay["error"])) + return None + + +def _catalog_error_result(source: Path | str | None, *, error: str) -> dict[str, object]: + return _invalid_catalog_result( + source, + modules=(), + channel=None, + sequence=None, + generated_at=None, + not_before=None, + expires_at=None, + signature_state=_unsigned_catalog_signature_state(), + read_state=_default_catalog_read_state(), + error=error, + ) + + +def _invalid_catalog_state_result(source: Path | str | None, state: _CatalogValidationState, *, error: str) -> dict[str, object]: + return _invalid_catalog_result( + source, + modules=(), + channel=state.channel, + sequence=state.sequence, + generated_at=state.generated_at, + not_before=state.not_before, + expires_at=state.expires_at, + signature_state=state.signature_state, + read_state=state.read_state, + error=error, + ) + + +def _valid_catalog_result(source: Path | str | None, state: _CatalogValidationState) -> dict[str, object]: return { "valid": True, - "configured": catalog_source is not None and _catalog_source_exists(catalog_source), - "path": str(catalog_source) if catalog_source is not None else None, - "source": str(catalog_source) if catalog_source is not None else None, - "source_type": _catalog_source_type(catalog_source), - "cache_used": bool(read_state.get("cache_used")), - "cache_path": read_state.get("cache_path") if isinstance(read_state.get("cache_path"), str) else None, - "modules": list(modules), - "channel": channel, - "sequence": sequence, - "generated_at": generated_at, - "not_before": not_before, - "expires_at": expires_at, - "signed": signature_state["signed"], - "trusted": signature_state["trusted"], - "key_id": signature_state["key_id"], - "warnings": warnings, + "configured": source is not None and _catalog_source_exists(source), + "path": str(source) if source is not None else None, + "source": str(source) if source is not None else None, + "source_type": _catalog_source_type(source), + "cache_used": bool(state.read_state.get("cache_used")), + "cache_path": state.read_state.get("cache_path") if isinstance(state.read_state.get("cache_path"), str) else None, + "modules": list(state.modules), + "channel": state.channel, + "sequence": state.sequence, + "generated_at": state.generated_at, + "not_before": state.not_before, + "expires_at": state.expires_at, + "signed": state.signature_state["signed"], + "trusted": state.signature_state["trusted"], + "key_id": state.signature_state["key_id"], + "warnings": _catalog_validation_warnings(state), "error": None, } +def _catalog_validation_warnings(state: _CatalogValidationState) -> list[str]: + warnings: list[str] = [] + warnings.extend(str(item) for item in state.freshness.get("warnings", ()) if item) + warnings.extend(str(item) for item in state.replay.get("warnings", ()) if item) + warnings.extend(_catalog_interface_warnings(state.modules)) + if not state.signature_state["signed"]: + warnings.append("Catalog is unsigned; use only for local development unless signature enforcement is disabled intentionally.") + elif not state.signature_state["trusted"]: + warnings.append(str(state.signature_state["error"] or "Catalog signature could not be verified against a trusted key.")) + return warnings + + +def _default_catalog_read_state() -> dict[str, object]: + cache_path = _configured_catalog_cache_path() + return {"cache_used": False, "cache_path": str(cache_path) if cache_path is not None else None} + + +def _unsigned_catalog_signature_state() -> dict[str, object]: + return {"signed": False, "trusted": False, "key_id": None} + + def sign_module_package_catalog( *, path: Path, @@ -350,17 +329,14 @@ def _configured_trusted_keys_cache_path() -> Path | None: def _read_trusted_keys_url(url: str) -> str: - if not _is_http_url(url): - raise ValueError("Trusted catalog key URL must use http:// or https://.") cache_path = _configured_trusted_keys_cache_path() try: - with urllib.request.urlopen(url, timeout=15) as response: # noqa: S310 - validated catalog key HTTP(S) URL. # nosemgrep: python.lang.security.audit.dynamic-urllib-use-detected.dynamic-urllib-use-detected - body = response.read().decode("utf-8") + body = fetch_http_text(url, timeout=15, label="Trusted catalog key URL") if cache_path is not None: cache_path.parent.mkdir(parents=True, exist_ok=True) cache_path.write_text(body, encoding="utf-8") return body - except (OSError, urllib.error.URLError): + except OSError: if cache_path is not None and cache_path.exists(): return cache_path.read_text(encoding="utf-8") raise @@ -421,13 +397,12 @@ def _read_catalog_url(url: str) -> str: def _read_catalog_url_with_metadata(url: str) -> tuple[str, bool]: cache_path = _configured_catalog_cache_path() try: - with urllib.request.urlopen(url, timeout=15) as response: # noqa: S310 - validated module catalog HTTP(S) URL. # nosemgrep: python.lang.security.audit.dynamic-urllib-use-detected.dynamic-urllib-use-detected - body = response.read().decode("utf-8") + body = fetch_http_text(url, timeout=15, label="Module package catalog URL") if cache_path is not None: cache_path.parent.mkdir(parents=True, exist_ok=True) cache_path.write_text(body, encoding="utf-8") return body, False - except (OSError, urllib.error.URLError): + except OSError: if cache_path is not None and cache_path.exists(): return cache_path.read_text(encoding="utf-8"), True raise @@ -935,8 +910,7 @@ def _catalog_source_type(source: Path | str | None) -> str | None: def _is_http_url(value: str) -> bool: - parsed = urllib.parse.urlparse(value) - return parsed.scheme in {"http", "https"} and bool(parsed.netloc) and not parsed.username and not parsed.password + return is_http_url(value) def _invalid_catalog_result( diff --git a/src/govoplan_core/core/notifications.py b/src/govoplan_core/core/notifications.py new file mode 100644 index 0000000..e768663 --- /dev/null +++ b/src/govoplan_core/core/notifications.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass, field +from datetime import datetime +from typing import Protocol, runtime_checkable + + +CAPABILITY_NOTIFICATIONS_DISPATCH = "notifications.dispatch" + + +@dataclass(frozen=True, slots=True) +class NotificationDispatchRequest: + tenant_id: str + source_module: str + source_resource_type: str + source_resource_id: str | None + event_kind: str + channel: str = "inbox" + recipient: str | None = None + recipient_type: str | None = None + recipient_id: str | None = None + recipient_label: str | None = None + subject: str | None = None + body_text: str | None = None + body_html: str | None = None + action_url: str | None = None + priority: int = 0 + not_before_at: datetime | None = None + payload: Mapping[str, object] = field(default_factory=dict) + metadata: Mapping[str, object] = field(default_factory=dict) + + +@runtime_checkable +class NotificationDispatchProvider(Protocol): + def enqueue_notification( + self, + session: object, + request: NotificationDispatchRequest, + *, + enqueue_delivery: bool = True, + ) -> Mapping[str, object]: + ... + + def deliver_notification(self, session: object, *, notification_id: str) -> Mapping[str, object]: + ... + + def deliver_pending( + self, + session: object, + *, + tenant_id: str | None = None, + limit: int = 50, + ) -> Mapping[str, object]: + ... + + +def notification_dispatch_provider(registry: object | None) -> NotificationDispatchProvider | None: + if registry is None or not hasattr(registry, "has_capability"): + return None + if not registry.has_capability(CAPABILITY_NOTIFICATIONS_DISPATCH): + return None + capability = registry.capability(CAPABILITY_NOTIFICATIONS_DISPATCH) + return capability if isinstance(capability, NotificationDispatchProvider) else None diff --git a/src/govoplan_core/core/registry.py b/src/govoplan_core/core/registry.py index 026a2a6..297086b 100644 --- a/src/govoplan_core/core/registry.py +++ b/src/govoplan_core/core/registry.py @@ -186,46 +186,20 @@ class PlatformRegistry: def validate(self) -> RegistrySnapshot: ordered = tuple(self._topologically_sorted()) available_capabilities = set(self._capability_factories) - seen_permissions: dict[str, PermissionDefinition] = {} for manifest in ordered: _validate_manifest_shape(manifest) - for dependency in manifest.dependencies: - if dependency not in self._manifests: - raise RegistryError(f"Module {manifest.id!r} depends on unknown module {dependency!r}") - for capability in manifest.required_capabilities: - if capability not in available_capabilities: - raise RegistryError(f"Module {manifest.id!r} requires unavailable capability {capability!r}") - for dependency in manifest.optional_dependencies: - if dependency == manifest.id: - raise RegistryError(f"Module {manifest.id!r} cannot list itself as an optional dependency") - for permission in manifest.permissions: - if permission.module_id != manifest.id: - raise RegistryError(f"Permission {permission.scope!r} has mismatched module id {permission.module_id!r}") - if not _SCOPE_RE.match(permission.scope): - raise RegistryError(f"Permission scope must be ::: {permission.scope!r}") - expected_prefix = f"{permission.module_id}:{permission.resource}:" - if not permission.scope.startswith(expected_prefix) or permission.scope.rsplit(":", 1)[-1] != permission.action: - raise RegistryError(f"Permission fields do not match scope {permission.scope!r}") - if permission.scope in seen_permissions: - raise RegistryError(f"Duplicate permission scope: {permission.scope}") - seen_permissions[permission.scope] = permission - + _validate_manifest_relationships( + manifest, + known_modules=self._manifests, + available_capabilities=available_capabilities, + ) + permissions = _collect_manifest_permissions(ordered) _validate_interface_closure(ordered) - - known_scopes = set(seen_permissions) - for manifest in ordered: - for template in manifest.role_templates: - for scope in template.permissions: - if scope in {"*", "tenant:*", "system:*"}: - continue - if _WILDCARD_RE.match(scope): - continue - if scope not in known_scopes: - raise RegistryError(f"Role template {template.slug!r} references unknown permission {scope!r}") + _validate_role_template_scopes(ordered, known_scopes=set(permissions)) return RegistrySnapshot( manifests=ordered, - permissions=tuple(seen_permissions.values()), + permissions=tuple(permissions.values()), role_templates=tuple(template for manifest in ordered for template in manifest.role_templates), nav_items=self.nav_items(), ) @@ -299,7 +273,80 @@ def _attribute_delete_veto_issue( return replace(issue, module_id=issue.module_id or registration.module_id, details=details) +def _validate_manifest_relationships( + manifest: ModuleManifest, + *, + known_modules: Mapping[str, ModuleManifest], + available_capabilities: set[str], +) -> None: + for dependency in manifest.dependencies: + if dependency not in known_modules: + raise RegistryError(f"Module {manifest.id!r} depends on unknown module {dependency!r}") + for capability in manifest.required_capabilities: + if capability not in available_capabilities: + raise RegistryError(f"Module {manifest.id!r} requires unavailable capability {capability!r}") + for dependency in manifest.optional_dependencies: + if dependency == manifest.id: + raise RegistryError(f"Module {manifest.id!r} cannot list itself as an optional dependency") + + +def _collect_manifest_permissions(manifests: tuple[ModuleManifest, ...]) -> dict[str, PermissionDefinition]: + permissions: dict[str, PermissionDefinition] = {} + for manifest in manifests: + for permission in manifest.permissions: + _validate_manifest_permission(manifest, permission, permissions) + permissions[permission.scope] = permission + return permissions + + +def _validate_manifest_permission( + manifest: ModuleManifest, + permission: PermissionDefinition, + seen_permissions: Mapping[str, PermissionDefinition], +) -> None: + if permission.module_id != manifest.id: + raise RegistryError(f"Permission {permission.scope!r} has mismatched module id {permission.module_id!r}") + if not _SCOPE_RE.match(permission.scope): + raise RegistryError(f"Permission scope must be ::: {permission.scope!r}") + expected_prefix = f"{permission.module_id}:{permission.resource}:" + if not permission.scope.startswith(expected_prefix) or permission.scope.rsplit(":", 1)[-1] != permission.action: + raise RegistryError(f"Permission fields do not match scope {permission.scope!r}") + if permission.scope in seen_permissions: + raise RegistryError(f"Duplicate permission scope: {permission.scope}") + + +def _validate_role_template_scopes( + manifests: tuple[ModuleManifest, ...], + *, + known_scopes: set[str], +) -> None: + for manifest in manifests: + for template in manifest.role_templates: + for scope in template.permissions: + if _role_template_scope_known(scope, known_scopes): + continue + raise RegistryError(f"Role template {template.slug!r} references unknown permission {scope!r}") + + +def _role_template_scope_known(scope: str, known_scopes: set[str]) -> bool: + if scope in {"*", "tenant:*", "system:*"}: + return True + if _WILDCARD_RE.match(scope): + return True + return scope in known_scopes + + def _validate_manifest_shape(manifest: ModuleManifest) -> None: + _validate_manifest_identity(manifest) + _validate_manifest_contract_lists(manifest) + _validate_manifest_overlaps(manifest) + _validate_manifest_migration_spec(manifest) + _validate_manifest_frontend(manifest) + for item in manifest.nav_items: + _validate_nav_item(manifest.id, item) + + +def _validate_manifest_identity(manifest: ModuleManifest) -> None: if not _MODULE_ID_RE.match(manifest.id): raise RegistryError(f"Module manifest id must match {_MODULE_ID_RE.pattern}: {manifest.id!r}") if not manifest.name.strip(): @@ -313,12 +360,17 @@ def _validate_manifest_shape(manifest: ModuleManifest) -> None: f"{SUPPORTED_MANIFEST_CONTRACT_VERSION!r}" ) + +def _validate_manifest_contract_lists(manifest: ModuleManifest) -> None: _validate_dependency_list(manifest.id, "dependencies", manifest.dependencies) _validate_dependency_list(manifest.id, "optional_dependencies", manifest.optional_dependencies) _validate_capability_list(manifest.id, "required_capabilities", manifest.required_capabilities) _validate_capability_list(manifest.id, "optional_capabilities", manifest.optional_capabilities) _validate_interface_providers(manifest.id, manifest.provides_interfaces) _validate_interface_requirements(manifest.id, manifest.requires_interfaces) + + +def _validate_manifest_overlaps(manifest: ModuleManifest) -> None: overlap = set(manifest.dependencies) & set(manifest.optional_dependencies) if overlap: joined = ", ".join(sorted(overlap)) @@ -328,36 +380,42 @@ def _validate_manifest_shape(manifest: ModuleManifest) -> None: joined = ", ".join(sorted(capability_overlap)) raise RegistryError(f"Module {manifest.id!r} lists capabilities as both required and optional: {joined}") + +def _validate_manifest_migration_spec(manifest: ModuleManifest) -> None: if manifest.migration_spec is not None: if manifest.migration_spec.module_id != manifest.id: raise RegistryError(f"Module {manifest.id!r} has migration spec for {manifest.migration_spec.module_id!r}") if manifest.migration_spec.metadata is None and not manifest.migration_spec.script_location: raise RegistryError(f"Module {manifest.id!r} migration spec must declare metadata or script location") - if manifest.frontend is not None: - frontend = manifest.frontend - if frontend.module_id != manifest.id: - raise RegistryError(f"Module {manifest.id!r} has frontend metadata for {frontend.module_id!r}") - if frontend.asset_manifest_contract_version != SUPPORTED_FRONTEND_ASSET_MANIFEST_CONTRACT_VERSION: - raise RegistryError( - f"Module {manifest.id!r} uses unsupported frontend asset manifest contract version " - f"{frontend.asset_manifest_contract_version!r}; supported version is " - f"{SUPPORTED_FRONTEND_ASSET_MANIFEST_CONTRACT_VERSION!r}" - ) - if frontend.package_name is not None and not _NPM_PACKAGE_RE.match(frontend.package_name): - raise RegistryError(f"Module {manifest.id!r} has invalid frontend package name {frontend.package_name!r}") - for route in (*frontend.routes, *frontend.settings_routes): - if not route.path.startswith("/"): - raise RegistryError(f"Frontend route for module {manifest.id!r} must start with '/': {route.path!r}") - if not route.component.strip(): - raise RegistryError(f"Frontend route {route.path!r} for module {manifest.id!r} must declare a component") - for item in frontend.nav_items: - _validate_nav_item(manifest.id, item) - for item in manifest.nav_items: +def _validate_manifest_frontend(manifest: ModuleManifest) -> None: + if manifest.frontend is None: + return + frontend = manifest.frontend + if frontend.module_id != manifest.id: + raise RegistryError(f"Module {manifest.id!r} has frontend metadata for {frontend.module_id!r}") + if frontend.asset_manifest_contract_version != SUPPORTED_FRONTEND_ASSET_MANIFEST_CONTRACT_VERSION: + raise RegistryError( + f"Module {manifest.id!r} uses unsupported frontend asset manifest contract version " + f"{frontend.asset_manifest_contract_version!r}; supported version is " + f"{SUPPORTED_FRONTEND_ASSET_MANIFEST_CONTRACT_VERSION!r}" + ) + if frontend.package_name is not None and not _NPM_PACKAGE_RE.match(frontend.package_name): + raise RegistryError(f"Module {manifest.id!r} has invalid frontend package name {frontend.package_name!r}") + for route in (*frontend.routes, *frontend.settings_routes): + _validate_frontend_route(manifest.id, route.path, route.component) + for item in frontend.nav_items: _validate_nav_item(manifest.id, item) +def _validate_frontend_route(module_id: str, path: str, component: str) -> None: + if not path.startswith("/"): + raise RegistryError(f"Frontend route for module {module_id!r} must start with '/': {path!r}") + if not component.strip(): + raise RegistryError(f"Frontend route {path!r} for module {module_id!r} must declare a component") + + def _validate_interface_closure(manifests: tuple[ModuleManifest, ...]) -> None: providers: dict[str, list[tuple[ModuleManifest, ModuleInterfaceProvider]]] = defaultdict(list) for manifest in manifests: diff --git a/src/govoplan_core/core/runtime.py b/src/govoplan_core/core/runtime.py index 17708d4..9c1a44a 100644 --- a/src/govoplan_core/core/runtime.py +++ b/src/govoplan_core/core/runtime.py @@ -1,5 +1,7 @@ from __future__ import annotations +from typing import Any + from govoplan_core.core.modules import ModuleContext _context: ModuleContext | None = None @@ -21,3 +23,41 @@ def get_runtime_context() -> ModuleContext | None: def get_registry() -> object | None: return _context.registry if _context is not None else None + + +class ModuleSettingsProxy: + def __init__(self, runtime: ModuleRuntimeState) -> None: + self._runtime = runtime + + def __getattr__(self, name: str) -> Any: + return getattr(self._runtime.get_settings(), name) + + +class ModuleRuntimeState: + def __init__(self, module_name: str) -> None: + self.module_name = module_name + self.settings = ModuleSettingsProxy(self) + self._registry: object | None = None + self._settings: object | None = None + + def configure_runtime(self, *, registry: object | None = None, settings: object | None = None) -> None: + if registry is not None: + self._registry = registry + if settings is not None: + self._settings = settings + + def clear_runtime(self) -> None: + self._registry = None + self._settings = None + + def get_registry(self) -> object | None: + return self._registry + + def get_settings(self) -> object: + if self._settings is not None: + return self._settings + try: + from govoplan_core.settings import settings as legacy_settings + except ModuleNotFoundError as exc: + raise RuntimeError(f"GovOPlaN {self.module_name} runtime settings are not configured") from exc + return legacy_settings diff --git a/src/govoplan_core/core/sqlalchemy_change_tracking.py b/src/govoplan_core/core/sqlalchemy_change_tracking.py new file mode 100644 index 0000000..3f9dfd7 --- /dev/null +++ b/src/govoplan_core/core/sqlalchemy_change_tracking.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +from collections.abc import Callable +from typing import Any, Literal + +from sqlalchemy import inspect + +ChangeOperation = Literal["created", "updated", "deleted"] + + +def object_state(obj: object) -> Any: + return inspect(obj) + + +def has_attr_changes(state: Any, attrs: tuple[str, ...]) -> bool: + return any(name in state.attrs and state.attrs[name].history.has_changes() for name in attrs) + + +def previous_value(obj: object, attr_name: str) -> str | None: + state = object_state(obj) + if attr_name not in state.attrs: + return None + history = state.attrs[attr_name].history + if not history.has_changes() or not history.deleted: + return None + value = history.deleted[0] + return str(value) if value is not None else None + + +def ensure_object_id(obj: object, new_id: Callable[[], str]) -> str: + resource_id = getattr(obj, "id", None) + if resource_id: + return str(resource_id) + resource_id = new_id() + setattr(obj, "id", resource_id) + return resource_id + + +def operation_for_object(obj: object, *, changed_attrs: tuple[str, ...]) -> ChangeOperation | None: + state = object_state(obj) + if state.pending: + return "created" + if state.deleted: + return "deleted" + if not has_attr_changes(state, changed_attrs): + return None + return "updated" + + +def operation_for_soft_deletable( + obj: object, + *, + changed_attrs: tuple[str, ...], + deleted_attr: str = "deleted_at", +) -> ChangeOperation | None: + state = object_state(obj) + if state.pending: + return "created" + if not has_attr_changes(state, changed_attrs): + return None + if deleted_attr in state.attrs: + history = state.attrs[deleted_attr].history + if history.has_changes(): + if any(value is not None for value in history.added): + return "deleted" + if any(value is not None for value in history.deleted): + return "created" + return "updated" diff --git a/src/govoplan_core/db/migrations.py b/src/govoplan_core/db/migrations.py index 503a413..bb45709 100644 --- a/src/govoplan_core/db/migrations.py +++ b/src/govoplan_core/db/migrations.py @@ -3,6 +3,7 @@ from __future__ import annotations from collections.abc import Mapping from dataclasses import dataclass, replace import json +import logging import os from pathlib import Path import re @@ -25,6 +26,8 @@ from govoplan_core.server.config import ManifestFactory from govoplan_core.server.registry import available_module_manifests, build_platform_registry from govoplan_core.settings import settings +logger = logging.getLogger(__name__) + # Historic development databases could be created partly through Alembic and # partly through Base.metadata.create_all(). In that state Alembic still says # "2c..." while the 3d/4e file-storage tables already exist, so a normal @@ -166,6 +169,15 @@ _CREATE_ALL_THROUGH_HIERARCHICAL_INDEXES = { } +@dataclass(frozen=True, slots=True) +class _LegacyCreateAllSchemaState: + current: str | None + has_no_revision: bool + has_file_storage: bool + has_file_folders: bool + has_create_all_hierarchical_schema: bool + + @dataclass(frozen=True, slots=True) class MigrationResult: previous_revision: str | None @@ -233,10 +245,7 @@ def run_registered_module_migration_tasks( dry_run: bool = False, manifest_factories: tuple[ManifestFactory, ...] = (), ) -> tuple[dict[str, object], ...]: - active_phases = tuple(dict.fromkeys(str(item).strip() for item in phases if str(item).strip())) - invalid_phases = tuple(phase for phase in active_phases if phase not in MIGRATION_TASK_PHASES) - if invalid_phases: - raise ValueError("Unsupported module migration task phase(s): " + ", ".join(invalid_phases)) + active_phases = _normalized_migration_task_phases(phases) url = database_url or settings.database_url registry = _registered_module_registry( database_url=url, @@ -244,91 +253,192 @@ def run_registered_module_migration_tasks( manifest_factories=manifest_factories, ) manifests = {manifest.id: manifest for manifest in registry.manifests()} - ordered_ids = tuple(dict.fromkeys([ - *(str(item).strip() for item in (migration_module_order or ()) if str(item).strip()), - *manifests.keys(), - ])) + ordered_ids = _ordered_migration_task_module_ids(migration_module_order, manifests) records: list[dict[str, object]] = [] database = get_database() with database.SessionLocal() as session: for phase in active_phases: - for module_id in ordered_ids: - manifest = manifests.get(module_id) - if manifest is None or manifest.migration_spec is None: - continue - for task in manifest.migration_spec.migration_tasks: - if task.phase != phase: - continue - record: dict[str, object] = { - "module_id": manifest.id, - "task_id": task.task_id, - "phase": task.phase, - "summary": task.summary, - "task_version": task.task_version, - "safety": task.safety, - "idempotent": task.idempotent, - "dry_run": dry_run, - } - if task.timeout_seconds is not None: - record["timeout_seconds"] = task.timeout_seconds - if not task.idempotent: - record.update({"status": "blocked", "message": "Task is not idempotent."}) - records.append(record) - raise ModuleMigrationTaskExecutionError( - f"Module migration task {manifest.id}/{task.task_id} is not idempotent.", - records=tuple(records), - ) - if dry_run: - record.update({"status": "skipped", "message": "Dry run; executor was not called."}) - records.append(record) - continue - if task.executor is None: - record.update({"status": "blocked", "message": "Task has no executor."}) - records.append(record) - raise ModuleMigrationTaskExecutionError( - f"Module migration task {manifest.id}/{task.task_id} has no executor.", - records=tuple(records), - ) - context = ModuleMigrationTaskContext( - module_id=manifest.id, - task_id=task.task_id, - phase=task.phase, - database_url=url, - target_version=manifest.version, - session=session, - dry_run=dry_run, - metadata=task.metadata, - ) - try: - result = task.executor(context) - normalized = _normalize_migration_task_result(result) - except Exception as exc: - session.rollback() - record.update({ - "status": "blocked", - "message": f"{type(exc).__name__}: {exc}", - }) - records.append(record) - raise ModuleMigrationTaskExecutionError( - f"Module migration task {manifest.id}/{task.task_id} failed.", - records=tuple(records), - ) from exc - record.update({ - "status": normalized.status, - "message": normalized.message, - "details": _jsonable_migration_task_details(normalized.details), - }) - records.append(record) - if normalized.status == "blocked": - session.rollback() - raise ModuleMigrationTaskExecutionError( - normalized.message or f"Module migration task {manifest.id}/{task.task_id} blocked migration.", - records=tuple(records), - ) - session.commit() + for manifest, task in _iter_phase_migration_tasks(phase, ordered_ids=ordered_ids, manifests=manifests): + _run_registered_module_migration_task( + session, + manifest=manifest, + task=task, + database_url=url, + dry_run=dry_run, + records=records, + ) return tuple(records) +def _normalized_migration_task_phases(phases: tuple[str, ...] | list[str]) -> tuple[str, ...]: + active_phases = tuple(dict.fromkeys(str(item).strip() for item in phases if str(item).strip())) + invalid_phases = tuple(phase for phase in active_phases if phase not in MIGRATION_TASK_PHASES) + if invalid_phases: + raise ValueError("Unsupported module migration task phase(s): " + ", ".join(invalid_phases)) + return active_phases + + +def _ordered_migration_task_module_ids( + migration_module_order: tuple[str, ...] | list[str] | None, + manifests: Mapping[str, object], +) -> tuple[str, ...]: + return tuple(dict.fromkeys([ + *(str(item).strip() for item in (migration_module_order or ()) if str(item).strip()), + *manifests.keys(), + ])) + + +def _iter_phase_migration_tasks( + phase: str, + *, + ordered_ids: tuple[str, ...], + manifests: Mapping[str, object], +) -> Iterable[tuple[object, object]]: + for module_id in ordered_ids: + manifest = manifests.get(module_id) + migration_spec = getattr(manifest, "migration_spec", None) + if manifest is None or migration_spec is None: + continue + for task in migration_spec.migration_tasks: + if task.phase == phase: + yield manifest, task + + +def _run_registered_module_migration_task( + session: object, + *, + manifest: object, + task: object, + database_url: str, + dry_run: bool, + records: list[dict[str, object]], +) -> None: + record = _migration_task_record(manifest, task, dry_run=dry_run) + _validate_migration_task_can_run(manifest, task, record, records, dry_run=dry_run) + if dry_run: + record.update({"status": "skipped", "message": "Dry run; executor was not called."}) + records.append(record) + return + normalized = _execute_module_migration_task( + session, + manifest=manifest, + task=task, + record=record, + records=records, + database_url=database_url, + dry_run=dry_run, + ) + record.update({ + "status": normalized.status, + "message": normalized.message, + "details": _jsonable_migration_task_details(normalized.details), + }) + records.append(record) + if normalized.status == "blocked": + session.rollback() + raise ModuleMigrationTaskExecutionError( + normalized.message or f"Module migration task {manifest.id}/{task.task_id} blocked migration.", + records=tuple(records), + ) + session.commit() + + +def _migration_task_record(manifest: object, task: object, *, dry_run: bool) -> dict[str, object]: + record: dict[str, object] = { + "module_id": manifest.id, + "task_id": task.task_id, + "phase": task.phase, + "summary": task.summary, + "task_version": task.task_version, + "safety": task.safety, + "idempotent": task.idempotent, + "dry_run": dry_run, + } + if task.timeout_seconds is not None: + record["timeout_seconds"] = task.timeout_seconds + return record + + +def _validate_migration_task_can_run( + manifest: object, + task: object, + record: dict[str, object], + records: list[dict[str, object]], + *, + dry_run: bool, +) -> None: + if not task.idempotent: + _block_migration_task( + manifest, + task, + record, + records, + message="Task is not idempotent.", + error=f"Module migration task {manifest.id}/{task.task_id} is not idempotent.", + ) + if not dry_run and task.executor is None: + _block_migration_task( + manifest, + task, + record, + records, + message="Task has no executor.", + error=f"Module migration task {manifest.id}/{task.task_id} has no executor.", + ) + + +def _block_migration_task( + manifest: object, + task: object, + record: dict[str, object], + records: list[dict[str, object]], + *, + message: str, + error: str, +) -> None: + record.update({"status": "blocked", "message": message}) + records.append(record) + raise ModuleMigrationTaskExecutionError( + error, + records=tuple(records), + ) + + +def _execute_module_migration_task( + session: object, + *, + manifest: object, + task: object, + record: dict[str, object], + records: list[dict[str, object]], + database_url: str, + dry_run: bool, +) -> ModuleMigrationTaskResult: + context = ModuleMigrationTaskContext( + module_id=manifest.id, + task_id=task.task_id, + phase=task.phase, + database_url=database_url, + target_version=manifest.version, + session=session, + dry_run=dry_run, + metadata=task.metadata, + ) + try: + return _normalize_migration_task_result(task.executor(context)) + except Exception as exc: + session.rollback() + record.update({ + "status": "blocked", + "message": f"{type(exc).__name__}: {exc}", + }) + records.append(record) + raise ModuleMigrationTaskExecutionError( + f"Module migration task {manifest.id}/{task.task_id} failed.", + records=tuple(records), + ) from exc + + def _normalize_migration_task_result(result: ModuleMigrationTaskResult | None) -> ModuleMigrationTaskResult: if result is None: return ModuleMigrationTaskResult() @@ -524,18 +634,21 @@ def _backfill_user_lock_state_for_create_all_schema(database_url: str) -> None: def _row_count(connection, table_name: str) -> int: quoted = _quoted_table_name(connection, table_name) - return int(connection.execute(text(f"SELECT COUNT(*) FROM {quoted}")).scalar_one()) # nosemgrep: python.sqlalchemy.security.audit.avoid-sqlalchemy-text.avoid-sqlalchemy-text + statement = text(f"SELECT COUNT(*) FROM {quoted}") # nosec B608 # nosemgrep: python.sqlalchemy.security.audit.avoid-sqlalchemy-text.avoid-sqlalchemy-text + return int(connection.execute(statement).scalar_one()) def _drop_table(connection, table_name: str) -> None: quoted = _quoted_table_name(connection, table_name) - connection.execute(text(f"DROP TABLE {quoted}")) # nosemgrep: python.sqlalchemy.security.audit.avoid-sqlalchemy-text.avoid-sqlalchemy-text + statement = text(f"DROP TABLE {quoted}") # nosec B608 # nosemgrep: python.sqlalchemy.security.audit.avoid-sqlalchemy-text.avoid-sqlalchemy-text + connection.execute(statement) def _rename_table(connection, old_name: str, new_name: str) -> None: quoted_old = _quoted_table_name(connection, old_name) quoted_new = _quoted_table_name(connection, new_name) - connection.execute(text(f"ALTER TABLE {quoted_old} RENAME TO {quoted_new}")) # nosemgrep: python.sqlalchemy.security.audit.avoid-sqlalchemy-text.avoid-sqlalchemy-text + statement = text(f"ALTER TABLE {quoted_old} RENAME TO {quoted_new}") # nosec B608 # nosemgrep: python.sqlalchemy.security.audit.avoid-sqlalchemy-text.avoid-sqlalchemy-text + connection.execute(statement) def _quoted_table_name(connection, table_name: str) -> str: @@ -715,7 +828,8 @@ def reconcile_covered_alembic_dependency_heads( for revision_id in current: try: revision = script.get_revision(revision_id) - except Exception: + except Exception as exc: + logger.debug("Skipping Alembic revision %s while pruning dependency heads: %s", revision_id, exc, exc_info=True) continue ancestors = { item.revision @@ -749,58 +863,68 @@ def reconcile_legacy_create_all_schema( """ url = database_url or settings.database_url - engine = create_engine(url) - try: - with engine.connect() as connection: - heads = MigrationContext.configure(connection).get_current_heads() - current = heads[0] if len(heads) == 1 else None - has_no_revision = len(heads) == 0 - schema = inspect(connection) - tables = set(schema.get_table_names()) - - has_file_storage = _FILE_STORAGE_TABLES.issubset(tables) and all( - _has_columns(schema, table, _FILE_STORAGE_COLUMNS[table]) - for table in _FILE_STORAGE_TABLES - ) - has_file_folders = _FILE_FOLDER_TABLES.issubset(tables) and _has_columns( - schema, - "file_folders", - _FILE_STORAGE_COLUMNS["file_folders"], - ) - has_create_all_hierarchical_schema = _has_create_all_schema_through_hierarchical_settings(schema, tables) - finally: - engine.dispose() - - target: str | None = None - if current == REVISION_AUTH_RBAC and has_file_storage and has_file_folders: - target = REVISION_FILE_FOLDERS - elif current == REVISION_AUTH_RBAC and has_file_storage: - target = REVISION_FILE_STORAGE - elif current == REVISION_FILE_STORAGE and has_file_folders: - target = REVISION_FILE_FOLDERS - elif current == REVISION_FILE_FOLDERS and has_create_all_hierarchical_schema: - # Development DBs may be stamped at 4e after earlier create_all - # reconciliation even though later tables/columns were already created - # by a newer model set. Skip only when that complete known schema is - # present, then let newer migrations such as mail credential usernames - # run normally. - _backfill_user_lock_state_for_create_all_schema(url) - target = REVISION_HIERARCHICAL_SETTINGS - elif has_no_revision and has_create_all_hierarchical_schema: - _backfill_user_lock_state_for_create_all_schema(url) - target = REVISION_HIERARCHICAL_SETTINGS - elif has_no_revision and has_file_storage and has_file_folders: - # This is the other create_all-only development shape. The strict - # column checks above ensure that we only stamp a complete known schema. - target = REVISION_FILE_FOLDERS - + state = _legacy_create_all_schema_state(url) + target = _legacy_create_all_reconciliation_target(state) if target is None: return None + if target == REVISION_HIERARCHICAL_SETTINGS: + _backfill_user_lock_state_for_create_all_schema(url) command.stamp(alembic_config(database_url=url, migration_track=migration_track), target) return target +def _legacy_create_all_schema_state(database_url: str) -> _LegacyCreateAllSchemaState: + engine = create_engine(database_url) + try: + with engine.connect() as connection: + heads = MigrationContext.configure(connection).get_current_heads() + schema = inspect(connection) + tables = set(schema.get_table_names()) + return _legacy_create_all_schema_state_from_inspection(heads, schema, tables) + finally: + engine.dispose() + + +def _legacy_create_all_schema_state_from_inspection( + heads: tuple[str, ...], + schema: object, + tables: set[str], +) -> _LegacyCreateAllSchemaState: + has_file_storage = _FILE_STORAGE_TABLES.issubset(tables) and all( + _has_columns(schema, table, _FILE_STORAGE_COLUMNS[table]) + for table in _FILE_STORAGE_TABLES + ) + has_file_folders = _FILE_FOLDER_TABLES.issubset(tables) and _has_columns( + schema, + "file_folders", + _FILE_STORAGE_COLUMNS["file_folders"], + ) + return _LegacyCreateAllSchemaState( + current=heads[0] if len(heads) == 1 else None, + has_no_revision=len(heads) == 0, + has_file_storage=has_file_storage, + has_file_folders=has_file_folders, + has_create_all_hierarchical_schema=_has_create_all_schema_through_hierarchical_settings(schema, tables), + ) + + +def _legacy_create_all_reconciliation_target(state: _LegacyCreateAllSchemaState) -> str | None: + if state.current == REVISION_AUTH_RBAC and state.has_file_storage and state.has_file_folders: + return REVISION_FILE_FOLDERS + if state.current == REVISION_AUTH_RBAC and state.has_file_storage: + return REVISION_FILE_STORAGE + if state.current == REVISION_FILE_STORAGE and state.has_file_folders: + return REVISION_FILE_FOLDERS + if state.current == REVISION_FILE_FOLDERS and state.has_create_all_hierarchical_schema: + return REVISION_HIERARCHICAL_SETTINGS + if state.has_no_revision and state.has_create_all_hierarchical_schema: + return REVISION_HIERARCHICAL_SETTINGS + if state.has_no_revision and state.has_file_storage and state.has_file_folders: + return REVISION_FILE_FOLDERS + return None + + def migrate_database( *, database_url: str | None = None, diff --git a/src/govoplan_core/db/query_metrics.py b/src/govoplan_core/db/query_metrics.py new file mode 100644 index 0000000..3b8df2c --- /dev/null +++ b/src/govoplan_core/db/query_metrics.py @@ -0,0 +1,100 @@ +from __future__ import annotations + +from contextlib import contextmanager +from contextvars import ContextVar +from dataclasses import dataclass +import time +import weakref +from collections.abc import Iterator + +from sqlalchemy import event +from sqlalchemy.engine import Connection, Engine +from sqlalchemy.engine.interfaces import ExecutionContext + + +@dataclass(slots=True) +class QueryMetrics: + query_count: int = 0 + total_ms: float = 0.0 + slowest_ms: float = 0.0 + error_count: int = 0 + + def record(self, elapsed_ms: float, *, failed: bool = False) -> None: + self.query_count += 1 + self.total_ms += elapsed_ms + self.slowest_ms = max(self.slowest_ms, elapsed_ms) + if failed: + self.error_count += 1 + + +_current_metrics: ContextVar[QueryMetrics | None] = ContextVar("govoplan_query_metrics", default=None) +_instrumented_engines: weakref.WeakSet[Engine] = weakref.WeakSet() +_START_STACK_KEY = "govoplan_query_metric_starts" + + +@contextmanager +def collect_query_metrics() -> Iterator[QueryMetrics]: + metrics = QueryMetrics() + token = _current_metrics.set(metrics) + try: + yield metrics + finally: + _current_metrics.reset(token) + + +def current_query_metrics() -> QueryMetrics | None: + return _current_metrics.get() + + +def instrument_engine(engine: Engine) -> Engine: + if engine in _instrumented_engines: + return engine + + event.listen(engine, "before_cursor_execute", _before_cursor_execute) + event.listen(engine, "after_cursor_execute", _after_cursor_execute) + event.listen(engine, "handle_error", _handle_error) + _instrumented_engines.add(engine) + return engine + + +def _before_cursor_execute( + conn: Connection, + _cursor: object, + _statement: str, + _parameters: object, + _context: ExecutionContext, + _executemany: bool, +) -> None: + if current_query_metrics() is None: + return + starts = conn.info.setdefault(_START_STACK_KEY, []) + starts.append(time.perf_counter()) + + +def _after_cursor_execute( + conn: Connection, + _cursor: object, + _statement: str, + _parameters: object, + _context: ExecutionContext, + _executemany: bool, +) -> None: + _record_elapsed(conn, failed=False) + + +def _handle_error(exception_context: object) -> None: + conn = getattr(exception_context, "connection", None) + if isinstance(conn, Connection): + _record_elapsed(conn, failed=True) + + +def _record_elapsed(conn: Connection, *, failed: bool) -> None: + metrics = current_query_metrics() + if metrics is None: + return + starts = conn.info.get(_START_STACK_KEY) + if not starts: + return + started_at = starts.pop() + elapsed_ms = (time.perf_counter() - started_at) * 1000 + metrics.record(elapsed_ms, failed=failed) diff --git a/src/govoplan_core/db/session.py b/src/govoplan_core/db/session.py index 5912824..07e312c 100644 --- a/src/govoplan_core/db/session.py +++ b/src/govoplan_core/db/session.py @@ -7,6 +7,8 @@ from sqlalchemy import create_engine from sqlalchemy.engine import Engine from sqlalchemy.orm import Session, sessionmaker +from govoplan_core.db.query_metrics import instrument_engine + def default_connect_args(database_url: str) -> dict[str, Any]: return {"check_same_thread": False} if database_url.startswith("sqlite") else {} @@ -22,13 +24,13 @@ def create_database_engine( merged_connect_args = dict(default_connect_args(database_url)) if connect_args: merged_connect_args.update(connect_args) - return create_engine(database_url, pool_pre_ping=pool_pre_ping, connect_args=merged_connect_args, **kwargs) + return instrument_engine(create_engine(database_url, pool_pre_ping=pool_pre_ping, connect_args=merged_connect_args, **kwargs)) class DatabaseHandle: def __init__(self, database_url: str, *, engine: Engine | None = None) -> None: self.database_url = database_url - self.engine = engine or create_database_engine(database_url) + self.engine = instrument_engine(engine) if engine is not None else create_database_engine(database_url) self.SessionLocal = sessionmaker(bind=self.engine, autoflush=False, autocommit=False, expire_on_commit=False) def session(self) -> Session: diff --git a/src/govoplan_core/mail/config.py b/src/govoplan_core/mail/config.py index 5110403..12fb67e 100644 --- a/src/govoplan_core/mail/config.py +++ b/src/govoplan_core/mail/config.py @@ -67,6 +67,32 @@ class ImapConfig(ImapServerConfig): password: str | None = None +def normalize_split_transport_credentials(value: object) -> object: + """Move legacy transport username/password fields into credentials.""" + if not isinstance(value, dict): + return value + data = dict(value) + credentials = data.get("credentials") if isinstance(data.get("credentials"), dict) else {} + credentials = {key: dict(item) for key, item in credentials.items() if isinstance(item, dict)} + for protocol in ("smtp", "imap"): + transport = data.get(protocol) + if not isinstance(transport, dict): + continue + next_transport = dict(transport) + next_credentials = dict(credentials.get(protocol) or {}) + for field in ("username", "password"): + if field in next_transport and field not in next_credentials: + next_credentials[field] = next_transport[field] + next_transport.pop(field, None) + next_transport.pop("enabled", None) + data[protocol] = next_transport + if next_credentials: + credentials[protocol] = next_credentials + if credentials: + data["credentials"] = credentials + return data + + def default_smtp_port(security: TransportSecurity | str | None) -> int: if security == TransportSecurity.TLS or security == "tls": return 465 diff --git a/src/govoplan_core/privacy/retention.py b/src/govoplan_core/privacy/retention.py index efcd62b..d67892d 100644 --- a/src/govoplan_core/privacy/retention.py +++ b/src/govoplan_core/privacy/retention.py @@ -1,5 +1,3 @@ -from __future__ import annotations - """Compatibility facade for privacy retention policy. Policy-owned behavior is provided by ``govoplan-policy`` through the @@ -7,27 +5,22 @@ Policy-owned behavior is provided by ``govoplan-policy`` through the without importing policy implementation code from core. """ +from __future__ import annotations + from dataclasses import dataclass from typing import Any, Mapping from govoplan_core.core.policy import CAPABILITY_POLICY_PRIVACY_RETENTION, PrivacyRetentionService from govoplan_core.core.runtime import get_registry +from govoplan_core.privacy.schemas import ( + RETENTION_DAY_KEYS as RETENTION_DAY_KEYS, + RETENTION_POLICY_FIELD_KEYS, + default_allow_lower_level_limits as default_allow_lower_level_limits, + normalize_allow_lower_level_limits, +) PRIVACY_POLICY_SETTINGS_KEY = "privacy_retention_policy" -RETENTION_DAY_KEYS = ( - "raw_campaign_json_retention_days", - "generated_eml_retention_days", - "stored_report_detail_retention_days", - "mock_mailbox_retention_days", - "audit_detail_retention_days", -) -RETENTION_POLICY_FIELD_KEYS = ( - "store_raw_campaign_json", - *RETENTION_DAY_KEYS, - "audit_detail_level", -) - _DEFAULT_POLICY = { "store_raw_campaign_json": True, "raw_campaign_json_retention_days": None, @@ -122,28 +115,10 @@ def _policy_data(settings_payload: Mapping[str, Any] | None) -> dict[str, Any]: for key in RETENTION_POLICY_FIELD_KEYS: if key in raw: payload[key] = raw[key] - payload["allow_lower_level_limits"] = _normalize_allow_lower_level_limits(raw.get("allow_lower_level_limits"), fill_defaults=True) + payload["allow_lower_level_limits"] = normalize_allow_lower_level_limits(raw.get("allow_lower_level_limits"), fill_defaults=True) return payload -def _normalize_allow_lower_level_limits(value: Any, *, fill_defaults: bool) -> dict[str, bool] | None: - if value in (None, ""): - return default_allow_lower_level_limits() if fill_defaults else None - if not isinstance(value, Mapping): - raise ValueError("allow_lower_level_limits must be an object") - normalized = default_allow_lower_level_limits() if fill_defaults else {} - for key, allowed in value.items(): - clean_key = str(key) - if clean_key not in RETENTION_POLICY_FIELD_KEYS: - raise ValueError(f"Unknown retention policy field: {clean_key}") - normalized[clean_key] = bool(allowed) - return normalized - - -def default_allow_lower_level_limits() -> dict[str, bool]: - return {key: True for key in RETENTION_POLICY_FIELD_KEYS} - - def privacy_policy_from_settings(item: object) -> Any: service = _runtime_service() if service is not None: diff --git a/src/govoplan_core/privacy/schemas.py b/src/govoplan_core/privacy/schemas.py new file mode 100644 index 0000000..345ad77 --- /dev/null +++ b/src/govoplan_core/privacy/schemas.py @@ -0,0 +1,83 @@ +from __future__ import annotations + +from typing import Any, Literal + +from pydantic import BaseModel, ConfigDict, Field, field_validator + + +RETENTION_DAY_KEYS = ( + "raw_campaign_json_retention_days", + "generated_eml_retention_days", + "stored_report_detail_retention_days", + "mock_mailbox_retention_days", + "audit_detail_retention_days", +) +RETENTION_POLICY_FIELD_KEYS = ( + "store_raw_campaign_json", + *RETENTION_DAY_KEYS, + "audit_detail_level", +) + + +def default_allow_lower_level_limits() -> dict[str, bool]: + return {key: True for key in RETENTION_POLICY_FIELD_KEYS} + + +def normalize_allow_lower_level_limits(value: Any, *, fill_defaults: bool) -> dict[str, bool] | None: + if value in (None, ""): + return default_allow_lower_level_limits() if fill_defaults else None + if not isinstance(value, dict): + raise ValueError("allow_lower_level_limits must be an object") + normalized = default_allow_lower_level_limits() if fill_defaults else {} + for key, allowed in value.items(): + clean_key = str(key) + if clean_key not in RETENTION_POLICY_FIELD_KEYS: + raise ValueError(f"Unknown retention policy field: {clean_key}") + normalized[clean_key] = bool(allowed) + return normalized + + +class PrivacyRetentionPolicyItem(BaseModel): + model_config = ConfigDict(extra="forbid") + + store_raw_campaign_json: bool = True + raw_campaign_json_retention_days: int | None = Field(default=None, ge=0) + generated_eml_retention_days: int | None = Field(default=None, ge=0) + stored_report_detail_retention_days: int | None = Field(default=None, ge=0) + mock_mailbox_retention_days: int | None = Field(default=None, ge=0) + audit_detail_retention_days: int | None = Field(default=None, ge=0) + audit_detail_level: Literal["full", "redacted", "minimal"] = "full" + allow_lower_level_limits: dict[str, bool] = Field(default_factory=default_allow_lower_level_limits) + + @field_validator("allow_lower_level_limits", mode="before") + @classmethod + def _normalize_allow_lower_level_limits(cls, value: Any) -> Any: + return normalize_allow_lower_level_limits(value, fill_defaults=True) + + +class PrivacyRetentionPolicyPatchItem(BaseModel): + model_config = ConfigDict(extra="forbid") + + store_raw_campaign_json: bool | None = None + raw_campaign_json_retention_days: int | None = Field(default=None, ge=0) + generated_eml_retention_days: int | None = Field(default=None, ge=0) + stored_report_detail_retention_days: int | None = Field(default=None, ge=0) + mock_mailbox_retention_days: int | None = Field(default=None, ge=0) + audit_detail_retention_days: int | None = Field(default=None, ge=0) + audit_detail_level: Literal["full", "redacted", "minimal"] | None = None + allow_lower_level_limits: dict[str, bool] | None = None + + @field_validator("allow_lower_level_limits", mode="before") + @classmethod + def _normalize_allow_lower_level_limits(cls, value: Any) -> Any: + return normalize_allow_lower_level_limits(value, fill_defaults=False) + + +__all__ = [ + "RETENTION_DAY_KEYS", + "RETENTION_POLICY_FIELD_KEYS", + "PrivacyRetentionPolicyItem", + "PrivacyRetentionPolicyPatchItem", + "default_allow_lower_level_limits", + "normalize_allow_lower_level_limits", +] diff --git a/src/govoplan_core/security/http_fetch.py b/src/govoplan_core/security/http_fetch.py new file mode 100644 index 0000000..a900155 --- /dev/null +++ b/src/govoplan_core/security/http_fetch.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +import urllib.parse +import urllib.request +from dataclasses import dataclass +from typing import Mapping + + +@dataclass(frozen=True, slots=True) +class HttpFetchResponse: + status: int + headers: dict[str, str] + body: bytes + + def text(self, encoding: str = "utf-8") -> str: + return self.body.decode(encoding) + + +def is_http_url(value: str) -> bool: + try: + validate_http_url(value) + except ValueError: + return False + return True + + +def validate_http_url(value: str, *, label: str = "URL") -> str: + parsed = urllib.parse.urlparse(str(value).strip()) + if parsed.scheme not in {"http", "https"} or not parsed.netloc: + raise ValueError(f"{label} must be an absolute HTTP(S) URL.") + if parsed.username or parsed.password: + raise ValueError(f"{label} must not include embedded credentials.") + return urllib.parse.urlunparse(parsed) + + +def fetch_http( + url: str, + *, + timeout: float = 15, + label: str = "URL", + method: str = "GET", + headers: Mapping[str, str] | None = None, +) -> HttpFetchResponse: + request = urllib.request.Request( + validate_http_url(url, label=label), + headers=dict(headers or {}), + method=method, + ) + with urllib.request.urlopen(request, timeout=timeout) as response: # noqa: S310 - URL is validated by validate_http_url. # nosec B310 # nosemgrep: python.lang.security.audit.dynamic-urllib-use-detected.dynamic-urllib-use-detected + return HttpFetchResponse( + status=int(getattr(response, "status", 0)), + headers=dict(response.headers.items()), + body=response.read(), + ) + + +def fetch_http_text( + url: str, + *, + timeout: float = 15, + label: str = "URL", + method: str = "GET", + headers: Mapping[str, str] | None = None, + encoding: str = "utf-8", +) -> str: + return fetch_http(url, timeout=timeout, label=label, method=method, headers=headers).text(encoding) diff --git a/src/govoplan_core/security/module_permissions.py b/src/govoplan_core/security/module_permissions.py index 8991982..af2db63 100644 --- a/src/govoplan_core/security/module_permissions.py +++ b/src/govoplan_core/security/module_permissions.py @@ -95,6 +95,11 @@ MODULE_SYSTEM_SCOPES = frozenset( for legacy, module in LEGACY_TO_MODULE_SCOPES.items() if legacy.startswith("system:") ) +LEGACY_SYSTEM_SCOPES = frozenset( + legacy + for legacy in LEGACY_TO_MODULE_SCOPES + if legacy.startswith("system:") +) MODULE_TENANT_SCOPES = frozenset(LEGACY_TO_MODULE_SCOPES.values()) - MODULE_SYSTEM_SCOPES @@ -110,7 +115,7 @@ def compatible_required_scopes(required: str) -> tuple[str, ...]: def scopes_grant_compatible(scopes: Iterable[str], required: str) -> bool: granted = list(scopes) - if required in MODULE_SYSTEM_SCOPES: + if required in MODULE_SYSTEM_SCOPES or required in LEGACY_SYSTEM_SCOPES: return "*" in granted or "system:*" in granted or any( scope != "tenant:*" and scopes_grant([scope], alias) for scope in granted diff --git a/src/govoplan_core/security/permissions.py b/src/govoplan_core/security/permissions.py index 22052d4..3dfb9bc 100644 --- a/src/govoplan_core/security/permissions.py +++ b/src/govoplan_core/security/permissions.py @@ -3,6 +3,8 @@ from __future__ import annotations from dataclasses import dataclass from typing import Iterable +from govoplan_core.security.scope_aliases import LEGACY_SCOPE_ALIASES + @dataclass(frozen=True, slots=True) class PermissionDefinition: @@ -96,28 +98,6 @@ KNOWN_SCOPES = frozenset(item.scope for item in ALL_PERMISSIONS) TENANT_SCOPES = frozenset(item.scope for item in TENANT_PERMISSIONS) SYSTEM_SCOPES = frozenset(item.scope for item in SYSTEM_PERMISSIONS) -LEGACY_SCOPE_ALIASES: dict[str, frozenset[str]] = { - # Only names that are no longer canonical remain runtime aliases. Canonical - # permissions keep their narrow meaning; the Alembic migration expands old - # role records once so upgraded installations do not lose prior access. - "campaign:write": frozenset({ - "campaign:create", "campaign:update", "campaign:copy", "campaign:archive", "campaign:delete", "campaign:share", - "recipients:read", "recipients:write", "recipients:import", - }), - "attachments:read": frozenset({"files:read", "files:download"}), - "attachments:write": frozenset({"files:upload", "files:organize", "files:share", "files:delete"}), - "admin:users": frozenset({ - "admin:users:read", "admin:users:create", "admin:users:update", "admin:users:suspend", - "admin:groups:read", "admin:groups:write", "admin:groups:manage_members", - "admin:roles:read", "admin:roles:write", "admin:roles:assign", - }), - "admin:users:write": frozenset({"admin:users:create", "admin:users:update", "admin:users:suspend", "admin:roles:assign", "admin:groups:manage_members"}), - "admin:api_keys:write": frozenset({"admin:api_keys:create", "admin:api_keys:revoke"}), - "admin:settings": frozenset({"admin:settings:read", "admin:settings:write", "admin:api_keys:read", "admin:api_keys:create", "admin:api_keys:revoke"}), - "system:tenants:write": frozenset({"system:tenants:create", "system:tenants:update", "system:tenants:suspend"}), - "system:access:write": frozenset({"system:access:assign", "system:roles:assign", "system:accounts:create", "system:accounts:update", "system:accounts:suspend"}), -} - DEFAULT_TENANT_ROLES: dict[str, dict[str, object]] = { "owner": {"name": "Owner", "description": "Full tenant access, including administration and delivery.", "permissions": ["tenant:*"], "is_builtin": True, "is_assignable": True}, "tenant_admin": {"name": "Tenant administrator", "description": "Manage tenant settings, users, groups and roles. Real delivery remains separately delegable.", "permissions": [ diff --git a/src/govoplan_core/security/scope_aliases.py b/src/govoplan_core/security/scope_aliases.py new file mode 100644 index 0000000..183874a --- /dev/null +++ b/src/govoplan_core/security/scope_aliases.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +LEGACY_SCOPE_ALIASES: dict[str, frozenset[str]] = { + # Only names that are no longer canonical remain runtime aliases. Canonical + # permissions keep their narrow meaning; migrations expand old role records + # once so upgraded installations do not lose prior access. + "campaign:write": frozenset({ + "campaign:create", + "campaign:update", + "campaign:copy", + "campaign:archive", + "campaign:delete", + "campaign:share", + "recipients:read", + "recipients:write", + "recipients:import", + }), + "attachments:read": frozenset({"files:read", "files:download"}), + "attachments:write": frozenset({"files:upload", "files:organize", "files:share", "files:delete"}), + "admin:users": frozenset({ + "admin:users:read", + "admin:users:create", + "admin:users:update", + "admin:users:suspend", + "admin:groups:read", + "admin:groups:write", + "admin:groups:manage_members", + "admin:roles:read", + "admin:roles:write", + "admin:roles:assign", + }), + "admin:users:write": frozenset({ + "admin:users:create", + "admin:users:update", + "admin:users:suspend", + "admin:roles:assign", + "admin:groups:manage_members", + }), + "admin:api_keys:write": frozenset({"admin:api_keys:create", "admin:api_keys:revoke"}), + "admin:settings": frozenset({ + "admin:settings:read", + "admin:settings:write", + "admin:api_keys:read", + "admin:api_keys:create", + "admin:api_keys:revoke", + }), + "system:tenants:write": frozenset({"system:tenants:create", "system:tenants:update", "system:tenants:suspend"}), + "system:access:write": frozenset({ + "system:access:assign", + "system:roles:assign", + "system:accounts:create", + "system:accounts:update", + "system:accounts:suspend", + }), +} diff --git a/src/govoplan_core/security/secrets.py b/src/govoplan_core/security/secrets.py index a0230e7..182cc8d 100644 --- a/src/govoplan_core/security/secrets.py +++ b/src/govoplan_core/security/secrets.py @@ -38,8 +38,8 @@ def _normalize_fernet_key(value: str) -> bytes: try: Fernet(candidate) return candidate - except Exception: - pass + except (TypeError, ValueError): + raw = None try: raw = base64.b64decode(candidate) except Exception as exc: diff --git a/src/govoplan_core/server/app.py b/src/govoplan_core/server/app.py index d8c9939..d728ef6 100644 --- a/src/govoplan_core/server/app.py +++ b/src/govoplan_core/server/app.py @@ -13,52 +13,13 @@ from govoplan_core.server.route_validation import validate_no_route_collisions def create_app(config: GovoplanServerConfig | str | None = None): - if isinstance(config, str) or config is None: - server_config = load_server_config(config) - else: - server_config = config - - database_url = getattr(server_config.settings, "database_url", None) if server_config.settings is not None else None - if database_url: - dispose_previous = getattr(server_config.settings, "app_env", None) == "test" - configure_database(str(database_url), dispose_previous=dispose_previous) - - raw_enabled_modules = load_startup_enabled_modules(server_config.enabled_modules) - candidate_modules = startup_candidate_module_ids(server_config.enabled_modules, raw_enabled_modules) - startup_available_modules = available_module_manifests( - server_config.manifest_factories, - enabled_modules=candidate_modules, - ignore_load_errors=True, - ) - available_modules = available_module_manifests(server_config.manifest_factories, ignore_load_errors=True) - enabled_modules = load_startup_enabled_modules(server_config.enabled_modules, available=startup_available_modules) - registry = build_platform_registry(enabled_modules, manifest_factories=server_config.manifest_factories) - api_router = APIRouter(prefix=server_config.api_prefix) - - for router in server_config.base_routers: - api_router.include_router(router) - - lifecycle = ModuleLifecycleManager( - registry=registry, - available_modules=available_modules, - settings=server_config.settings, - api_prefix=server_config.api_prefix, - manifest_factories=tuple(server_config.manifest_factories), - module_context_data=server_config.module_context_data, - ) + server_config = _server_config(config) + _configure_app_database(server_config) + registry, available_modules = _server_module_registry(server_config) + api_router = _server_api_router(server_config, registry) + lifecycle = _server_lifecycle(server_config, registry=registry, available_modules=available_modules) lifecycle.configure_runtime() - api_router.include_router(create_platform_router(settings=server_config.settings)) - - for router in server_config.post_module_routers: - api_router.include_router(router) - - for contribution in server_config.extra_routers: - if contribution.should_include(server_config.settings, registry): - api_router.include_router(contribution.router) - - validate_no_route_collisions(api_router, owner="server startup routes") - app = create_govoplan_app( title=server_config.title, version=server_config.version, @@ -74,4 +35,61 @@ def create_app(config: GovoplanServerConfig | str | None = None): return app +def _server_config(config: GovoplanServerConfig | str | None) -> GovoplanServerConfig: + if isinstance(config, str) or config is None: + return load_server_config(config) + return config + + +def _configure_app_database(server_config: GovoplanServerConfig) -> None: + database_url = getattr(server_config.settings, "database_url", None) if server_config.settings is not None else None + if database_url: + dispose_previous = getattr(server_config.settings, "app_env", None) == "test" + configure_database(str(database_url), dispose_previous=dispose_previous) + + +def _server_module_registry(server_config: GovoplanServerConfig): + raw_enabled_modules = load_startup_enabled_modules(server_config.enabled_modules) + candidate_modules = startup_candidate_module_ids(server_config.enabled_modules, raw_enabled_modules) + startup_available_modules = available_module_manifests( + server_config.manifest_factories, + enabled_modules=candidate_modules, + ignore_load_errors=True, + ) + available_modules = available_module_manifests(server_config.manifest_factories, ignore_load_errors=True) + enabled_modules = load_startup_enabled_modules(server_config.enabled_modules, available=startup_available_modules) + registry = build_platform_registry(enabled_modules, manifest_factories=server_config.manifest_factories) + return registry, available_modules + + +def _server_api_router(server_config: GovoplanServerConfig, registry) -> APIRouter: + api_router = APIRouter(prefix=server_config.api_prefix) + for router in server_config.base_routers: + api_router.include_router(router) + api_router.include_router(create_platform_router(settings=server_config.settings)) + for router in server_config.post_module_routers: + api_router.include_router(router) + for contribution in server_config.extra_routers: + if contribution.should_include(server_config.settings, registry): + api_router.include_router(contribution.router) + validate_no_route_collisions(api_router, owner="server startup routes") + return api_router + + +def _server_lifecycle( + server_config: GovoplanServerConfig, + *, + registry, + available_modules, +) -> ModuleLifecycleManager: + return ModuleLifecycleManager( + registry=registry, + available_modules=available_modules, + settings=server_config.settings, + api_prefix=server_config.api_prefix, + manifest_factories=tuple(server_config.manifest_factories), + module_context_data=server_config.module_context_data, + ) + + app = create_app() diff --git a/src/govoplan_core/server/fastapi.py b/src/govoplan_core/server/fastapi.py index 15876a5..dfecc38 100644 --- a/src/govoplan_core/server/fastapi.py +++ b/src/govoplan_core/server/fastapi.py @@ -1,5 +1,8 @@ from __future__ import annotations +import logging +import os +import time from collections.abc import AsyncIterator, Callable, Iterable from contextlib import AbstractAsyncContextManager from typing import Any @@ -9,9 +12,19 @@ from fastapi.middleware.cors import CORSMiddleware from govoplan_core.core.events import event_context, new_event_id, normalize_trace_id from govoplan_core.core.registry import PlatformRegistry +from govoplan_core.db.query_metrics import collect_query_metrics from govoplan_core.server.conditional_requests import conditional_json_get_middleware LifespanFactory = Callable[[FastAPI], AbstractAsyncContextManager[None] | AsyncIterator[None]] +logger = logging.getLogger("govoplan.request") + + +def _slow_request_threshold_ms() -> float: + raw = os.getenv("GOVOPLAN_SLOW_REQUEST_MS", "500").strip() + try: + return float(raw) + except ValueError: + return 500.0 def create_govoplan_app( @@ -26,6 +39,7 @@ def create_govoplan_app( ) -> FastAPI: app = FastAPI(title=title, version=version, lifespan=lifespan) app.state.govoplan_registry = registry + slow_request_threshold_ms = _slow_request_threshold_ms() @app.middleware("http") async def request_correlation_context(request: Request, call_next): @@ -39,6 +53,42 @@ def create_govoplan_app( response.headers["X-Correlation-ID"] = correlation_id return response + @app.middleware("http") + async def slow_request_logging(request: Request, call_next): + started_at = time.perf_counter() + with collect_query_metrics() as db_metrics: + try: + response = await call_next(request) + except Exception: + elapsed_ms = (time.perf_counter() - started_at) * 1000 + if slow_request_threshold_ms > 0 and elapsed_ms >= slow_request_threshold_ms: + logger.warning( + "slow request failed method=%s path=%s duration_ms=%.1f db_query_count=%s db_time_ms=%.1f db_slowest_ms=%.1f db_error_count=%s", + request.method, + request.url.path, + elapsed_ms, + db_metrics.query_count, + db_metrics.total_ms, + db_metrics.slowest_ms, + db_metrics.error_count, + exc_info=True, + ) + raise + elapsed_ms = (time.perf_counter() - started_at) * 1000 + if slow_request_threshold_ms > 0 and elapsed_ms >= slow_request_threshold_ms: + logger.warning( + "slow request method=%s path=%s status=%s duration_ms=%.1f db_query_count=%s db_time_ms=%.1f db_slowest_ms=%.1f db_error_count=%s", + request.method, + request.url.path, + response.status_code, + elapsed_ms, + db_metrics.query_count, + db_metrics.total_ms, + db_metrics.slowest_ms, + db_metrics.error_count, + ) + return response + app.middleware("http")(conditional_json_get_middleware) origins = [item.strip() for item in cors_origins if item.strip()] diff --git a/src/govoplan_core/settings.py b/src/govoplan_core/settings.py index 2a89d95..70992ec 100644 --- a/src/govoplan_core/settings.py +++ b/src/govoplan_core/settings.py @@ -17,15 +17,15 @@ class Settings(BaseSettings): tenant_data_mode: str = Field(default="shared", alias="TENANT_DATA_MODE") tenant_db_url_template: str | None = Field(default=None, alias="TENANT_DB_URL_TEMPLATE") tenant_schema_template: str | None = Field(default=None, alias="TENANT_SCHEMA_TEMPLATE") - enabled_modules: str = Field(default="tenancy,organizations,identity,access,admin,dashboard,policy,audit,campaigns,files,mail,calendar,docs,ops", alias="ENABLED_MODULES") + enabled_modules: str = Field(default="tenancy,organizations,identity,access,admin,dashboard,policy,audit,campaigns,files,mail,calendar,poll,scheduling,notifications,docs,ops", alias="ENABLED_MODULES") migration_track: str = Field(default="release", alias="GOVOPLAN_MIGRATION_TRACK") redis_url: str = Field(default="redis://redis:6379/0", alias="REDIS_URL") celery_enabled: bool = Field(default=False, alias="CELERY_ENABLED") s3_endpoint_url: str = Field(default="http://garage:3900", alias="S3_ENDPOINT_URL") s3_region: str = Field(default="garage", alias="S3_REGION") - s3_access_key_id: str = Field(default="GKmultimailerdev0000000000000000", alias="S3_ACCESS_KEY_ID") - s3_secret_access_key: str = Field(default="multimailer-dev-secret-change-me", alias="S3_SECRET_ACCESS_KEY") + s3_access_key_id: str = Field(default="GKgovoplandev0000000000000000000", alias="S3_ACCESS_KEY_ID") + s3_secret_access_key: str = Field(default="govoplan-dev-secret-change-me", alias="S3_SECRET_ACCESS_KEY") s3_bucket: str = Field(default="attachments", alias="S3_BUCKET") # Managed file storage. Development defaults to local filesystem storage; @@ -41,19 +41,19 @@ class Settings(BaseSettings): file_upload_max_bytes: int = Field(default=50 * 1024 * 1024, alias="FILE_UPLOAD_MAX_BYTES") file_upload_zip_max_bytes: int = Field(default=250 * 1024 * 1024, alias="FILE_UPLOAD_ZIP_MAX_BYTES") - auth_session_cookie_name: str = Field(default="msm_session", alias="AUTH_SESSION_COOKIE_NAME") - auth_csrf_cookie_name: str = Field(default="msm_csrf", alias="AUTH_CSRF_COOKIE_NAME") + auth_session_cookie_name: str = Field(default="govoplan_session", alias="AUTH_SESSION_COOKIE_NAME") + auth_csrf_cookie_name: str = Field(default="govoplan_csrf", alias="AUTH_CSRF_COOKIE_NAME") auth_cookie_secure: bool = Field(default=False, alias="AUTH_COOKIE_SECURE") auth_cookie_samesite: str = Field(default="lax", alias="AUTH_COOKIE_SAMESITE") auth_cookie_domain: str | None = Field(default=None, alias="AUTH_COOKIE_DOMAIN") auth_session_hours: int = Field(default=12, alias="AUTH_SESSION_HOURS") master_key_b64: str | None = Field(default=None, alias="MASTER_KEY_B64") - celery_queues: str = Field(default="send_email,append_sent,default", alias="CELERY_QUEUES") + celery_queues: str = Field(default="send_email,append_sent,notifications,default", alias="CELERY_QUEUES") mock_mailbox_dir: str = Field(default="runtime/mock-mailbox", alias="MOCK_MAILBOX_DIR") # Development bootstrap only. Do not use this in production. - dev_bootstrap_api_key: str | None = Field(default="dev-multimailer-api-key", alias="DEV_BOOTSTRAP_API_KEY") + dev_bootstrap_api_key: str | None = Field(default="dev-govoplan-api-key", alias="DEV_BOOTSTRAP_API_KEY") dev_auto_migrate_enabled: bool = Field(default=True, alias="DEV_AUTO_MIGRATE_ENABLED") dev_bootstrap_enabled: bool = Field(default=False, alias="DEV_BOOTSTRAP_ENABLED") dev_bootstrap_password: str = Field(default="dev-admin", alias="DEV_BOOTSTRAP_PASSWORD") diff --git a/tests/test_api_smoke.py b/tests/test_api_smoke.py index 2ecb49b..f75c130 100644 --- a/tests/test_api_smoke.py +++ b/tests/test_api_smoke.py @@ -364,12 +364,79 @@ class ApiSmokeTests(unittest.TestCase): json={"email": "admin@example.local", "password": "test-admin"}, ) self.assertEqual(response.status_code, 200, response.text) - csrf = response.cookies.get("msm_csrf") + csrf = response.cookies.get("govoplan_csrf") self.assertTrue(csrf) + session_summary = self.client.get("/api/v1/auth/session") + self.assertEqual(session_summary.status_code, 200, session_summary.text) + session_payload = session_summary.json() + self.assertTrue(session_payload["authenticated"]) + self.assertEqual(session_payload["auth_method"], "session") + self.assertNotIn("scopes", session_payload) + self.assertNotIn("roles", session_payload) + self.assertNotIn("groups", session_payload) + + shell_summary = self.client.get("/api/v1/auth/shell") + self.assertEqual(shell_summary.status_code, 200, shell_summary.text) + shell_payload = shell_summary.json() + self.assertEqual(shell_payload["principal"]["auth_method"], "session") + self.assertFalse(shell_payload["profile_loaded"]) + self.assertFalse(shell_payload["roles_loaded"]) + self.assertFalse(shell_payload["groups_loaded"]) + self.assertIsInstance(shell_payload["scopes"], list) + self.assertTrue(shell_payload["scopes"]) + self.assertTrue(shell_payload["tenants"]) + self.assertTrue(all(membership["roles"] == [] for membership in shell_payload["tenants"])) + self.assertNotIn("roles", shell_payload) + self.assertNotIn("groups", shell_payload) + self.assertNotIn("available_languages", shell_payload) + self.assertNotIn("default_language", shell_payload) + + profile_summary = self.client.get("/api/v1/auth/profile") + self.assertEqual(profile_summary.status_code, 200, profile_summary.text) + profile_payload = profile_summary.json() + self.assertTrue(profile_payload["profile_loaded"]) + self.assertEqual(profile_payload["user"]["id"], session_payload["user"]["id"]) + self.assertIn("available_languages", profile_payload) + self.assertIn("default_language", profile_payload) + self.assertNotIn("roles", profile_payload) + self.assertNotIn("groups", profile_payload) + + roles_summary = self.client.get("/api/v1/auth/roles") + self.assertEqual(roles_summary.status_code, 200, roles_summary.text) + roles_payload = roles_summary.json() + self.assertTrue(roles_payload["roles_loaded"]) + self.assertTrue(roles_payload["roles"]) + self.assertNotIn("groups", roles_payload) + + groups_summary = self.client.get("/api/v1/auth/groups") + self.assertEqual(groups_summary.status_code, 200, groups_summary.text) + groups_payload = groups_summary.json() + self.assertTrue(groups_payload["groups_loaded"]) + self.assertIsInstance(groups_payload["groups"], list) + self.assertNotIn("roles", groups_payload) + + api_key_session = self.client.get("/api/v1/auth/session", headers={"X-API-Key": "test-api-key"}) + self.assertEqual(api_key_session.status_code, 200, api_key_session.text) + self.assertEqual(api_key_session.json()["auth_method"], "api_key") + + api_key_shell = self.client.get("/api/v1/auth/shell", headers={"X-API-Key": "test-api-key"}) + self.assertEqual(api_key_shell.status_code, 200, api_key_shell.text) + api_key_shell_payload = api_key_shell.json() + self.assertEqual(api_key_shell_payload["principal"]["auth_method"], "api_key") + self.assertFalse(api_key_shell_payload["profile_loaded"]) + self.assertFalse(api_key_shell_payload["roles_loaded"]) + self.assertFalse(api_key_shell_payload["groups_loaded"]) + self.assertIsInstance(api_key_shell_payload["scopes"], list) + me = self.client.get("/api/v1/auth/me") self.assertEqual(me.status_code, 200, me.text) me_payload = me.json() + self.assertEqual(session_payload["user"]["id"], me_payload["user"]["id"]) + self.assertEqual(session_payload["tenant"]["id"], me_payload["tenant"]["id"]) + self.assertEqual(shell_payload["user"]["id"], me_payload["user"]["id"]) + self.assertEqual(shell_payload["tenant"]["id"], me_payload["tenant"]["id"]) + self.assertTrue(me_payload["profile_loaded"]) self.assertEqual(me_payload["principal"]["account_id"], me_payload["user"]["account_id"]) self.assertEqual(me_payload["principal"]["membership_id"], me_payload["user"]["id"]) self.assertEqual(me_payload["principal"]["tenant_id"], me_payload["tenant"]["id"]) @@ -423,6 +490,20 @@ class ApiSmokeTests(unittest.TestCase): imap_host="mock.imap", ) + bootstrapped = self.client.get( + f"/api/v1/mail/profiles/{profile_id}/mailbox/bootstrap", + headers=headers, + params={"folder": "INBOX", "limit": 2}, + ) + self.assertEqual(bootstrapped.status_code, 200, bootstrapped.text) + bootstrap_payload = bootstrapped.json() + self.assertEqual(bootstrap_payload["folder"], "INBOX") + self.assertTrue(bootstrap_payload["folders"]["ok"]) + self.assertFalse(bootstrap_payload["folders"]["from_cache"]) + self.assertFalse(bootstrap_payload["messages"]["from_cache"]) + self.assertEqual(bootstrap_payload["messages"]["total_count"], 3) + self.assertEqual(len(bootstrap_payload["messages"]["messages"]), 2) + listed = self.client.get( f"/api/v1/mail/profiles/{profile_id}/mailbox/messages", headers=headers, @@ -433,6 +514,7 @@ class ApiSmokeTests(unittest.TestCase): self.assertEqual(payload["folder"], "INBOX") self.assertEqual(payload["total_count"], 3) self.assertEqual(len(payload["messages"]), 2) + self.assertTrue(payload["from_cache"]) self.assertTrue(all(message["folder"] == "INBOX" for message in payload["messages"])) self.assertTrue(payload["cursor_stable"]) self.assertFalse(payload["full"]) @@ -449,6 +531,14 @@ class ApiSmokeTests(unittest.TestCase): self.assertEqual(len(next_payload["messages"]), 1) self.assertFalse(next_payload["next_cursor"]) + cached_next_page = self.client.get( + f"/api/v1/mail/profiles/{profile_id}/mailbox/messages", + headers=headers, + params={"folder": "INBOX", "cursor": payload["next_cursor"]}, + ) + self.assertEqual(cached_next_page.status_code, 200, cached_next_page.text) + self.assertTrue(cached_next_page.json()["from_cache"]) + stale_cursor = encode_keyset_cursor( "mail.mailbox.messages.v1", fingerprint=keyset_query_fingerprint( @@ -739,7 +829,7 @@ class ApiSmokeTests(unittest.TestCase): schema = self.client.get("/api/v1/schemas/campaign", headers=headers) self.assertEqual(schema.status_code, 200, schema.text) - self.assertEqual(schema.json()["title"], "MultiMailer Campaign") + self.assertEqual(schema.json()["title"], "GovOPlaN Campaign") dev_mailbox = self.client.get("/api/v1/dev/mailbox/messages", headers=headers) self.assertEqual(dev_mailbox.status_code, 404, dev_mailbox.text) @@ -3269,7 +3359,7 @@ class ApiSmokeTests(unittest.TestCase): "invoices/archive/202605-010001-report.XLSX", "invoices/202605-010001-90100010-9601741.XLSX", }) - self.assertFalse(any("multimailer-managed-build" in value for value in resolved_paths)) + self.assertFalse(any("govoplan-managed-build" in value for value in resolved_paths)) jobs = self.client.get( f"/api/v1/campaigns/{campaign_id}/jobs", diff --git a/tests/test_http_fetch.py b/tests/test_http_fetch.py new file mode 100644 index 0000000..ad5ff86 --- /dev/null +++ b/tests/test_http_fetch.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +import unittest + +from govoplan_core.security.http_fetch import is_http_url, validate_http_url + + +class HttpFetchTests(unittest.TestCase): + def test_validate_http_url_accepts_absolute_http_urls_without_credentials(self) -> None: + self.assertEqual("https://example.test/catalog.json", validate_http_url("https://example.test/catalog.json")) + self.assertTrue(is_http_url("http://example.test/catalog.json")) + + def test_validate_http_url_rejects_non_http_urls_and_embedded_credentials(self) -> None: + for value in ( + "file:///etc/passwd", + "/relative/catalog.json", + "https://user@example.test/catalog.json", + "https://user:secret@example.test/catalog.json", + ): + with self.subTest(value=value): + self.assertFalse(is_http_url(value)) + with self.assertRaises(ValueError): + validate_http_url(value) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_mail_config.py b/tests/test_mail_config.py new file mode 100644 index 0000000..804682b --- /dev/null +++ b/tests/test_mail_config.py @@ -0,0 +1,32 @@ +from __future__ import annotations + +import unittest + +from govoplan_core.mail.config import normalize_split_transport_credentials + + +class MailConfigTests(unittest.TestCase): + def test_normalize_split_transport_credentials_moves_legacy_auth_fields(self) -> None: + payload = normalize_split_transport_credentials( + { + "smtp": {"host": "smtp.example.test", "username": "smtp-user", "password": "smtp-secret"}, + "imap": {"host": "imap.example.test", "enabled": True, "username": "imap-user"}, + "credentials": {"smtp": {"username": "existing"}}, + } + ) + + self.assertEqual( + { + "smtp": {"host": "smtp.example.test"}, + "imap": {"host": "imap.example.test"}, + "credentials": { + "smtp": {"username": "existing", "password": "smtp-secret"}, + "imap": {"username": "imap-user"}, + }, + }, + payload, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_module_system.py b/tests/test_module_system.py index 91ab4c9..4753884 100644 --- a/tests/test_module_system.py +++ b/tests/test_module_system.py @@ -14,7 +14,6 @@ import tempfile import textwrap import tomllib import unittest -import urllib.error from dataclasses import replace from pathlib import Path from types import SimpleNamespace @@ -35,6 +34,7 @@ from cryptography.hazmat.primitives import serialization from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey from govoplan_core.core.events import event_context +from govoplan_core.core import module_installer as module_installer_module from govoplan_core.core.migrations import migration_metadata_plan from govoplan_core.core.module_management import ( ModuleInstallPlan, @@ -227,7 +227,7 @@ class ModuleSystemTests(unittest.TestCase): self.assertTrue({"access", "admin", "tenancy", "policy", "audit", "dashboard", "files", "mail", "campaigns", "docs", "ops"}.issubset(manifests)) self.assertEqual(manifests["campaigns"].dependencies, ()) self.assertTrue(manifests["campaigns"].required_capabilities) - self.assertEqual(manifests["campaigns"].optional_dependencies, ("files", "mail")) + self.assertEqual(manifests["campaigns"].optional_dependencies, ("files", "mail", "notifications", "addresses")) self.assertEqual(manifests["dashboard"].dependencies, ()) self.assertTrue(manifests["dashboard"].required_capabilities) self.assertEqual(manifests["docs"].dependencies, ()) @@ -272,14 +272,16 @@ class ModuleSystemTests(unittest.TestCase): self.assertTrue(scopes_grant_compatible(["admin:users:read"], "access:membership:read")) self.assertTrue(scopes_grant_compatible(["access:tenant:read"], "system:tenants:read")) self.assertTrue(scopes_grant_compatible(["system:*"], "access:tenant:read")) + self.assertFalse(scopes_grant_compatible(["tenant:*"], "system:tenants:read")) self.assertFalse(scopes_grant_compatible(["tenant:*"], "access:tenant:read")) def test_core_webui_retired_legacy_admin_api_surface(self) -> None: webui_src = Path(__file__).resolve().parents[1] / "webui" / "src" + legacy_admin_import_pattern = re.compile(r"api/admin(?=[\"'#?]|$)") self.assertFalse((webui_src / "api" / "admin.ts").exists()) for path in webui_src.rglob("*.ts*"): with self.subTest(path=path.relative_to(webui_src)): - self.assertNotIn("api/admin", path.read_text(encoding="utf-8")) + self.assertIsNone(legacy_admin_import_pattern.search(path.read_text(encoding="utf-8"))) def test_platform_modules_own_live_legacy_model_tables(self) -> None: from govoplan_admin.backend.db.models import GovernanceTemplate @@ -2474,7 +2476,8 @@ finally: Base.metadata.create_all(bind=database.engine, tables=[SystemSettings.__table__]) def fake_run(*_args, **kwargs): - if kwargs.get("shell"): + argv = tuple(_args[0]) if _args else () + if kwargs.get("shell") or argv == ("restart-fails",): return SimpleNamespace(returncode=1, stdout="", stderr="restart failed") return SimpleNamespace(returncode=0, stdout="", stderr="") @@ -2562,6 +2565,43 @@ finally: self.assertEqual(database_url, (result.record_path.parent / database_backup["database_url_secret"]).read_text(encoding="utf-8")) self.assertEqual("backup", (result.record_path.parent / "database.external.backup").read_text(encoding="utf-8")) + def test_module_installer_rejects_unsafe_external_database_hook_command(self) -> None: + root = Path(tempfile.mkdtemp(prefix="govoplan-installer-unsafe-hook-", dir=_TEST_ROOT)) + settings = _settings(root) + configure_database(settings.database_url) + database = get_database() + Base.metadata.create_all(bind=database.engine, tables=[SystemSettings.__table__]) + backup_command = f"{sys.executable} -c {shlex.quote('print(1)')} && {sys.executable} -c {shlex.quote('print(2)')}" + + with database.session() as session: + save_maintenance_mode(session, MaintenanceMode(enabled=True)) + plan = save_module_install_plan(session, [{ + "module_id": "files", + "action": "install", + "python_package": "govoplan-files", + "python_ref": "govoplan-files==0.1.4", + }]) + session.commit() + + with self.assertRaisesRegex(module_installer_module.ModuleInstallerError, "Database backup command failed"): + run_module_install_plan( + session=session, + plan=plan, + available=available_module_manifests(), + current_enabled=("tenancy", "access"), + desired_enabled=("tenancy", "access"), + database_url="postgresql://db.example.invalid/govoplan", + runtime_dir=root / "installer", + migrate_database=True, + database_backup_command=backup_command, + database_restore_command="restore-database", + dry_run=True, + ) + + rejected = module_installer_module._run_operator_command("DATABASE_URL=postgres://db.example.invalid/govoplan pg_dump") + self.assertEqual(2, rejected.returncode) + self.assertIn("Environment assignments", rejected.stderr) + def test_module_installer_external_database_restore_check_runs_before_migrations(self) -> None: root = Path(tempfile.mkdtemp(prefix="govoplan-installer-restore-check-", dir=_TEST_ROOT)) settings = _settings(root) @@ -2654,11 +2694,13 @@ finally: queued = queue_module_installer_request( runtime_dir=runtime_dir, requested_by="user-1", + tenant_id="tenant-1", options={"migrate_database": True, "health_urls": ["http://127.0.0.1:8000/health"]}, ) request_id = str(queued["request_id"]) self.assertEqual("queued", queued["status"]) + self.assertEqual("tenant-1", queued["tenant_id"]) self.assertEqual("installer-trace-1", queued["trace"]["correlation_id"]) self.assertEqual((request_id,), tuple(item["request_id"] for item in list_module_installer_requests(runtime_dir=runtime_dir))) claimed = claim_next_module_installer_request(runtime_dir=runtime_dir) @@ -2675,7 +2717,7 @@ finally: self.assertEqual("completed", updated["status"]) self.assertEqual("completed", read_module_installer_request(runtime_dir=runtime_dir, request_id=request_id)["status"]) - cancellable = queue_module_installer_request(runtime_dir=runtime_dir, requested_by="user-1") + cancellable = queue_module_installer_request(runtime_dir=runtime_dir, requested_by="user-1", tenant_id="tenant-1") cancelled = cancel_module_installer_request( runtime_dir=runtime_dir, request_id=str(cancellable["request_id"]), @@ -2691,6 +2733,7 @@ finally: ) self.assertEqual("queued", retry["status"]) self.assertEqual(cancellable["request_id"], retry["retry_of"]) + self.assertEqual("tenant-1", retry["tenant_id"]) settings = _settings(root) configure_database(settings.database_url) @@ -3030,11 +3073,11 @@ finally: "version_min": "0.1.0", "version_max_exclusive": "0.2.0", }, modules["campaigns"]["requires_interfaces"]) - self.assertEqual(["files", "mail"], modules["campaigns"]["optional_dependencies"]) + self.assertEqual(["files", "mail", "notifications", "addresses"], modules["campaigns"]["optional_dependencies"]) self.assertEqual("requires_review", modules["files"]["migration_safety"]) self.assertIn("migration", modules["files"]["migration_notes"].lower()) - self.assertEqual("0.1.7", modules["files"]["version"]) - self.assertIn("@v0.1.7", modules["files"]["python_ref"]) + self.assertEqual("0.1.8", modules["files"]["version"]) + self.assertIn("@v0.1.8", modules["files"]["python_ref"]) def test_module_package_catalog_validates_remote_url_and_cache_fallback(self) -> None: root = Path(tempfile.mkdtemp(prefix="govoplan-module-package-catalog-remote-", dir=_TEST_ROOT)) @@ -3067,16 +3110,6 @@ finally: ) body = signed_path.read_text(encoding="utf-8") - class _Response: - def __enter__(self): - return self - - def __exit__(self, *args): - return False - - def read(self) -> bytes: - return body.encode("utf-8") - env = { "GOVOPLAN_MODULE_PACKAGE_CATALOG_URL": "https://catalog.example.invalid/stable.json", "GOVOPLAN_MODULE_PACKAGE_CATALOG_CACHE": str(cache_path), @@ -3085,11 +3118,11 @@ finally: } trusted_keys = {"release-remote": base64.b64encode(public_key).decode("ascii")} with patch.dict(os.environ, env): - with patch("govoplan_core.core.module_package_catalog.urllib.request.urlopen", return_value=_Response()): + with patch("govoplan_core.core.module_package_catalog.fetch_http_text", return_value=body): fetched = validate_module_package_catalog(trusted_keys=trusted_keys) with patch( - "govoplan_core.core.module_package_catalog.urllib.request.urlopen", - side_effect=urllib.error.URLError("offline"), + "govoplan_core.core.module_package_catalog.fetch_http_text", + side_effect=OSError("offline"), ): cached = validate_module_package_catalog(trusted_keys=trusted_keys) @@ -3800,12 +3833,14 @@ finally: self._run_physical_absence_probe(enabled_modules=enabled_modules, blocked_modules=blocked_modules) def test_module_route_factories_receive_runtime_settings(self) -> None: - app, settings = self._app_for_modules(("files", "mail")) + app, settings = self._app_for_modules(("calendar", "files", "mail")) self.assertIsNotNone(app) + from govoplan_calendar.backend.runtime import get_settings as get_calendar_settings from govoplan_files.backend.runtime import get_settings as get_files_settings from govoplan_mail.backend.runtime import get_settings as get_mail_settings + self.assertIs(settings, get_calendar_settings()) self.assertIs(settings, get_files_settings()) self.assertIs(settings, get_mail_settings()) diff --git a/tests/test_privacy_schema_contracts.py b/tests/test_privacy_schema_contracts.py new file mode 100644 index 0000000..f6991bc --- /dev/null +++ b/tests/test_privacy_schema_contracts.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +import unittest + +from pydantic import ValidationError + +from govoplan_core.privacy.schemas import ( + RETENTION_POLICY_FIELD_KEYS, + PrivacyRetentionPolicyItem, + PrivacyRetentionPolicyPatchItem, + default_allow_lower_level_limits, + normalize_allow_lower_level_limits, +) + + +class PrivacySchemaContractTests(unittest.TestCase): + def test_default_allow_lower_level_limits_cover_every_retention_field(self) -> None: + self.assertEqual({key: True for key in RETENTION_POLICY_FIELD_KEYS}, default_allow_lower_level_limits()) + + def test_policy_item_fills_missing_lower_level_limits(self) -> None: + item = PrivacyRetentionPolicyItem.model_validate({"allow_lower_level_limits": {"audit_detail_level": False}}) + + self.assertFalse(item.allow_lower_level_limits["audit_detail_level"]) + self.assertTrue(item.allow_lower_level_limits["store_raw_campaign_json"]) + + def test_policy_patch_keeps_lower_level_limits_sparse(self) -> None: + patch = PrivacyRetentionPolicyPatchItem.model_validate({"allow_lower_level_limits": {"audit_detail_level": False}}) + + self.assertEqual({"audit_detail_level": False}, patch.allow_lower_level_limits) + self.assertIsNone(normalize_allow_lower_level_limits("", fill_defaults=False)) + + def test_unknown_lower_level_limit_field_is_rejected(self) -> None: + with self.assertRaises(ValidationError): + PrivacyRetentionPolicyItem.model_validate({"allow_lower_level_limits": {"unknown": True}}) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_query_metrics.py b/tests/test_query_metrics.py new file mode 100644 index 0000000..7eecc70 --- /dev/null +++ b/tests/test_query_metrics.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +import os +from unittest import TestCase +from unittest.mock import patch + +from fastapi import APIRouter +from fastapi.testclient import TestClient +from sqlalchemy import text + +from govoplan_core.core.registry import PlatformRegistry +from govoplan_core.db.query_metrics import collect_query_metrics +from govoplan_core.db.session import create_database_engine +from govoplan_core.server.fastapi import create_govoplan_app + + +class QueryMetricsTests(TestCase): + def test_collect_query_metrics_counts_instrumented_engine_queries(self) -> None: + engine = create_database_engine("sqlite:///:memory:") + try: + with collect_query_metrics() as metrics: + with engine.connect() as connection: + self.assertEqual(1, connection.execute(text("select 1")).scalar_one()) + + self.assertEqual(1, metrics.query_count) + self.assertGreaterEqual(metrics.total_ms, 0.0) + self.assertGreaterEqual(metrics.slowest_ms, 0.0) + self.assertEqual(0, metrics.error_count) + finally: + engine.dispose() + + def test_slow_request_log_includes_query_metrics(self) -> None: + engine = create_database_engine("sqlite:///:memory:") + router = APIRouter() + + @router.get("/query") + def query_route() -> dict[str, bool]: + with engine.connect() as connection: + connection.execute(text("select 1")).scalar_one() + return {"ok": True} + + try: + with patch.dict(os.environ, {"GOVOPLAN_SLOW_REQUEST_MS": "0.001"}): + app = create_govoplan_app( + title="query metrics test", + version="0", + registry=PlatformRegistry(), + api_router=router, + ) + + with TestClient(app) as client, self.assertLogs("govoplan.request", level="WARNING") as logs: + response = client.get("/query") + + self.assertEqual(200, response.status_code, response.text) + output = "\n".join(logs.output) + self.assertIn("db_query_count=1", output) + self.assertIn("db_time_ms=", output) + self.assertIn("db_slowest_ms=", output) + self.assertIn("db_error_count=0", output) + finally: + engine.dispose() diff --git a/tests/test_sqlalchemy_change_tracking.py b/tests/test_sqlalchemy_change_tracking.py new file mode 100644 index 0000000..9564f9e --- /dev/null +++ b/tests/test_sqlalchemy_change_tracking.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +import unittest +from datetime import datetime, timezone + +from sqlalchemy import DateTime, String, create_engine +from sqlalchemy.orm import DeclarativeBase, Mapped, Session, mapped_column + +from govoplan_core.core.sqlalchemy_change_tracking import ( + ensure_object_id, + operation_for_object, + operation_for_soft_deletable, + previous_value, +) + + +class Base(DeclarativeBase): + pass + + +class ExampleRecord(Base): + __tablename__ = "example_records" + + id: Mapped[str] = mapped_column(String, primary_key=True) + name: Mapped[str] = mapped_column(String) + deleted_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + + +class SqlalchemyChangeTrackingTests(unittest.TestCase): + def setUp(self) -> None: + self.engine = create_engine("sqlite:///:memory:") + Base.metadata.create_all(self.engine) + self.session = Session(self.engine, expire_on_commit=False) + + def tearDown(self) -> None: + self.session.close() + self.engine.dispose() + + def test_pending_object_is_created_and_can_receive_id(self) -> None: + record = ExampleRecord(id="", name="Draft", deleted_at=None) + self.session.add(record) + + self.assertEqual(ensure_object_id(record, lambda: "record-1"), "record-1") + self.assertEqual(record.id, "record-1") + self.assertEqual(operation_for_object(record, changed_attrs=("name",)), "created") + + def test_changed_object_reports_update_and_previous_value(self) -> None: + record = ExampleRecord(id="record-1", name="Before", deleted_at=None) + self.session.add(record) + self.session.flush() + + record.name = "After" + + self.assertEqual(operation_for_object(record, changed_attrs=("name",)), "updated") + self.assertEqual(previous_value(record, "name"), "Before") + + def test_soft_delete_and_restore_are_classified(self) -> None: + deleted_at = datetime(2026, 7, 13, tzinfo=timezone.utc) + record = ExampleRecord(id="record-1", name="Record", deleted_at=None) + self.session.add(record) + self.session.flush() + + record.deleted_at = deleted_at + self.assertEqual(operation_for_soft_deletable(record, changed_attrs=("deleted_at",)), "deleted") + + self.session.flush() + record.deleted_at = None + self.assertEqual(operation_for_soft_deletable(record, changed_attrs=("deleted_at",)), "created") + + +if __name__ == "__main__": + unittest.main() diff --git a/webui/package-lock.json b/webui/package-lock.json index 3b7a68e..ae4fe05 100644 --- a/webui/package-lock.json +++ b/webui/package-lock.json @@ -9,6 +9,7 @@ "version": "0.1.8", "dependencies": { "@govoplan/access-webui": "file:../../govoplan-access/webui", + "@govoplan/addresses-webui": "file:../../govoplan-addresses/webui", "@govoplan/admin-webui": "file:../../govoplan-admin/webui", "@govoplan/audit-webui": "file:../../govoplan-audit/webui", "@govoplan/calendar-webui": "file:../../govoplan-calendar/webui", @@ -18,6 +19,7 @@ "@govoplan/files-webui": "file:../../govoplan-files/webui", "@govoplan/idm-webui": "file:../../govoplan-idm/webui", "@govoplan/mail-webui": "file:../../govoplan-mail/webui", + "@govoplan/notifications-webui": "file:../../govoplan-notifications/webui", "@govoplan/ops-webui": "file:../../govoplan-ops/webui", "@govoplan/organizations-webui": "file:../../govoplan-organizations/webui", "@govoplan/policy-webui": "file:../../govoplan-policy/webui", @@ -61,6 +63,22 @@ } } }, + "../../govoplan-addresses/webui": { + "name": "@govoplan/addresses-webui", + "version": "0.1.8", + "peerDependencies": { + "@govoplan/core-webui": "^0.1.8", + "lucide-react": "^1.23.0", + "react": "^19.0.0", + "react-dom": "^19.0.0", + "react-router-dom": "^7.1.1" + }, + "peerDependenciesMeta": { + "@govoplan/core-webui": { + "optional": true + } + } + }, "../../govoplan-admin/webui": { "name": "@govoplan/admin-webui", "version": "0.1.8", @@ -229,6 +247,25 @@ } } }, + "../../govoplan-notifications/webui": { + "name": "@govoplan/notifications-webui", + "version": "0.1.8", + "peerDependencies": { + "@govoplan/core-webui": "^0.1.8", + "@vitejs/plugin-react": "^4.3.4", + "lucide-react": "^1.23.0", + "react": "^19.0.0", + "react-dom": "^19.0.0", + "react-router-dom": "^7.1.1", + "typescript": "^5.7.2", + "vite": "^6.0.6" + }, + "peerDependenciesMeta": { + "@govoplan/core-webui": { + "optional": true + } + } + }, "../../govoplan-ops/webui": { "name": "@govoplan/ops-webui", "version": "0.1.8", @@ -1030,6 +1067,10 @@ "resolved": "../../govoplan-access/webui", "link": true }, + "node_modules/@govoplan/addresses-webui": { + "resolved": "../../govoplan-addresses/webui", + "link": true + }, "node_modules/@govoplan/admin-webui": { "resolved": "../../govoplan-admin/webui", "link": true @@ -1066,6 +1107,10 @@ "resolved": "../../govoplan-mail/webui", "link": true }, + "node_modules/@govoplan/notifications-webui": { + "resolved": "../../govoplan-notifications/webui", + "link": true + }, "node_modules/@govoplan/ops-webui": { "resolved": "../../govoplan-ops/webui", "link": true diff --git a/webui/package.json b/webui/package.json index ea105cc..698ce4d 100644 --- a/webui/package.json +++ b/webui/package.json @@ -28,6 +28,7 @@ "dependencies": { "@govoplan/access-webui": "file:../../govoplan-access/webui", "@govoplan/admin-webui": "file:../../govoplan-admin/webui", + "@govoplan/addresses-webui": "file:../../govoplan-addresses/webui", "@govoplan/audit-webui": "file:../../govoplan-audit/webui", "@govoplan/calendar-webui": "file:../../govoplan-calendar/webui", "@govoplan/campaign-webui": "file:../../govoplan-campaign/webui", @@ -36,6 +37,7 @@ "@govoplan/files-webui": "file:../../govoplan-files/webui", "@govoplan/idm-webui": "file:../../govoplan-idm/webui", "@govoplan/mail-webui": "file:../../govoplan-mail/webui", + "@govoplan/notifications-webui": "file:../../govoplan-notifications/webui", "@govoplan/organizations-webui": "file:../../govoplan-organizations/webui", "@govoplan/ops-webui": "file:../../govoplan-ops/webui", "@govoplan/policy-webui": "file:../../govoplan-policy/webui", diff --git a/webui/package.release.json b/webui/package.release.json index af7d217..306c323 100644 --- a/webui/package.release.json +++ b/webui/package.release.json @@ -24,6 +24,7 @@ "dependencies": { "@govoplan/access-webui": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-access.git#v0.1.8", "@govoplan/admin-webui": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-admin.git#v0.1.8", + "@govoplan/addresses-webui": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-addresses.git#v0.1.8", "@govoplan/audit-webui": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-audit.git#v0.1.8", "@govoplan/calendar-webui": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-calendar.git#v0.1.8", "@govoplan/dashboard-webui": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-dashboard.git#v0.1.8", @@ -31,6 +32,7 @@ "@govoplan/files-webui": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-files.git#v0.1.8", "@govoplan/idm-webui": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-idm.git#v0.1.8", "@govoplan/mail-webui": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-mail.git#v0.1.8", + "@govoplan/notifications-webui": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-notifications.git#v0.1.8", "@govoplan/campaign-webui": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-campaign.git#v0.1.8", "@govoplan/organizations-webui": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-organizations.git#v0.1.8", "@govoplan/ops-webui": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-ops.git#v0.1.8", diff --git a/webui/scripts/test-module-permutations.mjs b/webui/scripts/test-module-permutations.mjs index a47071b..dfb6d7b 100644 --- a/webui/scripts/test-module-permutations.mjs +++ b/webui/scripts/test-module-permutations.mjs @@ -3,6 +3,7 @@ import { spawnSync } from "node:child_process"; const packageByModule = { access: "@govoplan/access-webui", admin: "@govoplan/admin-webui", + addresses: "@govoplan/addresses-webui", audit: "@govoplan/audit-webui", campaigns: "@govoplan/campaign-webui", dashboard: "@govoplan/dashboard-webui", @@ -20,6 +21,7 @@ const cases = [ { name: "core-only", modules: [] }, { name: "access-only", modules: ["access"] }, { name: "admin-only", modules: ["admin"] }, + { name: "addresses-only", modules: ["addresses"] }, { name: "access-with-admin", modules: ["access", "admin"] }, { name: "admin-with-policy-and-audit", modules: ["access", "admin", "policy", "audit"] }, { name: "dashboard-only", modules: ["dashboard"] }, @@ -33,7 +35,7 @@ const cases = [ { name: "scheduling-only", modules: ["scheduling"] }, { name: "scheduling-with-calendar", modules: ["scheduling", "calendar"] }, { name: "docs-and-ops", modules: ["access", "docs", "ops"] }, - { name: "full-product", modules: ["access", "admin", "policy", "audit", "dashboard", "organizations", "idm", "campaigns", "files", "mail", "docs", "ops", "scheduling"] } + { name: "full-product", modules: ["access", "admin", "addresses", "policy", "audit", "dashboard", "organizations", "idm", "campaigns", "files", "mail", "docs", "ops", "scheduling"] } ]; const npmExec = process.env.npm_execpath; diff --git a/webui/src/App.tsx b/webui/src/App.tsx index c94a2f2..ca416aa 100644 --- a/webui/src/App.tsx +++ b/webui/src/App.tsx @@ -1,9 +1,9 @@ import { Navigate, Route, Routes } from "react-router-dom"; import { lazy, Suspense, useEffect, useMemo, useState } from "react"; -import { fetchMe, updateProfile } from "./api/auth"; +import { fetchSession, fetchShellAuth, updateProfile } from "./api/auth"; import { fetchPlatformModules, fetchPlatformStatus } from "./api/platform"; import { AUTH_REQUIRED_EVENT, loadApiSettings, saveApiSettings, type AuthRequiredEventDetail } from "./api/client"; -import type { ApiSettings, AuthInfo, AuthTenant, AuthTenantMembership, AuthUser, LoginResponse, PlatformModuleInfo, PlatformWebModule, UserUiPreferences } from "./types"; +import type { ApiSettings, AuthInfo, AuthSessionInfo, AuthUpdate, AuthUser, LoginResponse, PlatformModuleInfo, PlatformWebModule, UserUiPreferences } from "./types"; import AppShell from "./layout/AppShell"; import PublicLandingPage from "./features/auth/PublicLandingPage"; import LoginModal from "./features/auth/LoginModal"; @@ -47,9 +47,9 @@ export default function App() { saveApiSettings(next); } - function updateAuth(next: AuthInfo | null, accessToken?: string) { + function updateAuth(next: AuthUpdate | null, accessToken?: string) { const nextSettings = accessToken !== undefined ? { ...settings, accessToken } : settings; - setAuth(next ? normalizeAuthInfo(next) : null); + setAuth((current) => next ? normalizeAuthInfo(mergeAuthPayload(current, next)) : null); if (accessToken !== undefined) { setSettings(nextSettings); saveApiSettings(nextSettings); @@ -117,21 +117,26 @@ export default function App() { useEffect(() => { let cancelled = false; setCheckingSession(true); - fetchMe(settings). - then((me) => {if (!cancelled) setAuth(normalizeAuthInfo(me));}). - catch(() => { - if (!cancelled) { - const cleared = { ...settings, accessToken: "" }; - setSettings(cleared); - saveApiSettings(cleared); - setAuth(null); - setPlatformModules(null); - setRemoteWebModules([]); + + async function bootstrapAuth() { + try { + const shellAuth = await fetchShellAuth(settings); + if (!cancelled) setAuth(normalizeAuthInfo(shellAuth)); + } catch { + if (!cancelled) { + const cleared = { ...settings, accessToken: "" }; + setSettings(cleared); + saveApiSettings(cleared); + setAuth(null); + setPlatformModules(null); + setRemoteWebModules([]); + } + } finally { + if (!cancelled) setCheckingSession(false); } - }). - finally(() => { - if (!cancelled) setCheckingSession(false); - }); + } + + void bootstrapAuth(); return () => {cancelled = true;}; }, [settings.apiBaseUrl, settings.apiKey]); @@ -180,11 +185,25 @@ export default function App() { root.classList.toggle("ui-hide-help-hints", !preferences.show_inline_help_hints); root.classList.toggle("ui-reduce-motion", preferences.reduce_motion); root.classList.toggle("ui-no-sticky-section-sidebars", !preferences.sticky_section_sidebars); - if (preferences.theme === "system") { - delete root.dataset.theme; - } else { - root.dataset.theme = preferences.theme; + + const systemDarkQuery = window.matchMedia?.("(prefers-color-scheme: dark)") ?? null; + const applyTheme = () => { + const resolvedTheme = preferences.theme === "system" ? + systemDarkQuery?.matches ? "dark" : "light" : + preferences.theme; + root.dataset.theme = resolvedTheme; + root.dataset.themePreference = preferences.theme; + }; + + applyTheme(); + if (preferences.theme !== "system" || !systemDarkQuery) { + return undefined; } + + systemDarkQuery.addEventListener("change", applyTheme); + return () => { + systemDarkQuery.removeEventListener("change", applyTheme); + }; }, [ auth?.user.ui_preferences?.compact_tables, auth?.user.ui_preferences?.show_inline_help_hints, @@ -208,8 +227,11 @@ export default function App() { useEffect(() => { if (!auth) return; + const currentAuth = auth; + let cancelled = false; let inFlight = false; let lastRefreshAt = 0; + let lastShellRefreshAt = Date.now(); async function refreshVisibleSession() { if (document.visibilityState === "hidden" || inFlight) return; @@ -219,7 +241,17 @@ export default function App() { inFlight = true; lastRefreshAt = now; try { - setAuth(normalizeAuthInfo(await fetchMe(settings))); + const sessionInfo = await fetchSession(settings); + if (cancelled) return; + const shellRefreshDue = now - lastShellRefreshAt >= 60_000; + if (!sessionMatchesAuth(sessionInfo, currentAuth) || shellRefreshDue) { + const shellAuth = await fetchShellAuth(settings); + if (cancelled) return; + lastShellRefreshAt = Date.now(); + setAuth((current) => current && sessionMatchesAuth(sessionInfo, current) + ? normalizeAuthInfo(mergeAuthPayload(current, shellAuth)) + : normalizeAuthInfo(shellAuth)); + } } catch { @@ -230,6 +262,7 @@ export default function App() { window.addEventListener("focus", refreshVisibleSession); document.addEventListener("visibilitychange", refreshVisibleSession); return () => { + cancelled = true; window.removeEventListener("focus", refreshVisibleSession); document.removeEventListener("visibilitychange", refreshVisibleSession); }; @@ -327,18 +360,40 @@ export default function App() { } -type AuthPayload = Partial & { - principal?: AuthInfo["principal"]; - user?: Partial | null; - tenant?: AuthTenant | null; - active_tenant?: AuthTenant | null; - tenants?: AuthTenantMembership[] | null; -}; +type AuthPayload = AuthUpdate; + +function mergeAuthPayload(current: AuthInfo | null, next: AuthPayload): AuthPayload { + if (!current) return next; + const nextActiveTenant = next.active_tenant ?? next.tenant ?? null; + const currentActiveTenant = current.active_tenant ?? current.tenant; + const tenantChanged = Boolean(nextActiveTenant && nextActiveTenant.id !== currentActiveTenant.id); + return { + ...current, + ...next, + user: next.user ? { ...current.user, ...next.user } : current.user, + tenant: nextActiveTenant ?? current.tenant, + active_tenant: nextActiveTenant ?? currentActiveTenant, + tenants: next.tenants ?? current.tenants, + scopes: next.scopes ?? current.scopes, + roles: next.roles ?? (next.roles_loaded === false || tenantChanged ? [] : current.roles), + groups: next.groups ?? (next.groups_loaded === false || tenantChanged ? [] : current.groups), + principal: next.principal === undefined ? current.principal : next.principal, + available_languages: next.available_languages ?? (tenantChanged ? undefined : current.available_languages), + enabled_language_codes: next.enabled_language_codes ?? (tenantChanged ? undefined : current.enabled_language_codes), + default_language: next.default_language ?? (tenantChanged ? undefined : current.default_language), + profile_loaded: next.profile_loaded ?? (tenantChanged ? false : current.profile_loaded), + roles_loaded: next.roles_loaded ?? (tenantChanged ? false : current.roles_loaded), + groups_loaded: next.groups_loaded ?? (tenantChanged ? false : current.groups_loaded) + }; +} function normalizeAuthInfo(response: AuthPayload): AuthInfo { const principal = response.principal ?? null; const activeTenant = response.active_tenant ?? response.tenant ?? response.tenants?.[0] ?? null; const user = normalizeAuthUser(response.user, principal); + const profileLoaded = response.profile_loaded ?? hasFullProfilePayload(response); + const rolesLoaded = response.roles_loaded ?? response.roles !== undefined; + const groupsLoaded = response.groups_loaded ?? response.groups !== undefined; if (!activeTenant) { throw new Error("Authentication response did not include an active tenant."); @@ -356,12 +411,25 @@ function normalizeAuthInfo(response: AuthPayload): AuthInfo { roles: response.roles ?? [], groups: response.groups ?? [], principal, - available_languages: response.available_languages ?? [], - enabled_language_codes: response.enabled_language_codes ?? activeTenant.enabled_language_codes ?? [], - default_language: response.default_language ?? user.preferred_language ?? activeTenant.default_locale ?? "en" + available_languages: response.available_languages, + enabled_language_codes: profileLoaded ? response.enabled_language_codes ?? activeTenant.enabled_language_codes ?? [] : undefined, + default_language: profileLoaded ? response.default_language ?? user.preferred_language ?? activeTenant.default_locale ?? "en" : undefined, + profile_loaded: profileLoaded, + roles_loaded: rolesLoaded, + groups_loaded: groupsLoaded }; } +function hasFullProfilePayload(response: AuthPayload): boolean { + return Boolean( + response.default_language || + response.available_languages || + response.enabled_language_codes || + response.roles || + response.groups + ); +} + function normalizeAuthUser(user: Partial | null | undefined, principal: AuthInfo["principal"]): AuthUser | null { if (user?.id && user.account_id) { return { @@ -396,6 +464,17 @@ function normalizeAuthUser(user: Partial | null | undefined, principal }; } +function sessionMatchesAuth(sessionInfo: AuthSessionInfo, auth: AuthInfo): boolean { + const activeTenant = auth.active_tenant ?? auth.tenant; + if (sessionInfo.user.id !== auth.user.id) return false; + if (sessionInfo.user.account_id !== auth.user.account_id) return false; + if ((sessionInfo.active_tenant ?? sessionInfo.tenant).id !== activeTenant.id) return false; + if (auth.principal?.auth_method && sessionInfo.auth_method !== auth.principal.auth_method) return false; + if (auth.principal?.session_id && sessionInfo.session_id && auth.principal.session_id !== sessionInfo.session_id) return false; + if (auth.principal?.api_key_id && sessionInfo.api_key_id && auth.principal.api_key_id !== sessionInfo.api_key_id) return false; + return true; +} + function normalizeUiPreferences(value: Partial | null | undefined): UserUiPreferences { const theme = value?.theme === "light" || value?.theme === "dark" || value?.theme === "system" ? value.theme : "system"; return { diff --git a/webui/src/api/adminCommon.ts b/webui/src/api/adminCommon.ts new file mode 100644 index 0000000..e40ae83 --- /dev/null +++ b/webui/src/api/adminCommon.ts @@ -0,0 +1,56 @@ +import type { ApiSettings } from "../types"; +import { apiFetch, apiGetList } from "./client"; + +export type PermissionItem = { + scope: string; + label: string; + description: string; + category: string; + level: "tenant" | "system"; + system_template_id?: string | null; + system_required?: boolean; +}; + +export type AdminOverview = { + active_tenant_id: string; + active_tenant_name: string; + tenant_count?: number | null; + system_account_count?: number | null; + system_group_template_count?: number | null; + system_role_template_count?: number | null; + user_count: number; + active_user_count: number; + group_count: number; + role_count: number; + active_api_key_count: number; + capabilities: string[]; +}; + +export type TenantAdminItem = { + id: string; + slug: string; + name: string; + description?: string | null; + default_locale: string; + settings: Record; + allow_custom_groups?: boolean | null; + allow_custom_roles?: boolean | null; + allow_api_keys?: boolean | null; + effective_governance: Record; + is_active: boolean; + counts: Record; + created_at: string; + updated_at: string; +}; + +export function fetchAdminOverview(settings: ApiSettings): Promise { + return apiFetch(settings, "/api/v1/admin/overview"); +} + +export async function fetchPermissionCatalog(settings: ApiSettings): Promise { + return apiGetList(settings, "/api/v1/admin/permissions", "permissions"); +} + +export async function fetchTenants(settings: ApiSettings): Promise { + return apiGetList(settings, "/api/v1/admin/tenants", "tenants"); +} diff --git a/webui/src/api/auth.ts b/webui/src/api/auth.ts index 685e6c2..c7addae 100644 --- a/webui/src/api/auth.ts +++ b/webui/src/api/auth.ts @@ -1,4 +1,4 @@ -import type { ApiSettings, AuthInfo, LoginResponse, UserUiPreferences } from "../types"; +import type { ApiSettings, AuthGroupsInfo, AuthInfo, AuthProfileInfo, AuthRolesInfo, AuthSessionInfo, AuthShellInfo, LoginResponse, UserUiPreferences } from "../types"; import { apiFetch } from "./client"; export async function login( @@ -15,8 +15,28 @@ export async function fetchMe(settings: ApiSettings): Promise { return apiFetch(settings, "/api/v1/auth/me"); } -export async function switchTenant(settings: ApiSettings, tenantId: string): Promise { - return apiFetch(settings, "/api/v1/auth/switch-tenant", { +export async function fetchSession(settings: ApiSettings): Promise { + return apiFetch(settings, "/api/v1/auth/session", { cache: "no-store" }); +} + +export async function fetchShellAuth(settings: ApiSettings): Promise { + return apiFetch(settings, "/api/v1/auth/shell", { cache: "no-store" }); +} + +export async function fetchAuthProfile(settings: ApiSettings): Promise { + return apiFetch(settings, "/api/v1/auth/profile", { cache: "no-store" }); +} + +export async function fetchAuthRoles(settings: ApiSettings): Promise { + return apiFetch(settings, "/api/v1/auth/roles", { cache: "no-store" }); +} + +export async function fetchAuthGroups(settings: ApiSettings): Promise { + return apiFetch(settings, "/api/v1/auth/groups", { cache: "no-store" }); +} + +export async function switchTenant(settings: ApiSettings, tenantId: string): Promise { + return apiFetch(settings, "/api/v1/auth/switch-tenant", { method: "POST", body: JSON.stringify({ tenant_id: tenantId }) }); @@ -35,8 +55,8 @@ export async function updateProfile( enabled_language_codes?: string[] | null; ui_preferences?: Partial | null; } -): Promise { - return apiFetch(settings, "/api/v1/auth/profile", { +): Promise { + return apiFetch(settings, "/api/v1/auth/profile", { method: "PATCH", body: JSON.stringify(payload) }); diff --git a/webui/src/api/client.ts b/webui/src/api/client.ts index 754797b..dea019f 100644 --- a/webui/src/api/client.ts +++ b/webui/src/api/client.ts @@ -1,9 +1,9 @@ import type { ApiSettings } from "../types"; -const STORAGE_KEY = "multimailer.apiSettings"; -const LEGACY_STORAGE_KEYS = ["i18n:govoplan-core.multimailer_apisettings.1d1601d4"]; -const SESSION_STORAGE_KEY = "multimailer.session"; -const CSRF_COOKIE_NAME = import.meta.env.VITE_CSRF_COOKIE_NAME ?? "msm_csrf"; +const STORAGE_KEY = "govoplan.apiSettings"; +const LEGACY_STORAGE_KEYS: string[] = []; +const SESSION_STORAGE_KEY = "govoplan.session"; +const CSRF_COOKIE_NAME = import.meta.env.VITE_CSRF_COOKIE_NAME ?? "govoplan_csrf"; const RECENT_SAFE_REQUEST_TTL_MS = 750; const MAX_RECENT_SAFE_REQUESTS = 100; const MAX_CONDITIONAL_SAFE_REQUESTS = 200; @@ -77,6 +77,56 @@ export function apiUrl(settings: ApiSettings, path: string): string { return baseUrl ? `${baseUrl}${normalizedPath}` : normalizedPath; } +export type ApiQueryValue = string | number | boolean | null | undefined; +export type ApiQueryParams = Record; + +function queryValue(value: ApiQueryValue): string | null { + if (value === null || value === undefined || value === "") return null; + return String(value); +} + +export function apiQuery(params: ApiQueryParams = {}): string { + const search = new URLSearchParams(); + for (const [key, rawValue] of Object.entries(params)) { + const values = Array.isArray(rawValue) ? rawValue : [rawValue]; + for (const value of values) { + const normalized = queryValue(value); + if (normalized !== null) search.append(key, normalized); + } + } + const query = search.toString(); + return query ? `?${query}` : ""; +} + +export function apiPath(path: string, params: ApiQueryParams = {}): string { + const query = apiQuery(params); + if (!query) return path; + return path.includes("?") ? `${path}&${query.slice(1)}` : `${path}${query}`; +} + +export async function apiGetList( + settings: ApiSettings, + path: string, + key: K, + params: ApiQueryParams = {} +): Promise { + const response = await apiFetch>(settings, apiPath(path, params)); + return response[key] ?? []; +} + +export function apiPost(settings: ApiSettings, path: string, init: Omit = {}): Promise { + return apiFetch(settings, path, { ...init, method: "POST" }); +} + +export function apiPostJson( + settings: ApiSettings, + path: string, + payload: TPayload, + init: Omit = {} +): Promise { + return apiPost(settings, path, { ...init, body: JSON.stringify(payload) }); +} + export function loadApiSettings(): ApiSettings { const storedBaseUrl = loadStoredSetting("baseUrl"); const storedApiKey = sessionStorage.getItem(`${SESSION_STORAGE_KEY}.apiKey`); diff --git a/webui/src/api/mailContracts.ts b/webui/src/api/mailContracts.ts new file mode 100644 index 0000000..838ebcd --- /dev/null +++ b/webui/src/api/mailContracts.ts @@ -0,0 +1,143 @@ +import type { + MailImapTransportSettings, + MailProfilePatternKey, + MailProfilePolicy, + MailProfileScope, + MailSecurity, + MailServerProfile, + MailServerProfileCredentials, + MailTransportCredentials, + MailTransportSettings +} from "../types"; + +export type { + MailCredentialPolicy, + MailProfilePatternKey, + MailProfilePolicy, + MailProfileScope, + MailSecurity, + MailServerProfile +} from "../types"; + +export type MailSmtpTestPayload = MailTransportSettings; +export type MailImapTestPayload = MailImapTransportSettings; +export type MailTransportCredentialsPayload = MailTransportCredentials; +export type MailServerProfileCredentialsPayload = MailServerProfileCredentials; +export type MailServerProfileListResponse = { profiles: MailServerProfile[] }; +export type MailProfilePatternRules = Partial>; + +export type MailConnectionTestResponse = { + ok: boolean; + protocol: "smtp" | "imap"; + host?: string | null; + port?: number | null; + security?: MailSecurity | string | null; + message: string; + details?: Record; +}; + +export type MailImapFolderResponse = { + name: string; + flags?: string[]; + message_count?: number | null; + unseen_count?: number | null; +}; + +export type MailImapFolderListResponse = { + ok: boolean; + protocol: "imap"; + host?: string | null; + port?: number | null; + security?: MailSecurity | string | null; + message: string; + folders: MailImapFolderResponse[]; + detected_sent_folder?: string | null; + from_cache?: boolean; + refreshing?: boolean; + indexed_at?: string | null; + details?: Record; +}; + +export const mailProfilePatternKeys = [ + "smtp_hosts", + "imap_hosts", + "envelope_senders", + "from_headers", + "recipient_domains" +] as const satisfies readonly MailProfilePatternKey[]; + +export const mailProfilePolicyLimitKeys = [ + "allowed_profile_ids", + "allow_user_profiles", + "allow_group_profiles", + "smtp_credentials.inherit", + "imap_credentials.inherit", + "whitelist.smtp_hosts", + "whitelist.imap_hosts", + "whitelist.envelope_senders", + "whitelist.from_headers", + "whitelist.recipient_domains", + "blacklist.smtp_hosts", + "blacklist.imap_hosts", + "blacklist.envelope_senders", + "blacklist.from_headers", + "blacklist.recipient_domains" +] as const; + +export type MailProfilePolicyLimitKey = typeof mailProfilePolicyLimitKeys[number]; +export type MailProfilePolicyLimitPermissions = Partial>; + +export type MailPolicySourceStep = { + scope_type: string; + scope_id?: string | null; + label: string; + applied_fields?: string[]; + policy?: MailProfilePolicy | null; +}; + +export type MailProfilePolicyResponse = { + scope_type: MailProfileScope; + scope_id?: string | null; + policy: MailProfilePolicy; + effective_policy?: MailProfilePolicy | null; + parent_policy?: MailProfilePolicy | null; + effective_policy_sources?: MailPolicySourceStep[]; + parent_policy_sources?: MailPolicySourceStep[]; +}; + +export type MailServerProfilePayload = { + name: string; + slug?: string | null; + description?: string | null; + is_active?: boolean; + scope_type?: MailProfileScope; + scope_id?: string | null; + smtp: MailSmtpTestPayload; + imap?: MailImapTestPayload | null; + credentials?: MailServerProfileCredentialsPayload | null; +}; + +export type MockMailboxMessage = { + id: string; + kind: "smtp" | "imap_append" | string; + created_at: string; + envelope_from?: string | null; + envelope_recipients?: string[]; + subject?: string | null; + from_header?: string | null; + to_header?: string | null; + cc_header?: string | null; + bcc_header?: string | null; + message_id?: string | null; + size_bytes?: number; + body_preview?: string | null; + attachment_count?: number; + folder?: string | null; + raw_eml?: string | null; + headers?: Record; + attachments?: Array<{ filename?: string | null; content_type?: string | null; size_bytes?: number }>; +}; + +export type MockMailboxMessageResponse = { + message: MockMailboxMessage; +}; diff --git a/webui/src/api/privacyRetention.ts b/webui/src/api/privacyRetention.ts index 6d5fc82..ab21466 100644 --- a/webui/src/api/privacyRetention.ts +++ b/webui/src/api/privacyRetention.ts @@ -1,5 +1,5 @@ import type { ApiSettings } from "../types"; -import { apiFetch } from "./client"; +import { apiFetch, apiPath } from "./client"; export type PrivacyRetentionPolicyFieldKey = | "store_raw_campaign_json" @@ -78,13 +78,6 @@ export type RetentionRunResponse = { }; }; -function retentionPolicyQuery(scopeId?: string | null): string { - const params = new URLSearchParams(); - if (scopeId) params.set("scope_id", scopeId); - const suffix = params.toString(); - return suffix ? `?${suffix}` : ""; -} - export function getPrivacyRetentionPolicy( settings: ApiSettings, scope: PrivacyRetentionPolicyScope, @@ -92,7 +85,7 @@ export function getPrivacyRetentionPolicy( ): Promise { return apiFetch( settings, - `/api/v1/admin/privacy-retention/policies/${encodeURIComponent(scope)}${retentionPolicyQuery(scopeId)}` + apiPath(`/api/v1/admin/privacy-retention/policies/${encodeURIComponent(scope)}`, { scope_id: scopeId }) ); } @@ -103,7 +96,7 @@ export function explainPrivacyRetentionPolicy( ): Promise { return apiFetch( settings, - `/api/v1/admin/privacy-retention/policies/${encodeURIComponent(scope)}/explain${retentionPolicyQuery(scopeId)}` + apiPath(`/api/v1/admin/privacy-retention/policies/${encodeURIComponent(scope)}/explain`, { scope_id: scopeId }) ); } @@ -116,7 +109,7 @@ export function updatePrivacyRetentionPolicy( ): Promise { return apiFetch( settings, - `/api/v1/admin/privacy-retention/policies/${encodeURIComponent(scope)}${retentionPolicyQuery(scopeId)}`, + apiPath(`/api/v1/admin/privacy-retention/policies/${encodeURIComponent(scope)}`, { scope_id: scopeId }), { method: "PUT", body: JSON.stringify({ policy, change_request_id: changeRequestId ?? null }) } ); } diff --git a/webui/src/components/CredentialPanel.tsx b/webui/src/components/CredentialPanel.tsx new file mode 100644 index 0000000..a052a69 --- /dev/null +++ b/webui/src/components/CredentialPanel.tsx @@ -0,0 +1,103 @@ +import type { ReactNode } from "react"; +import FormField from "./FormField"; +import PasswordField from "./PasswordField"; + +export type CredentialValues = { + username?: string | null; + password?: string | null; +}; + +export type CredentialFieldsProps = { + values: CredentialValues; + onChange: (patch: Partial) => void; + disabled?: boolean; + usernameDisabled?: boolean; + passwordDisabled?: boolean; + savedPassword?: boolean; + savedPasswordPlaceholder?: string; + heading?: ReactNode; + headingClassName?: string; + usernameLabel?: ReactNode; + passwordLabel?: ReactNode; + usernamePlaceholder?: string; + passwordPlaceholder?: string; + usernameAutoComplete?: string; + passwordAutoComplete?: string; + showUsername?: boolean; + showPassword?: boolean; +}; + +export type CredentialPanelProps = CredentialFieldsProps & { + className?: string; + gridClassName?: string; + children?: ReactNode; +}; + +export function CredentialFields({ + values, + onChange, + disabled = false, + usernameDisabled = disabled, + passwordDisabled = disabled, + savedPassword = false, + savedPasswordPlaceholder = "••••••••", + heading, + headingClassName = "credential-panel-heading", + usernameLabel = "i18n:govoplan-core.username.84c29015", + passwordLabel = "i18n:govoplan-core.password.8be3c943", + usernamePlaceholder, + passwordPlaceholder, + usernameAutoComplete = "username", + passwordAutoComplete = "new-password", + showUsername = true, + showPassword = true +}: CredentialFieldsProps) { + return ( + <> + {heading &&
{heading}
} + {showUsername && + + onChange({ username: event.target.value })} /> + + } + {showPassword && + + onChange({ password })} /> + + } + ); + +} + +export default function CredentialPanel({ + className = "", + gridClassName = "", + children, + ...fieldProps +}: CredentialPanelProps) { + return ( +
+
+ +
+ {children &&
{children}
} +
); + +} + +function stringValue(value: string | number | null | undefined): string { + if (value === null || value === undefined) return ""; + return String(value); +} diff --git a/webui/src/components/DisabledActionTooltip.tsx b/webui/src/components/DisabledActionTooltip.tsx new file mode 100644 index 0000000..9c38f7d --- /dev/null +++ b/webui/src/components/DisabledActionTooltip.tsx @@ -0,0 +1,21 @@ +import type { ReactNode } from "react"; +import HoverTooltip from "./HoverTooltip"; + +export type DisabledActionTooltipProps = { + reason?: ReactNode; + children: ReactNode; + className?: string; +}; + +export default function DisabledActionTooltip({ reason, children, className = "" }: DisabledActionTooltipProps) { + if (!reason) return <>{children}; + return ( + + {children} + + ); +} diff --git a/webui/src/components/HoverTooltip.tsx b/webui/src/components/HoverTooltip.tsx new file mode 100644 index 0000000..5526204 --- /dev/null +++ b/webui/src/components/HoverTooltip.tsx @@ -0,0 +1,228 @@ +import type { CSSProperties, ReactNode } from "react"; +import { useCallback, useEffect, useId, useLayoutEffect, useRef, useState } from "react"; +import { createPortal } from "react-dom"; +import { usePlatformLanguage } from "../i18n/LanguageContext"; + +export type HoverTooltipTone = "default" | "danger"; + +type TooltipPosition = { + top: number; + left: number; + arrowLeft: number; + placement: "top" | "bottom"; +}; + +export type HoverTooltipProps = { + content: ReactNode; + children: ReactNode; + className?: string; + tone?: HoverTooltipTone; + ariaLabel?: string; + triggerTabIndex?: number; + openDelayMs?: number; +}; + +const VIEWPORT_MARGIN = 12; +const TRIGGER_GAP = 10; + +const BASE_TOOLTIP_STYLE: CSSProperties = { + position: "fixed", + zIndex: 20000, + width: "max-content", + maxWidth: "min(320px, calc(100vw - 48px))", + borderRadius: 7, + boxShadow: "var(--shadow-popover)", + fontSize: 12, + lineHeight: 1.4, + padding: "9px 10px", + whiteSpace: "normal", + pointerEvents: "none" +}; + +function clamp(value: number, min: number, max: number) { + if (max < min) return min; + return Math.min(Math.max(value, min), max); +} + +function tooltipStyleForTone(tone: HoverTooltipTone): CSSProperties { + if (tone === "danger") { + return { + border: "1px solid var(--border-danger-soft)", + background: "var(--danger-muted-bg)", + color: "var(--danger-text-tooltip)", + fontWeight: 700 + }; + } + return { + border: "1px solid var(--line-dark)", + background: "var(--surface)", + color: "var(--text)", + fontWeight: 500 + }; +} + +function arrowStyleForTone(tone: HoverTooltipTone): Pick { + if (tone === "danger") { + return { + borderColor: "var(--border-danger-soft)", + background: "var(--danger-muted-bg)" + }; + } + return { + borderColor: "var(--line-dark)", + background: "var(--surface)" + }; +} + +export default function HoverTooltip({ + content, + children, + className = "", + tone = "default", + ariaLabel, + triggerTabIndex, + openDelayMs = 350 +}: HoverTooltipProps) { + const { translateText } = usePlatformLanguage(); + const tooltipId = useId(); + const triggerRef = useRef(null); + const tooltipRef = useRef(null); + const openTimerRef = useRef(null); + const [isOpen, setIsOpen] = useState(false); + const [position, setPosition] = useState(null); + + const clearOpenTimer = useCallback(() => { + if (openTimerRef.current !== null) { + window.clearTimeout(openTimerRef.current); + openTimerRef.current = null; + } + }, []); + + const openWithDelay = useCallback(() => { + if (!content) return; + clearOpenTimer(); + openTimerRef.current = window.setTimeout(() => { + openTimerRef.current = null; + setIsOpen(true); + }, openDelayMs); + }, [clearOpenTimer, content, openDelayMs]); + + const close = useCallback(() => { + clearOpenTimer(); + setIsOpen(false); + }, [clearOpenTimer]); + + const updatePosition = useCallback(() => { + const trigger = triggerRef.current; + const tooltip = tooltipRef.current; + if (!trigger || !tooltip) return; + + const triggerRect = trigger.getBoundingClientRect(); + const tooltipRect = tooltip.getBoundingClientRect(); + const triggerCenterX = triggerRect.left + triggerRect.width / 2; + const preferredLeft = triggerCenterX - tooltipRect.width / 2; + const left = clamp(preferredLeft, VIEWPORT_MARGIN, window.innerWidth - tooltipRect.width - VIEWPORT_MARGIN); + const topCandidate = triggerRect.top - tooltipRect.height - TRIGGER_GAP; + const hasRoomAbove = topCandidate >= VIEWPORT_MARGIN; + const bottomCandidate = triggerRect.bottom + TRIGGER_GAP; + const top = hasRoomAbove + ? topCandidate + : clamp(bottomCandidate, VIEWPORT_MARGIN, window.innerHeight - tooltipRect.height - VIEWPORT_MARGIN); + + setPosition({ + top, + left, + arrowLeft: clamp(triggerCenterX - left, 12, tooltipRect.width - 12), + placement: hasRoomAbove ? "top" : "bottom" + }); + }, []); + + useLayoutEffect(() => { + if (!isOpen) { + setPosition(null); + return; + } + + updatePosition(); + const frame = window.requestAnimationFrame(updatePosition); + return () => window.cancelAnimationFrame(frame); + }, [isOpen, updatePosition]); + + useEffect(() => { + if (!isOpen) return undefined; + + const handleScrollOrResize = () => updatePosition(); + const handleKeyDown = (event: KeyboardEvent) => { + if (event.key === "Escape") close(); + }; + + window.addEventListener("scroll", handleScrollOrResize, true); + window.addEventListener("resize", handleScrollOrResize); + window.addEventListener("keydown", handleKeyDown); + + return () => { + window.removeEventListener("scroll", handleScrollOrResize, true); + window.removeEventListener("resize", handleScrollOrResize); + window.removeEventListener("keydown", handleKeyDown); + }; + }, [close, isOpen, updatePosition]); + + useEffect(() => clearOpenTimer, [clearOpenTimer]); + + const translatedAriaLabel = ariaLabel ? translateText(ariaLabel) : undefined; + const tooltipStyle: CSSProperties = { + ...BASE_TOOLTIP_STYLE, + ...tooltipStyleForTone(tone), + top: position?.top ?? -9999, + left: position?.left ?? -9999, + opacity: position ? 1 : 0 + }; + const arrowColors = arrowStyleForTone(tone); + const arrowStyle: CSSProperties = position?.placement === "bottom" + ? { + position: "absolute", + left: position.arrowLeft, + top: 0, + width: 9, + height: 9, + borderLeft: "1px solid", + borderTop: "1px solid", + ...arrowColors, + transform: "translate(-50%, -5px) rotate(45deg)" + } + : { + position: "absolute", + left: position?.arrowLeft ?? 16, + top: "100%", + width: 9, + height: 9, + borderRight: "1px solid", + borderBottom: "1px solid", + ...arrowColors, + transform: "translate(-50%, -5px) rotate(45deg)" + }; + + return ( + <> + + {children} + + {isOpen && typeof document !== "undefined" && createPortal( + , + document.body + )} + + ); +} diff --git a/webui/src/components/email/EmailAddressInput.tsx b/webui/src/components/email/EmailAddressInput.tsx index 58612a6..9ae7c0b 100644 --- a/webui/src/components/email/EmailAddressInput.tsx +++ b/webui/src/components/email/EmailAddressInput.tsx @@ -1,4 +1,5 @@ -import { i18nMessage } from "../../i18n/LanguageContext";import { useEffect, useId, useMemo, useRef, useState } from "react"; +import { i18nMessage, usePlatformLanguage } from "../../i18n/LanguageContext"; +import { useEffect, useId, useMemo, useRef, useState } from "react"; import type { CSSProperties, KeyboardEvent } from "react"; import { createPortal } from "react-dom"; import Button from "../Button"; @@ -15,6 +16,7 @@ type EmailAddressInputProps = { value: MailboxAddress[]; onChange?: (value: MailboxAddress[]) => void; onAddressAdded?: (address: MailboxAddress) => void; + onSuggestionQueryChange?: (query: string) => void; suggestions?: MailboxAddress[]; allowMultiple?: boolean; clearOnAdd?: boolean; @@ -31,6 +33,7 @@ export default function EmailAddressInput({ value, onChange, onAddressAdded, + onSuggestionQueryChange, suggestions = [], allowMultiple = true, clearOnAdd = false, @@ -42,6 +45,7 @@ export default function EmailAddressInput({ compact = false, showAddButton }: EmailAddressInputProps) { + const { translateText } = usePlatformLanguage(); const inputId = useId(); const normalizedValue = useMemo(() => dedupeAddresses(value), [value]); const normalizedSuggestions = useMemo(() => dedupeAddresses(suggestions), [suggestions]); @@ -51,9 +55,18 @@ export default function EmailAddressInput({ const [dialogEmail, setDialogEmail] = useState(""); const [error, setError] = useState(""); const [popoverStyle, setPopoverStyle] = useState({}); + const lastSuggestionQueryRef = useRef(null); const addButtonRef = useRef(null); const canUseAddButton = showAddButton ?? allowMultiple; + useEffect(() => { + const query = entryText.trim(); + if (query === "" && lastSuggestionQueryRef.current === null) return; + if (query === lastSuggestionQueryRef.current) return; + lastSuggestionQueryRef.current = query; + onSuggestionQueryChange?.(query); + }, [entryText, onSuggestionQueryChange]); + const filteredSuggestions = useMemo(() => { const query = entryText.trim().toLowerCase(); if (!query) return normalizedSuggestions.slice(0, 6); @@ -168,18 +181,18 @@ export default function EmailAddressInput({ if (event.key === "Escape") setDialogOpen(false); }}> -

i18n:govoplan-core.add_address.a71075c4

+

{translateText("i18n:govoplan-core.add_address.a71075c4")}

- - + +
, document.body @@ -189,11 +202,11 @@ export default function EmailAddressInput({
- {normalizedValue.length === 0 && !entryText && {emptyText}} + {normalizedValue.length === 0 && !entryText && {translateText(emptyText)}} {normalizedValue.map((address) => { const valid = isValidEmailAddress(address.email); return ( - + {addressDisplayName(address)} {address.name && {address.email}} {!disabled && @@ -217,11 +230,11 @@ export default function EmailAddressInput({ setError(""); }} onKeyDown={handleTextKeyDown} - placeholder={i18nMessage("i18n:govoplan-core.value_value.27085f09", { value0: namePlaceholder, value1: emailPlaceholder })} - aria-label="i18n:govoplan-core.type_a_name_and_email_address_then_press_enter.7c8d43f0" /> + placeholder={translateText(i18nMessage("i18n:govoplan-core.value_value.27085f09", { value0: translateText(namePlaceholder), value1: emailPlaceholder }))} + aria-label={translateText("i18n:govoplan-core.type_a_name_and_email_address_then_press_enter.7c8d43f0")} /> {canUseAddButton && - } @@ -230,7 +243,7 @@ export default function EmailAddressInput({
{!disabled && filteredSuggestions.length > 0 && entryText.trim() && -
+
{filteredSuggestions.map((item) =>
} - {error &&

{error}

} + {error &&

{translateText(error)}

} {addressDialog}
); diff --git a/webui/src/components/help/InlineHelp.tsx b/webui/src/components/help/InlineHelp.tsx index a470572..0b00720 100644 --- a/webui/src/components/help/InlineHelp.tsx +++ b/webui/src/components/help/InlineHelp.tsx @@ -1,188 +1,20 @@ -import type { CSSProperties, ReactNode } from "react"; -import { useCallback, useEffect, useId, useLayoutEffect, useRef, useState } from "react"; -import { createPortal } from "react-dom"; -import { usePlatformLanguage } from "../../i18n/LanguageContext"; +import type { ReactNode } from "react"; +import HoverTooltip from "../HoverTooltip"; type InlineHelpProps = { children: ReactNode; className?: string; }; -type TooltipPosition = { - top: number; - left: number; - arrowLeft: number; - placement: "top" | "bottom"; -}; - -const OPEN_DELAY_MS = 350; -const VIEWPORT_MARGIN = 12; -const TRIGGER_GAP = 10; - -const tooltipBaseStyle: CSSProperties = { - position: "fixed", - zIndex: 20000, - width: "max-content", - maxWidth: "min(320px, calc(100vw - 48px))", - border: "1px solid var(--line-dark)", - borderRadius: 7, - background: "var(--surface)", - boxShadow: "var(--shadow-popover)", - color: "var(--text)", - fontSize: 12, - fontWeight: 500, - lineHeight: 1.4, - padding: "9px 10px", - whiteSpace: "normal", - pointerEvents: "none" -}; - -function clamp(value: number, min: number, max: number) { - if (max < min) return min; - return Math.min(Math.max(value, min), max); -} - export default function InlineHelp({ children, className = "" }: InlineHelpProps) { - const { translateText } = usePlatformLanguage(); - const tooltipId = useId(); - const triggerRef = useRef(null); - const tooltipRef = useRef(null); - const openTimerRef = useRef(null); - const [isOpen, setIsOpen] = useState(false); - const [position, setPosition] = useState(null); - - const clearOpenTimer = useCallback(() => { - if (openTimerRef.current !== null) { - window.clearTimeout(openTimerRef.current); - openTimerRef.current = null; - } - }, []); - - const openWithDelay = useCallback(() => { - clearOpenTimer(); - openTimerRef.current = window.setTimeout(() => { - openTimerRef.current = null; - setIsOpen(true); - }, OPEN_DELAY_MS); - }, [clearOpenTimer]); - - const close = useCallback(() => { - clearOpenTimer(); - setIsOpen(false); - }, [clearOpenTimer]); - - const updatePosition = useCallback(() => { - const trigger = triggerRef.current; - const tooltip = tooltipRef.current; - if (!trigger || !tooltip) return; - - const triggerRect = trigger.getBoundingClientRect(); - const tooltipRect = tooltip.getBoundingClientRect(); - const triggerCenterX = triggerRect.left + triggerRect.width / 2; - const preferredLeft = triggerCenterX - tooltipRect.width / 2; - const left = clamp(preferredLeft, VIEWPORT_MARGIN, window.innerWidth - tooltipRect.width - VIEWPORT_MARGIN); - const topCandidate = triggerRect.top - tooltipRect.height - TRIGGER_GAP; - const hasRoomAbove = topCandidate >= VIEWPORT_MARGIN; - const bottomCandidate = triggerRect.bottom + TRIGGER_GAP; - const top = hasRoomAbove ? - topCandidate : - clamp(bottomCandidate, VIEWPORT_MARGIN, window.innerHeight - tooltipRect.height - VIEWPORT_MARGIN); - - setPosition({ - top, - left, - arrowLeft: clamp(triggerCenterX - left, 12, tooltipRect.width - 12), - placement: hasRoomAbove ? "top" : "bottom" - }); - }, []); - - useLayoutEffect(() => { - if (!isOpen) { - setPosition(null); - return; - } - - updatePosition(); - const frame = window.requestAnimationFrame(updatePosition); - return () => window.cancelAnimationFrame(frame); - }, [isOpen, updatePosition]); - - useEffect(() => { - if (!isOpen) return undefined; - - const handleScrollOrResize = () => updatePosition(); - const handleKeyDown = (event: KeyboardEvent) => { - if (event.key === "Escape") close(); - }; - - window.addEventListener("scroll", handleScrollOrResize, true); - window.addEventListener("resize", handleScrollOrResize); - window.addEventListener("keydown", handleKeyDown); - - return () => { - window.removeEventListener("scroll", handleScrollOrResize, true); - window.removeEventListener("resize", handleScrollOrResize); - window.removeEventListener("keydown", handleKeyDown); - }; - }, [close, isOpen, updatePosition]); - - useEffect(() => clearOpenTimer, [clearOpenTimer]); - if (!children) return null; - - const tooltipStyle: CSSProperties = { - ...tooltipBaseStyle, - top: position?.top ?? -9999, - left: position?.left ?? -9999, - opacity: position ? 1 : 0 - }; - - const arrowStyle: CSSProperties = position?.placement === "bottom" ? - { - position: "absolute", - left: position.arrowLeft, - top: 0, - width: 9, - height: 9, - borderLeft: "1px solid var(--line-dark)", - borderTop: "1px solid var(--line-dark)", - background: "var(--surface)", - transform: "translate(-50%, -5px) rotate(45deg)" - } : - { - position: "absolute", - left: position?.arrowLeft ?? 16, - top: "100%", - width: 9, - height: 9, - borderRight: "1px solid var(--line-dark)", - borderBottom: "1px solid var(--line-dark)", - background: "var(--surface)", - transform: "translate(-50%, -5px) rotate(45deg)" - }; - return ( - <> - - - - - {isOpen && createPortal( - , - document.body - )} - ); - + + + + ); } diff --git a/webui/src/components/mail/MailServerSettingsPanel.tsx b/webui/src/components/mail/MailServerSettingsPanel.tsx index 89cf707..79f4744 100644 --- a/webui/src/components/mail/MailServerSettingsPanel.tsx +++ b/webui/src/components/mail/MailServerSettingsPanel.tsx @@ -1,8 +1,8 @@ -import { useState } from "react"; +import { useEffect, useState } from "react"; import Button from "../Button"; +import { CredentialFields } from "../CredentialPanel"; import DismissibleAlert from "../DismissibleAlert"; import FormField from "../FormField"; -import PasswordField from "../PasswordField"; import SegmentedControl from "../SegmentedControl"; import ToggleSwitch from "../ToggleSwitch"; @@ -48,6 +48,9 @@ export type MailServerFolderLookupResult = { details?: Record | null; }; +export type MailServerSettingsSection = "smtp" | "imap" | "advanced"; +export type MailServerSettingsMode = "all" | "server" | "credentials"; + export type MailServerSettingsPanelProps = { smtp: MailServerSmtpSettings; imap: MailServerImapSettings; @@ -97,12 +100,13 @@ export type MailServerSettingsPanelProps = { className?: string; floatingResults?: boolean; initialSection?: MailServerSettingsSection; + visibleSections?: readonly MailServerSettingsSection[]; + mode?: MailServerSettingsMode; }; export const mailServerSecurityOptions = ["plain", "tls", "starttls"] as const; export type MailServerSecurityOption = typeof mailServerSecurityOptions[number]; const securityOptions = mailServerSecurityOptions; -type MailServerSettingsSection = "smtp" | "imap" | "advanced"; export function defaultSmtpPort(security: MailServerSecurity | null | undefined): number { if (security === "tls") return 465; @@ -234,7 +238,9 @@ export default function MailServerSettingsPanel({ disabled = false, className = "", floatingResults = false, - initialSection = "smtp" + initialSection = "smtp", + visibleSections, + mode = "all" }: MailServerSettingsPanelProps) { const smtpFieldsDisabled = disabled || smtpDisabled; const smtpCredentialFieldsDisabled = disabled || smtpCredentialDisabled; @@ -254,11 +260,29 @@ export default function MailServerSettingsPanel({ const appendTargetHelp = append ? "i18n:govoplan-core.folder_for_sent_message_copies_leave_as_auto_unl.a62586e9" : "i18n:govoplan-core.folder_used_when_this_imap_account_is_used_for_s.08503f5e"; - const [activeSection, setActiveSection] = useState(initialSection); - const sections: {id: MailServerSettingsSection;label: string;}[] = [ + const allSections: {id: MailServerSettingsSection;label: string;}[] = [ { id: "smtp", label: "i18n:govoplan-core.smtp.efff9cca" }, { id: "imap", label: "i18n:govoplan-core.imap.271f9ef2" }, { id: "advanced", label: "i18n:govoplan-core.advanced.4d064726" }]; + const visibleSectionSet = new Set(visibleSections ?? allSections.map((section) => section.id)); + const sections = allSections.filter((section) => visibleSectionSet.has(section.id)); + const fallbackSection = sections[0]?.id ?? "smtp"; + const resolvedInitialSection = sections.some((section) => section.id === initialSection) ? initialSection : fallbackSection; + const sectionKey = sections.map((section) => section.id).join("|"); + const [activeSection, setActiveSection] = useState(resolvedInitialSection); + const showSectionSwitcher = sections.length > 1; + const showServerFields = mode !== "credentials"; + const showCredentialFields = mode !== "server"; + + useEffect(() => { + if (!sections.some((section) => section.id === activeSection)) { + setActiveSection(resolvedInitialSection); + return; + } + if (initialSection !== activeSection && sections.some((section) => section.id === initialSection)) { + setActiveSection(initialSection); + } + }, [activeSection, initialSection, resolvedInitialSection, sectionKey]); function patchSmtpSecurity(security: MailServerSecurity) { @@ -283,6 +307,7 @@ export default function MailServerSettingsPanel({ return (
+ {showSectionSwitcher && + }
{activeSection === "smtp" &&
+ {showServerFields && + <>
i18n:govoplan-core.server.cb0cb170
onSmtpChange({ host: event.target.value })} /> onSmtpChange({ port: event.target.value })} /> -
i18n:govoplan-core.credentials.dd097a22
- patchSmtpCredentials({ username: event.target.value })} /> - - + } + {showCredentialFields && + patchSmtpCredentials({ password })} /> - - + savedPassword={smtpPasswordSaved} + savedPasswordPlaceholder={smtpSavedPasswordPlaceholder} + heading="i18n:govoplan-core.credentials.dd097a22" + headingClassName="mail-server-field-heading" /> + }
{onTestSmtp &&
@@ -325,22 +353,24 @@ export default function MailServerSettingsPanel({ {activeSection === "imap" &&
+ {showServerFields && + <>
i18n:govoplan-core.server.cb0cb170
onImapChange({ host: event.target.value })} /> onImapChange({ port: event.target.value })} /> -
i18n:govoplan-core.credentials.dd097a22
- patchImapCredentials({ username: event.target.value })} /> - - + } + {showCredentialFields && + patchImapCredentials({ password })} /> - - + savedPassword={imapPasswordSaved} + savedPasswordPlaceholder={imapSavedPasswordPlaceholder} + heading="i18n:govoplan-core.credentials.dd097a22" + headingClassName="mail-server-field-heading" /> + }
{onTestImap &&
diff --git a/webui/src/components/table/DataGrid.tsx b/webui/src/components/table/DataGrid.tsx index 8cb1d15..6a5559a 100644 --- a/webui/src/components/table/DataGrid.tsx +++ b/webui/src/components/table/DataGrid.tsx @@ -150,7 +150,7 @@ type ColumnResizeState = { behavior: DataGridResizeBehavior; }; -const STORAGE_PREFIX = "multimailer.datagrid."; +const STORAGE_PREFIX = "govoplan.datagrid."; const FILTER_POPOVER_WIDTH = 320; const FILTER_POPOVER_MARGIN = 12; const MIN_HEADER_LABEL_WIDTH = 72; diff --git a/webui/src/features/settings/SettingsPage.tsx b/webui/src/features/settings/SettingsPage.tsx index bc2462d..26842ef 100644 --- a/webui/src/features/settings/SettingsPage.tsx +++ b/webui/src/features/settings/SettingsPage.tsx @@ -1,6 +1,6 @@ import { useEffect, useMemo, useState } from "react"; import { useSearchParams } from "react-router-dom"; -import type { ApiSettings, AuthInfo, FilesConnectorsUiCapability, MailProfilesUiCapability, UserUiPreferences, UserUiTheme } from "../../types"; +import type { ApiSettings, AuthInfo, AuthUpdate, FilesConnectorsUiCapability, MailProfilesUiCapability, SettingsSectionContribution, SettingsSectionsUiCapability, UserUiPreferences, UserUiTheme } from "../../types"; import Card from "../../components/Card"; import FormField from "../../components/FormField"; import PasswordField from "../../components/PasswordField"; @@ -8,15 +8,16 @@ import Button from "../../components/Button"; import PageTitle from "../../components/PageTitle"; import ToggleSwitch from "../../components/ToggleSwitch"; import { apiFetch } from "../../api/client"; -import { updateProfile } from "../../api/auth"; +import { fetchAuthProfile, fetchAuthRoles, updateProfile } from "../../api/auth"; import ModuleSubnav, { type ModuleSubnavGroup } from "../../layout/ModuleSubnav"; import DismissibleAlert from "../../components/DismissibleAlert"; +import SegmentedControl from "../../components/SegmentedControl"; import { useUnsavedChanges, useUnsavedDraftGuard } from "../../components/UnsavedChangesGuard"; -import { usePlatformUiCapability } from "../../platform/ModuleContext"; +import { usePlatformUiCapabilities, usePlatformUiCapability } from "../../platform/ModuleContext"; import { hasAnyScope, hasScope } from "../../utils/permissions"; import { usePlatformLanguage } from "../../i18n/LanguageContext"; -type SettingsSection = "profile" | "mail-profiles" | "file-connectors" | "interface" | "workspace" | "local-connection" | "notifications"; +type SettingsSection = "profile" | "mail-profiles" | "file-connectors" | "interface" | "workspace" | "local-connection" | string; const DEFAULT_UI_PREFERENCES: UserUiPreferences = { compact_tables: false, @@ -32,8 +33,8 @@ const UI_THEME_OPTIONS: Array<{ value: UserUiTheme; label: string }> = [ { value: "dark", label: "i18n:govoplan-core.dark_theme.164a90d9" } ]; -function settingsGroups(canUseMailProfiles: boolean, canUseFileConnectors: boolean): ModuleSubnavGroup[] { - return [ +function settingsGroups(canUseMailProfiles: boolean, canUseFileConnectors: boolean, contributedSections: SettingsSectionContribution[]): ModuleSubnavGroup[] { + const groups: ModuleSubnavGroup[] = [ { title: "i18n:govoplan-core.account.f967543b", items: [ @@ -47,11 +48,29 @@ function settingsGroups(canUseMailProfiles: boolean, canUseFileConnectors: boole items: [ { id: "interface", label: "i18n:govoplan-core.interface.7b4db7ef" }, { id: "workspace", label: "i18n:govoplan-core.workspace.4ca0a75c" }, - { id: "local-connection", label: "i18n:govoplan-core.local_connection.42fba65a" }, - { id: "notifications", label: "i18n:govoplan-core.notifications.753a22b2" }] + { id: "local-connection", label: "i18n:govoplan-core.local_connection.42fba65a" }] }]; + for (const section of contributedSections) { + const groupId = section.group || "ui"; + const title = section.groupTitle || (groupId === "account" ? "i18n:govoplan-core.account.f967543b" : groupId === "ui" ? "i18n:govoplan-core.ui_settings.9e9cc5ea" : groupId); + let group = groups.find((item) => item.title === title); + if (!group) { + group = { title, items: [] }; + groups.push(group); + } + group.items.push({ id: section.id, label: section.label }); + } + + for (const group of groups) { + group.items.sort((left, right) => { + const leftId = "id" in left ? left.id : ""; + const rightId = "id" in right ? right.id : ""; + return (sectionOrder(leftId, contributedSections) - sectionOrder(rightId, contributedSections)) || left.label.localeCompare(right.label); + }); + } + return groups; } export default function SettingsPage({ @@ -64,19 +83,26 @@ export default function SettingsPage({ -}: {settings: ApiSettings;auth: AuthInfo;onSettingsChange: (settings: ApiSettings) => void;onAuthChange: (auth: AuthInfo | null, accessToken?: string) => void;}) { +}: {settings: ApiSettings;auth: AuthInfo;onSettingsChange: (settings: ApiSettings) => void;onAuthChange: (auth: AuthUpdate | null, accessToken?: string) => void;}) { const [searchParams, setSearchParams] = useSearchParams(); const { requestNavigation } = useUnsavedChanges(); const mailProfilesUi = usePlatformUiCapability("mail.profiles"); const fileConnectorsUi = usePlatformUiCapability("files.connectors"); + const settingsSectionCapabilities = usePlatformUiCapabilities("settings.sections"); const { language, languageLabel, selectableLanguages, availableLanguages, enabledLanguages, setLanguage } = usePlatformLanguage(); const MailProfileScopeManager = mailProfilesUi?.MailProfileScopeManager ?? null; const FileConnectorScopeManager = fileConnectorsUi?.FileConnectorScopeManager ?? null; const canUseMailProfiles = Boolean(MailProfileScopeManager) && hasAnyScope(auth, ["mail_servers:read", "mail_servers:write", "mail_servers:manage_credentials", "admin:policies:read", "admin:policies:write"]); const canUseFileConnectors = Boolean(FileConnectorScopeManager) && hasAnyScope(auth, ["files:file:read", "files:file:admin", "admin:settings:read", "admin:settings:write"]); - const settingsSubnav = useMemo(() => settingsGroups(canUseMailProfiles, canUseFileConnectors), [canUseFileConnectors, canUseMailProfiles]); - const requestedSection = searchParams.get("section") as SettingsSection | null; - const active: SettingsSection = settingsSectionAvailable(settingsSubnav, requestedSection) ? requestedSection : "interface"; + const contributedSections = useMemo( + () => settingsSectionCapabilities.flatMap((capability) => capability.sections ?? []).filter((section) => canUseSettingsContribution(auth, section)), + [auth, settingsSectionCapabilities] + ); + const settingsSubnav = useMemo(() => settingsGroups(canUseMailProfiles, canUseFileConnectors, contributedSections), [canUseFileConnectors, canUseMailProfiles, contributedSections]); + const availableSectionIds = useMemo(() => new Set(settingsSubnav.flatMap((group) => group.items.flatMap((item) => "id" in item ? [item.id] : []))), [settingsSubnav]); + const requestedSection = searchParams.get("section"); + const active: SettingsSection = settingsSectionAvailable(availableSectionIds, requestedSection) ? requestedSection : "interface"; + const activeContributedSection = contributedSections.find((section) => section.id === active) ?? null; const currentUiPreferences = normalizeUiPreferences(auth.user.ui_preferences); const [testing, setTesting] = useState(false); const [testResult, setTestResult] = useState(""); @@ -118,12 +144,30 @@ export default function SettingsPage({ }); useEffect(() => { - if (requestedSection && !settingsSectionAvailable(settingsSubnav, requestedSection)) { + if (requestedSection && !settingsSectionAvailable(availableSectionIds, requestedSection)) { const next = new URLSearchParams(searchParams); next.set("section", "interface"); setSearchParams(next, { replace: true }); } - }, [requestedSection, searchParams, setSearchParams, settingsSubnav]); + }, [availableSectionIds, requestedSection, searchParams, setSearchParams]); + + useEffect(() => { + if (auth.profile_loaded) return; + let cancelled = false; + fetchAuthProfile(settings) + .then((next) => {if (!cancelled) onAuthChange(next);}) + .catch(() => undefined); + return () => {cancelled = true;}; + }, [auth.profile_loaded, auth.user.id, auth.active_tenant?.id, auth.tenant.id, settings.accessToken, settings.apiBaseUrl, settings.apiKey]); + + useEffect(() => { + if (auth.roles_loaded) return; + let cancelled = false; + fetchAuthRoles(settings) + .then((next) => {if (!cancelled) onAuthChange(next);}) + .catch(() => undefined); + return () => {cancelled = true;}; + }, [auth.roles_loaded, auth.user.id, auth.active_tenant?.id, auth.tenant.id, settings.accessToken, settings.apiBaseUrl, settings.apiKey]); useEffect(() => { setProfileName(auth.user.display_name || ""); @@ -329,12 +373,16 @@ export default function SettingsPage({ - + ({ id: item.value, label: item.label }))} + value={theme} + onChange={setTheme} + role="group" + size="equal" + width="fill" + ariaLabel="i18n:govoplan-core.theme.a797e309" /> +
i18n:govoplan-core.theme.a797e309
{themeLabel(theme)}
i18n:govoplan-core.accent_color.e49578ed
i18n:govoplan-core.default_brand_accent.606ae693
@@ -413,24 +461,15 @@ export default function SettingsPage({
} - {active === "notifications" && -
- -

i18n:govoplan-core.prepared_for_later_personal_notification_prefere.3fe73f86

-
- i18n:govoplan-core.in_app_completion_notices.b68f2f4c - i18n:govoplan-core.email_summary_preferences.b6c1dfb1 - i18n:govoplan-core.failure_and_warning_alerts.939d21c2 -
- - -
- i18n:govoplan-core.mute_non_critical_banners.27b23a4a - i18n:govoplan-core.batch_repetitive_notices.cc893559 - i18n:govoplan-core.keep_validation_and_send_warnings_visible.9d6e0cf4 -
- -
+ {activeContributedSection && + activeContributedSection.render({ + settings, + auth, + onAuthChange, + activeSection: active, + availableSections: availableSectionIds, + selectSection + }) }
@@ -439,8 +478,30 @@ export default function SettingsPage({ } -function settingsSectionAvailable(groups: ModuleSubnavGroup[], section: SettingsSection | null | undefined): section is SettingsSection { - return Boolean(section && groups.some((group) => group.items.some((item) => "id" in item && item.id === section))); +function settingsSectionAvailable(sections: ReadonlySet, section: string | null | undefined): section is SettingsSection { + return Boolean(section && sections.has(section)); +} + +function canUseSettingsContribution(auth: AuthInfo, section: SettingsSectionContribution): boolean { + if (section.allOf?.some((scope) => !hasScope(auth, scope))) { + return false; + } + if (section.anyOf && !hasAnyScope(auth, section.anyOf)) { + return false; + } + return true; +} + +function sectionOrder(sectionId: string, contributedSections: SettingsSectionContribution[]): number { + const builtInOrder: Record = { + profile: 10, + "mail-profiles": 20, + "file-connectors": 30, + interface: 10, + workspace: 20, + "local-connection": 30 + }; + return builtInOrder[sectionId] ?? contributedSections.find((section) => section.id === sectionId)?.order ?? 100; } function normalizeUiPreferences(value: Partial | null | undefined): UserUiPreferences { @@ -457,3 +518,24 @@ function normalizeUiPreferences(value: Partial | null | undef function themeLabel(value: UserUiTheme): string { return UI_THEME_OPTIONS.find((item) => item.value === value)?.label ?? UI_THEME_OPTIONS[0].label; } + +function ThemePreview({ theme }: {theme: UserUiTheme;}) { + const variants: UserUiTheme[] = theme === "system" ? ["light", "dark"] : [theme]; + return ( +
+ {variants.map((variant) => +
+
+ {themeLabel(variant)} + +
+
+ GovOPlaN + + +
+
+ )} +
); + +} diff --git a/webui/src/index.ts b/webui/src/index.ts index fdaa121..de4af9c 100644 --- a/webui/src/index.ts +++ b/webui/src/index.ts @@ -3,6 +3,26 @@ export * from "./types"; export * from "./api/client"; export * from "./api/auth"; export * from "./api/platform"; +export * from "./api/adminCommon"; +export { mailProfilePatternKeys, mailProfilePolicyLimitKeys } from "./api/mailContracts"; +export type { + MailConnectionTestResponse, + MailImapFolderListResponse, + MailImapFolderResponse, + MailImapTestPayload, + MailPolicySourceStep, + MailProfilePatternRules, + MailProfilePolicyLimitKey, + MailProfilePolicyLimitPermissions, + MailProfilePolicyResponse, + MailServerProfileCredentialsPayload, + MailServerProfileListResponse, + MailServerProfilePayload, + MailSmtpTestPayload, + MailTransportCredentialsPayload, + MockMailboxMessage, + MockMailboxMessageResponse +} from "./api/mailContracts"; export * from "./api/privacyRetention"; export * from "./platform/modules"; export * from "./platform/ModuleContext"; @@ -32,8 +52,12 @@ export { default as ConfirmDialog } from "./components/ConfirmDialog"; export { default as ConnectionTree } from "./components/ConnectionTree"; export type { ConnectionTreeColumn, ConnectionTreeProps } from "./components/ConnectionTree"; export { default as ColorPickerField } from "./components/ColorPickerField"; +export { default as CredentialPanel, CredentialFields } from "./components/CredentialPanel"; +export type { CredentialFieldsProps, CredentialPanelProps, CredentialValues } from "./components/CredentialPanel"; export { default as DateField, TimeField, DateTimeField } from "./components/DateTimeField"; export { default as Dialog } from "./components/Dialog"; +export { default as DisabledActionTooltip } from "./components/DisabledActionTooltip"; +export type { DisabledActionTooltipProps } from "./components/DisabledActionTooltip"; export { default as DismissibleAlert } from "./components/DismissibleAlert"; export { default as EffectivePolicyBlock, EffectivePolicyValue } from "./components/EffectivePolicyBlock"; export type { EffectivePolicyBlockProps } from "./components/EffectivePolicyBlock"; @@ -44,6 +68,8 @@ export { default as GuidedConfigDialog } from "./components/GuidedConfigDialog"; export type { GuidedConfigDialogProps } from "./components/GuidedConfigDialog"; export { default as GuidedReviewList } from "./components/GuidedReviewList"; export type { GuidedReviewItem } from "./components/GuidedReviewList"; +export { default as HoverTooltip } from "./components/HoverTooltip"; +export type { HoverTooltipProps, HoverTooltipTone } from "./components/HoverTooltip"; export { default as LoadingFrame } from "./components/LoadingFrame"; export { default as LoadingIndicator } from "./components/LoadingIndicator"; export { default as ExplorerTree } from "./components/ExplorerTree"; @@ -71,7 +97,7 @@ export type { UnsavedDraftGuardOptions } from "./components/UnsavedChangesGuard" export type { UnsavedChangesRegistration, UnsavedNavigationAction } from "./components/UnsavedChangesGuard"; export { default as EmailAddressInput } from "./components/email/EmailAddressInput"; export { default as MailServerSettingsPanel, MailServerActionResult, MailServerFolderLookupResultView, defaultImapPort, defaultSmtpPort, hasMailImapSettings, mailImapSettingsPayload, mailNumberOrDefault, mailNumberOrNull, mailServerSecurityOptions, mailSmtpSettingsPayload, mailTextOrNull, mailTransportCredentialsPayload, mailTransportCredentialsPayloadFromRecords, normalizeMailServerSecurity } from "./components/mail/MailServerSettingsPanel"; -export type { MailServerConnectionTestResult, MailServerCredentialSettings, MailServerFolderLookupResult, MailServerImapSettings, MailServerSecurity, MailServerSecurityOption, MailServerSettingsPanelProps, MailServerSmtpSettings } from "./components/mail/MailServerSettingsPanel"; +export type { MailServerConnectionTestResult, MailServerCredentialSettings, MailServerFolderLookupResult, MailServerImapSettings, MailServerSecurity, MailServerSecurityOption, MailServerSettingsMode, MailServerSettingsPanelProps, MailServerSettingsSection, MailServerSmtpSettings } from "./components/mail/MailServerSettingsPanel"; export { default as FieldLabel } from "./components/help/FieldLabel"; export { default as InlineHelp } from "./components/help/InlineHelp"; export { default as DataGrid, DataGridEmptyAction, DataGridRowActions } from "./components/table/DataGrid"; diff --git a/webui/src/layout/AppShell.tsx b/webui/src/layout/AppShell.tsx index d7181c6..cf3f309 100644 --- a/webui/src/layout/AppShell.tsx +++ b/webui/src/layout/AppShell.tsx @@ -1,5 +1,5 @@ import { useLocation } from "react-router-dom"; -import type { ApiSettings, AuthInfo, PlatformNavItem } from "../types"; +import type { ApiSettings, AuthInfo, AuthUpdate, PlatformNavItem } from "../types"; import IconRail from "./IconRail"; import Titlebar from "./Titlebar"; import BreadcrumbBar from "./BreadcrumbBar"; @@ -9,7 +9,7 @@ type Props = { settings: ApiSettings; auth: AuthInfo | null; onSettingsChange: (settings: ApiSettings) => void; - onAuthChange: (auth: AuthInfo | null, accessToken?: string) => void; + onAuthChange: (auth: AuthUpdate | null, accessToken?: string) => void; publicMode?: boolean; navItems?: PlatformNavItem[]; maintenanceMode?: { enabled: boolean; message?: string | null }; diff --git a/webui/src/layout/HelpMenu.tsx b/webui/src/layout/HelpMenu.tsx index 7c8ca85..d370af9 100644 --- a/webui/src/layout/HelpMenu.tsx +++ b/webui/src/layout/HelpMenu.tsx @@ -10,6 +10,8 @@ import type { PlatformWebModule } from "../types"; import { helpContextForPathname, helpQueryForContext, type HelpContext } from "../utils/helpContext"; import { usePlatformLanguage } from "../i18n/LanguageContext"; +const EXTERNAL_DOCS_BASE_URL = "https://govoplan.add-ideas.de"; + export default function HelpMenu() { const [open, setOpen] = useState(false); const [aboutOpen, setAboutOpen] = useState(false); @@ -50,8 +52,11 @@ export default function HelpMenu() { } function openDocs(type: "user" | "admin") { - if (!docsAvailable) return; setOpen(false); + if (!docsAvailable) { + window.open(externalDocsUrl(type, helpContext), "_blank", "noopener,noreferrer"); + return; + } const params = new URLSearchParams({ type }); params.set("context", helpContext.id); navigate(`/docs?${params.toString()}`); @@ -79,10 +84,10 @@ export default function HelpMenu() { {translateText("i18n:govoplan-core.help.c47ae153")} i18n:govoplan-core.f1.88bfad9c
- -
@@ -96,6 +101,11 @@ export default function HelpMenu() { } +function externalDocsUrl(type: "user" | "admin", context: HelpContext): string { + const params = new URLSearchParams({ type, context: context.id }); + return `${EXTERNAL_DOCS_BASE_URL}/?${params.toString()}`; +} + function ContextHelpModal({ context, onClose }: {context: HelpContext;onClose: () => void;}) { const { translateText } = usePlatformLanguage(); return ( diff --git a/webui/src/layout/IconRail.tsx b/webui/src/layout/IconRail.tsx index 5697336..e68cce2 100644 --- a/webui/src/layout/IconRail.tsx +++ b/webui/src/layout/IconRail.tsx @@ -1,4 +1,4 @@ -import { Settings } from "lucide-react"; +import { PanelLeftClose, PanelLeftOpen, Settings } from "lucide-react"; import { NavLink, useLocation } from "react-router-dom"; import { useEffect, useMemo, useState, type MouseEvent } from "react"; import type { AuthInfo, PlatformNavItem } from "../types"; @@ -7,6 +7,7 @@ import { usePlatformLanguage } from "../i18n/LanguageContext"; import { useGuardedNavigate } from "../components/UnsavedChangesGuard"; const MODULE_NAV_STORAGE_KEY = "govoplan.lastModuleNav"; +const RAIL_EXPANDED_STORAGE_KEY = "govoplan.iconRailExpanded"; function visibleNavItems(auth: AuthInfo | null | undefined, navItems: PlatformNavItem[]): PlatformNavItem[] { return [...navItems]. @@ -22,14 +23,11 @@ export default function IconRail({ compact = false, auth = null, navItems = [] - - - - }: {compact?: boolean;auth?: AuthInfo | null;navItems?: PlatformNavItem[];}) { const location = useLocation(); const items = visibleNavItems(auth, navItems); const [rememberedTargets, setRememberedTargets] = useState>(() => loadRememberedTargets()); + const [expanded, setExpanded] = useState(() => loadRailExpanded()); const topLevelItems = useMemo(() => items.map((item) => item.to), [items]); const { translateText } = usePlatformLanguage(); const navigate = useGuardedNavigate(); @@ -46,15 +44,27 @@ export default function IconRail({ }); }, [location.hash, location.pathname, location.search, topLevelItems]); + function toggleExpanded() { + setExpanded((current) => { + const next = !current; + saveRailExpanded(next); + return next; + }); + } + function handleNavClick(event: MouseEvent, target: string) { if (event.defaultPrevented || event.button !== 0 || event.metaKey || event.altKey || event.ctrlKey || event.shiftKey) return; event.preventDefault(); navigate(target); } + const railExpanded = !compact && expanded; + return ( -