diff --git a/.gitignore b/.gitignore index 3c46444..32e22c2 100644 --- a/.gitignore +++ b/.gitignore @@ -326,3 +326,11 @@ cython_debug/ # Built Visual Studio Code Extensions *.vsix +*.db + +# GovOPlaN local runtime state +runtime/ + +# GovOPlaN WebUI test output +webui/.module-test-build/ +webui/.component-test-build/ diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..9b35991 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,48 @@ +# GovOPlaN Codex Guide + +## Scope + +This repository is the platform runner and shared core for GovOPlaN. It owns the server entry point, database/session primitives, auth, tenancy, RBAC, governance, module discovery, migrations, shared WebUI shell, and generic WebUI components. + +Sibling module repositories usually used with this repo: + +- `/mnt/DATA/git/govoplan-files` +- `/mnt/DATA/git/govoplan-mail` +- `/mnt/DATA/git/govoplan-campaign` + +Keep module-specific behavior in the owning module. Core may expose stable extension points, capabilities, and shared components, but should not directly import module WebUI pages or require optional module packages for core-only startup. + +## Local Commands + +Use targeted commands first: + +```bash +cd /mnt/DATA/git/govoplan-core +./.venv/bin/python -m govoplan_core.devserver --smoke --no-reload +./.venv/bin/python -m unittest tests.test_module_system +./.venv/bin/python -m unittest tests.test_api_smoke.ApiSmokeTests.test_mailbox_message_listing_reports_total_count +``` + +For WebUI checks: + +```bash +cd /mnt/DATA/git/govoplan-core/webui +PATH=/mnt/DATA/git/govoplan-core/webui/node_modules/.bin:/home/zemion/.nvm/versions/node/v22.22.3/bin:$PATH /home/zemion/.nvm/versions/node/v22.22.3/bin/npm run test:mail-components +PATH=/mnt/DATA/git/govoplan-core/webui/node_modules/.bin:/home/zemion/.nvm/versions/node/v22.22.3/bin:$PATH /home/zemion/.nvm/versions/node/v22.22.3/bin/npm run test:module-capabilities +PATH=/mnt/DATA/git/govoplan-core/webui/node_modules/.bin:/home/zemion/.nvm/versions/node/v22.22.3/bin:$PATH /home/zemion/.nvm/versions/node/v22.22.3/bin/npm run test:module-permutations +``` + +Run the consolidated focused check when a change touches module discovery, optional integrations, shared mail components, or mailbox listing: + +```bash +cd /mnt/DATA/git/govoplan-core +./scripts/check-focused.sh +``` + +## Working Rules + +- Prefer `rg`, `sed`, and targeted tests over broad recursive scans or full builds. +- Avoid DataGrid changes unless explicitly requested; it is intentionally brittle and has known deferred work. +- Do not add module-to-module imports for optional integrations. Use core registry/capability/module metadata paths. +- Do not keep generated WebUI test folders in git status; they should be ignored and removable. +- Do not start persistent dev servers unless the user asks. diff --git a/README.md b/README.md index 3bb4a43..83eeb8c 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ Canonical policy documents live in `docs/`: - [RBAC_MANIFEST.md](docs/RBAC_MANIFEST.md) - [SYSTEM_GOVERNANCE_MANIFEST.md](docs/SYSTEM_GOVERNANCE_MANIFEST.md) - [MODULE_ARCHITECTURE.md](docs/MODULE_ARCHITECTURE.md) +- [CODEX_WORKFLOW.md](docs/CODEX_WORKFLOW.md) Modules may define module-specific permissions and policy behavior, but the platform-level permission model and governance hierarchy belong here. @@ -34,20 +35,41 @@ cd /mnt/DATA/git/govoplan-core ./.venv/bin/python -m pip install -r requirements-dev.txt ``` -Run the platform server from core through the module-aware development runner. The default config loads the access/core module only; product configs can enable installed modules. +Run the platform server from core through the module-aware development runner. The default config reads `ENABLED_MODULES` and discovers installed module entry points. Local development defaults to `access,campaigns,files,mail`; set `ENABLED_MODULES` explicitly when testing a smaller module permutation. ```bash cd /mnt/DATA/git/govoplan-core ./.venv/bin/python -m govoplan_core.devserver \ - --config app.govoplan_config:get_server_config \ + --host 127.0.0.1 \ + --port 8000 +``` + +For example, to test campaign without files or mail: + +```bash +cd /mnt/DATA/git/govoplan-core +ENABLED_MODULES=access,campaigns ./.venv/bin/python -m govoplan_core.devserver \ --host 127.0.0.1 \ --port 8000 ``` The runner loads the same `GovoplanServerConfig` as `govoplan_core.server.app:app`, builds the platform registry, and passes core plus enabled module source roots to uvicorn as reload directories. After reinstalling the editable package, the same command is also available as `govoplan-devserver`. +The default development SQLite database lives at `runtime/multimailer-dev.db`, alongside other local runtime state. + Local devserver runs do not require Redis. `CELERY_ENABLED` defaults to `false`, so campaign queue actions update database state without publishing Celery tasks. Use the synchronous send flow for local send tests, or set `CELERY_ENABLED=true` only when a Redis broker and worker are running. +If the configured local SQLite database is missing or empty, `govoplan_core.devserver` enables the development bootstrap before loading settings. This creates the schema and the default development login on startup. Explicitly setting `DEV_BOOTSTRAP_ENABLED=false` disables this convenience. Production deployments should use migrations and managed database provisioning instead. + +To verify the effective runtime paths and missing-SQLite bootstrap without starting uvicorn, run the smoke mode: + +```bash +cd /mnt/DATA/git/govoplan-core +./.venv/bin/python -m govoplan_core.devserver --smoke --no-reload +``` + +The smoke mode prints the effective config, runtime root, database URL, modules, reload state, and bootstrap decision, then creates the ASGI app and runs startup once. + `requirements-dev.txt` links local `govoplan-files`, `govoplan-mail`, and `govoplan-campaign` checkouts for development. `requirements-release.txt` installs those modules from tagged git refs for release builds. See [RELEASE_DEPENDENCIES.md](docs/RELEASE_DEPENDENCIES.md). ## WebUI development diff --git a/alembic.ini b/alembic.ini index df23acf..2218e78 100644 --- a/alembic.ini +++ b/alembic.ini @@ -1,7 +1,7 @@ [alembic] script_location = alembic prepend_sys_path = . -sqlalchemy.url = sqlite:///./multimailer-dev.db +sqlalchemy.url = sqlite:///./runtime/multimailer-dev.db [loggers] keys = root,sqlalchemy,alembic diff --git a/alembic/versions/0a1b2c3d4e6f_mail_profile_credential_usernames.py b/alembic/versions/0a1b2c3d4e6f_mail_profile_credential_usernames.py new file mode 100644 index 0000000..68fd16c --- /dev/null +++ b/alembic/versions/0a1b2c3d4e6f_mail_profile_credential_usernames.py @@ -0,0 +1,85 @@ +"""split mail profile usernames from server config + +Revision ID: 0a1b2c3d4e6f +Revises: f5a6b7c8d9e0 +Create Date: 2026-06-25 15:20:00.000000 +""" +from __future__ import annotations + +from alembic import op +import sqlalchemy as sa + +revision = "0a1b2c3d4e6f" +down_revision = "f5a6b7c8d9e0" +branch_labels = None +depends_on = None + + +def _profiles_table() -> sa.Table: + return sa.table( + "mail_server_profiles", + sa.column("id", sa.String(length=36)), + sa.column("smtp_config", sa.JSON()), + sa.column("smtp_username", sa.String(length=320)), + sa.column("imap_config", sa.JSON()), + sa.column("imap_username", sa.String(length=320)), + ) + + +def _without_username(value): + if not isinstance(value, dict): + return value, None + data = dict(value) + username = data.pop("username", None) + data.pop("enabled", None) + return data, username + + +def upgrade() -> None: + bind = op.get_bind() + inspector = sa.inspect(bind) + if "mail_server_profiles" not in inspector.get_table_names(): + return + columns = {column["name"] for column in inspector.get_columns("mail_server_profiles")} + if "smtp_username" not in columns: + op.add_column("mail_server_profiles", sa.Column("smtp_username", sa.String(length=320), nullable=True)) + if "imap_username" not in columns: + op.add_column("mail_server_profiles", sa.Column("imap_username", sa.String(length=320), nullable=True)) + + profiles = _profiles_table() + rows = bind.execute(sa.select(profiles.c.id, profiles.c.smtp_config, profiles.c.smtp_username, profiles.c.imap_config, profiles.c.imap_username)).mappings().all() + for row in rows: + smtp_config, smtp_username = _without_username(row["smtp_config"] or {}) + imap_config, imap_username = _without_username(row["imap_config"] or {}) if row["imap_config"] is not None else (None, None) + values = { + "smtp_config": smtp_config, + "imap_config": imap_config, + } + if row["smtp_username"] in (None, "") and smtp_username not in (None, ""): + values["smtp_username"] = str(smtp_username) + if row["imap_username"] in (None, "") and imap_username not in (None, ""): + values["imap_username"] = str(imap_username) + bind.execute(sa.update(profiles).where(profiles.c.id == row["id"]).values(**values)) + + +def downgrade() -> None: + bind = op.get_bind() + inspector = sa.inspect(bind) + if "mail_server_profiles" not in inspector.get_table_names(): + return + columns = {column["name"] for column in inspector.get_columns("mail_server_profiles")} + profiles = _profiles_table() + if {"smtp_username", "imap_username"}.issubset(columns): + rows = bind.execute(sa.select(profiles.c.id, profiles.c.smtp_config, profiles.c.smtp_username, profiles.c.imap_config, profiles.c.imap_username)).mappings().all() + for row in rows: + smtp_config = dict(row["smtp_config"] or {}) + if row["smtp_username"] not in (None, ""): + smtp_config["username"] = row["smtp_username"] + imap_config = dict(row["imap_config"] or {}) if row["imap_config"] is not None else None + if imap_config is not None and row["imap_username"] not in (None, ""): + imap_config["username"] = row["imap_username"] + bind.execute(sa.update(profiles).where(profiles.c.id == row["id"]).values(smtp_config=smtp_config, imap_config=imap_config)) + if "imap_username" in columns: + op.drop_column("mail_server_profiles", "imap_username") + if "smtp_username" in columns: + op.drop_column("mail_server_profiles", "smtp_username") diff --git a/alembic/versions/1b2c3d4e5f70_drop_mail_credential_override_policy.py b/alembic/versions/1b2c3d4e5f70_drop_mail_credential_override_policy.py new file mode 100644 index 0000000..8f8bc82 --- /dev/null +++ b/alembic/versions/1b2c3d4e5f70_drop_mail_credential_override_policy.py @@ -0,0 +1,108 @@ +"""drop mail credential override policy fields + +Revision ID: 1b2c3d4e5f70 +Revises: 0a1b2c3d4e6f +Create Date: 2026-06-25 18:30:00.000000 +""" +from __future__ import annotations + +import json +from typing import Any + +from alembic import op +import sqlalchemy as sa + +revision = "1b2c3d4e5f70" +down_revision = "0a1b2c3d4e6f" +branch_labels = None +depends_on = None + +_MAIL_POLICY_KEY = "mail_profile_policy" +_CREDENTIAL_KEYS = ("smtp_credentials", "imap_credentials") +_DEPRECATED_LIMIT_KEYS = ("smtp_credentials.allow_override", "imap_credentials.allow_override") + + +def _json_table(table_name: str, column_name: str) -> sa.Table: + return sa.table( + table_name, + sa.column("id", sa.String(length=36)), + sa.column(column_name, sa.JSON()), + ) + + +def _as_dict(value: Any) -> dict[str, Any] | None: + if isinstance(value, dict): + return dict(value) + if isinstance(value, str): + try: + parsed = json.loads(value) + except json.JSONDecodeError: + return None + return dict(parsed) if isinstance(parsed, dict) else None + return None + + +def _scrub_policy(value: Any) -> tuple[dict[str, Any] | None, bool]: + policy = _as_dict(value) + if policy is None: + return None, False + changed = False + + for key in _CREDENTIAL_KEYS: + credential = _as_dict(policy.get(key)) + if credential is not None and "allow_override" in credential: + credential.pop("allow_override", None) + policy[key] = credential + changed = True + + limits = _as_dict(policy.get("allow_lower_level_limits")) + if limits is not None: + for key in _DEPRECATED_LIMIT_KEYS: + if key in limits: + limits.pop(key, None) + changed = True + if changed: + policy["allow_lower_level_limits"] = limits + + return policy, changed + + +def _scrub_settings_table(table_name: str) -> None: + bind = op.get_bind() + table = _json_table(table_name, "settings") + rows = bind.execute(sa.select(table.c.id, table.c.settings)).mappings().all() + for row in rows: + settings = _as_dict(row["settings"]) + if settings is None: + continue + policy, changed = _scrub_policy(settings.get(_MAIL_POLICY_KEY)) + if changed and policy is not None: + settings[_MAIL_POLICY_KEY] = policy + bind.execute(sa.update(table).where(table.c.id == row["id"]).values(settings=settings)) + + +def _scrub_policy_column(table_name: str) -> None: + bind = op.get_bind() + table = _json_table(table_name, "mail_profile_policy") + rows = bind.execute(sa.select(table.c.id, table.c.mail_profile_policy)).mappings().all() + for row in rows: + policy, changed = _scrub_policy(row["mail_profile_policy"]) + if changed and policy is not None: + bind.execute(sa.update(table).where(table.c.id == row["id"]).values(mail_profile_policy=policy)) + + +def upgrade() -> None: + inspector = sa.inspect(op.get_bind()) + tables = set(inspector.get_table_names()) + for table_name in ("system_settings", "tenants"): + if table_name in tables and "settings" in {column["name"] for column in inspector.get_columns(table_name)}: + _scrub_settings_table(table_name) + for table_name in ("users", "groups", "campaigns"): + if table_name in tables and "mail_profile_policy" in {column["name"] for column in inspector.get_columns(table_name)}: + _scrub_policy_column(table_name) + + +def downgrade() -> None: + # The removed fields are redundant with the surviving lower-level limit + # switches, so they cannot be reconstructed safely. + pass diff --git a/docs/CODEX_WORKFLOW.md b/docs/CODEX_WORKFLOW.md new file mode 100644 index 0000000..4a09c76 --- /dev/null +++ b/docs/CODEX_WORKFLOW.md @@ -0,0 +1,73 @@ +# Codex Workflow + +This project is split across the core runner and sibling module repositories. Codex works best when all active repositories are writable from the start and routine checks use targeted commands. + +## Personal Codex Config + +Put machine-specific access in `~/.codex/config.toml`, not in a tracked project file: + +```toml +model = "gpt-5.5" +model_reasoning_effort = "xhigh" +personality = "pragmatic" + +sandbox_mode = "workspace-write" +approval_policy = "on-request" +approvals_reviewer = "user" + +[sandbox_workspace_write] +writable_roots = [ + "/mnt/DATA/git", +] +network_access = false + +[projects."/mnt/DATA/git/govoplan-core"] +trust_level = "trusted" + +[projects."/mnt/DATA/git/govoplan-mail"] +trust_level = "trusted" + +[projects."/mnt/DATA/git/govoplan-files"] +trust_level = "trusted" + +[projects."/mnt/DATA/git/govoplan-campaign"] +trust_level = "trusted" +``` + +The broad writable root reduces approval churn. The explicit project trust entries allow project-local `AGENTS.md` guidance to load for each repository. + +## Repository Guidance + +Each active repository has an `AGENTS.md` file. These files define ownership, module boundaries, and focused commands for Codex. Keep durable project conventions there instead of repeating them in every prompt. + +Use `~/.codex/config.toml` for personal defaults, auth/runtime settings, writable roots, and trust decisions. Avoid checking in absolute-path writable roots or model preferences unless they are intentionally team-wide. + +## Focused Verification + +Use the consolidated script after changes that touch module discovery, optional integrations, shared mail components, mailbox listing, or cross-module WebUI behavior: + +```bash +cd /mnt/DATA/git/govoplan-core +./scripts/check-focused.sh +``` + +For smaller changes, prefer the narrow command named in the relevant `AGENTS.md` file. Examples: + +```bash +cd /mnt/DATA/git/govoplan-core +./.venv/bin/python -m unittest tests.test_module_system + +cd /mnt/DATA/git/govoplan-mail +/mnt/DATA/git/govoplan-core/.venv/bin/python -m unittest discover -s tests + +cd /mnt/DATA/git/govoplan-core/webui +PATH=/mnt/DATA/git/govoplan-core/webui/node_modules/.bin:/home/zemion/.nvm/versions/node/v22.22.3/bin:$PATH /home/zemion/.nvm/versions/node/v22.22.3/bin/npm run test:module-permutations +``` + +## Usage Discipline + +- Prefer `rg`, `sed`, and targeted test commands. +- Avoid broad recursive scans and full builds unless the change warrants them. +- Keep generated build/test folders ignored. +- Keep optional module behavior behind core registry/capability/module metadata boundaries. +- Do not start persistent dev servers unless the user asks. diff --git a/docs/MODULE_ARCHITECTURE.md b/docs/MODULE_ARCHITECTURE.md index d9a629f..07cd9d7 100644 --- a/docs/MODULE_ARCHITECTURE.md +++ b/docs/MODULE_ARCHITECTURE.md @@ -166,13 +166,6 @@ cd /mnt/DATA/git/govoplan-core/webui PATH=/home/zemion/.nvm/versions/node/v22.22.3/bin:$PATH /home/zemion/.nvm/versions/node/v22.22.3/bin/npm run build ``` -Wrapper WebUI verification, while the legacy wrapper still exists: - -```bash -cd /mnt/DATA/git/multi-seal-mail-webui -PATH=/home/zemion/.nvm/versions/node/v22.22.3/bin:$PATH /home/zemion/.nvm/versions/node/v22.22.3/bin/npm run build -``` - Clean generated `dist`, `.vite`, and source-tree `__pycache__` artifacts after verification unless they are intentionally part of a release artifact. ## Release Dependency Rules diff --git a/docs/RBAC_MANIFEST.md b/docs/RBAC_MANIFEST.md index 5ae51cb..c31d632 100644 --- a/docs/RBAC_MANIFEST.md +++ b/docs/RBAC_MANIFEST.md @@ -244,7 +244,7 @@ Recipient-complete campaign JSON, message data and job detail require `recipient | `mail_servers:write` | define/edit profiles in allowed scopes | | `mail_servers:manage_credentials` | create/replace SMTP/IMAP secrets or campaign-level credentials where policy allows | -Reusable encrypted profiles now exist. Effective usability is also constrained by hierarchical mail-profile policy, ownership, allowed/forced profile sets, credential inheritance/override rules and allow/deny patterns. +Reusable encrypted profiles now exist. Effective usability is also constrained by hierarchical mail-profile policy, ownership, allowed/forced profile sets, credential inheritance mode, the lower-level override switch for that mode, and allow/deny patterns. ## Sessions, API Keys and CSRF diff --git a/docs/RELEASE_DEPENDENCIES.md b/docs/RELEASE_DEPENDENCIES.md index dc818a9..6ba5b57 100644 --- a/docs/RELEASE_DEPENDENCIES.md +++ b/docs/RELEASE_DEPENDENCIES.md @@ -48,10 +48,17 @@ PATH=/home/zemion/.nvm/versions/node/v22.22.3/bin:$PATH /home/zemion/.nvm/versio The module repositories include root-level npm package manifests so git installs can resolve `@govoplan/files-webui`, `@govoplan/mail-webui`, and `@govoplan/campaign-webui` from repository roots even though their source lives below `webui/src`. +### Release lockfile strategy + +The supported release composition currently is the full Multi Seal Mail product: core plus files, mail, and campaign. Keep one committed full-product release lockfile at `webui/package-lock.release.json`, generated from `webui/package.release.json` in a clean release workspace. Development `package-lock.json` may continue to point at local `file:` dependencies. + +Frontend module permutations are regression-tested through `GOVOPLAN_WEBUI_MODULE_PACKAGES` and temporary build output, not through committed lockfiles for every possible combination. If a smaller composition becomes a separately shipped product, add an explicit release manifest and lockfile pair for that product, for example `package.release.files-mail.json` and `package-lock.release.files-mail.json`, generated in a clean release workspace from tagged git dependencies. + ## Release Checklist - Keep Python package versions, WebUI package versions, and git tags aligned. - Tag core, files, mail, and campaign repositories together. - Update `requirements-release.txt` and `webui/package.release.json` when the release tag changes. -- Generate release lockfiles from release manifests in a clean build workspace. +- Generate the committed full-product release lockfile from `package.release.json` in a clean build workspace. +- Add separate release manifest/lockfile pairs only for module compositions that are shipped as their own products. - Do not commit local sibling paths into release manifests. diff --git a/docs/SYSTEM_GOVERNANCE_MANIFEST.md b/docs/SYSTEM_GOVERNANCE_MANIFEST.md index 925b3c7..4b37741 100644 --- a/docs/SYSTEM_GOVERNANCE_MANIFEST.md +++ b/docs/SYSTEM_GOVERNANCE_MANIFEST.md @@ -82,7 +82,8 @@ Policy semantics: - forced profiles mean the lower level must choose from the forced set; - a forced set with one profile effectively enforces that profile; - campaign-level profile creation is allowed only if the effective policy permits it; -- credentials may be inherited from the authoritative profile or supplied by a lower level only when override is allowed; +- SMTP/IMAP credentials use one inheritance decision per protocol: lower levels must inherit profile credentials, may inherit profile credentials, or must provide local credentials; +- the lower-level override switch for `smtp_credentials.inherit` and `imap_credentials.inherit` controls whether descendants may change that inheritance decision; - deny patterns always win over allow patterns; - empty or `*` allowlist means allow all except denied; - non-empty allowlist means at least one allow rule must match and no deny rule may match. diff --git a/pyproject.toml b/pyproject.toml index 0149e3a..0e088e7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "govoplan-core" -version = "0.1.1" +version = "0.1.2" description = "Reusable GovOPlaN platform core, access, tenancy, and RBAC components." readme = "README.md" requires-python = ">=3.12" diff --git a/scripts/check-focused.sh b/scripts/check-focused.sh new file mode 100644 index 0000000..73eea1c --- /dev/null +++ b/scripts/check-focused.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="/mnt/DATA/git/govoplan-core" +NODE="/home/zemion/.nvm/versions/node/v22.22.3/bin" +NPM="$NODE/npm" +WEBUI_BIN="$ROOT/webui/node_modules/.bin" + +export PATH="$WEBUI_BIN:$NODE:$PATH" + +cd "$ROOT" + +./.venv/bin/python - <<'PY' +import ast +import pathlib +import sys + +roots = [ + pathlib.Path("/mnt/DATA/git/govoplan-core/src"), + pathlib.Path("/mnt/DATA/git/govoplan-mail/src"), + pathlib.Path("/mnt/DATA/git/govoplan-files/src"), + pathlib.Path("/mnt/DATA/git/govoplan-campaign/src"), + pathlib.Path("/mnt/DATA/git/govoplan-mail/tests"), +] + +errors = [] +count = 0 +for root in roots: + if not root.exists(): + continue + for path in root.rglob("*.py"): + count += 1 + try: + ast.parse(path.read_text(), filename=str(path)) + except SyntaxError as exc: + errors.append(f"{path}:{exc.lineno}:{exc.offset}: {exc.msg}") + +if errors: + print("\n".join(errors)) + sys.exit(1) + +print(f"AST syntax check passed for {count} Python files") +PY + +./.venv/bin/python -c 'import govoplan_core.db.bootstrap; import govoplan_core.admin.service; import govoplan_files.backend.router; import govoplan_mail.backend.sending.imap; print("targeted backend imports passed")' +./.venv/bin/python -m unittest tests.test_module_system +./.venv/bin/python -m unittest discover -s /mnt/DATA/git/govoplan-mail/tests +./.venv/bin/python -m unittest tests.test_api_smoke.ApiSmokeTests.test_mailbox_message_listing_reports_total_count + +cd "$ROOT/webui" +"$NPM" run test:mail-components +"$NPM" run test:module-capabilities +"$NPM" run test:module-permutations + +cd /mnt/DATA/git/govoplan-mail/webui +"$NPM" run test:mail-ui + +cd /mnt/DATA/git/govoplan-campaign/webui +"$NPM" run test:policy-ui +"$NPM" run test:template-preview diff --git a/scripts/push-release-tag.sh b/scripts/push-release-tag.sh new file mode 100644 index 0000000..30d0ad0 --- /dev/null +++ b/scripts/push-release-tag.sh @@ -0,0 +1,424 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat <<'USAGE' +Usage: + scripts/push-release-tag.sh [options] + +Bumps all GovOPlaN package versions, commits all changes in all four repos, +creates an annotated release tag in each repo, and pushes branch+tag. + +By default the script asks which version part to bump: + major X.Y.Z -> (X+1).0.0 + minor X.Y.Z -> X.(Y+1).0 + subversion X.Y.Z -> X.Y.(Z+1) + +Options: + --bump Version bump: major, minor, subversion, or patch. + --version Explicit target version instead of a bump. + -r, --remote Remote to push to. Defaults to origin. + -b, --branch Branch to update in every repo. Defaults to each current branch. + -m, --message Commit message. Defaults to "Release v". + --tag-message Annotated tag message. Defaults to the commit message. + -y, --yes Do not prompt before committing, tagging, and pushing. + -n, --dry-run Print intended actions without changing files or git state. + -h, --help Show this help. + +Repos: + govoplan-core + govoplan-files + govoplan-mail + govoplan-campaign + +Examples: + scripts/push-release-tag.sh + scripts/push-release-tag.sh --bump subversion + scripts/push-release-tag.sh --version 0.2.0 --message "Release v0.2.0" +USAGE +} + +fail() { + echo "error: $*" >&2 + exit 1 +} + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +PARENT="$(dirname "$ROOT")" +REPOS=( + "$ROOT" + "$PARENT/govoplan-files" + "$PARENT/govoplan-mail" + "$PARENT/govoplan-campaign" +) + +REMOTE="origin" +BRANCH="" +BUMP="" +TARGET_VERSION="" +COMMIT_MESSAGE="" +TAG_MESSAGE="" +DRY_RUN=0 +YES=0 + +if [[ -z "${PYTHON:-}" ]]; then + if [[ -x "$ROOT/.venv/bin/python" ]]; then + PYTHON="$ROOT/.venv/bin/python" + else + PYTHON="python3" + fi +fi + +normalize_bump() { + local value="${1,,}" + case "$value" in + major|minor|subversion) + printf '%s\n' "$value" + ;; + patch|sub|sub-version) + printf '%s\n' "subversion" + ;; + *) + return 1 + ;; + esac +} + +validate_version() { + [[ "$1" =~ ^[0-9]+[.][0-9]+[.][0-9]+$ ]] +} + +version_gt() { + local new_major new_minor new_patch old_major old_minor old_patch + IFS=. read -r new_major new_minor new_patch <<< "$1" + IFS=. read -r old_major old_minor old_patch <<< "$2" + + (( new_major > old_major )) && return 0 + (( new_major < old_major )) && return 1 + (( new_minor > old_minor )) && return 0 + (( new_minor < old_minor )) && return 1 + (( new_patch > old_patch )) && return 0 + return 1 +} + +bump_version() { + local version="$1" + local bump="$2" + local major minor patch + IFS=. read -r major minor patch <<< "$version" + + case "$bump" in + major) + printf '%s.0.0\n' "$((major + 1))" + ;; + minor) + printf '%s.%s.0\n' "$major" "$((minor + 1))" + ;; + subversion) + printf '%s.%s.%s\n' "$major" "$minor" "$((patch + 1))" + ;; + *) + fail "unknown bump kind: $bump" + ;; + esac +} + +get_project_name() { + "$PYTHON" -c 'import pathlib, sys, tomllib; print(tomllib.loads(pathlib.Path(sys.argv[1]).read_text())["project"]["name"])' "$1/pyproject.toml" +} + +get_project_version() { + "$PYTHON" -c 'import pathlib, sys, tomllib; print(tomllib.loads(pathlib.Path(sys.argv[1]).read_text())["project"]["version"])' "$1/pyproject.toml" +} + +update_pyproject() { + local repo="$1" + local version="$2" + + "$PYTHON" - "$repo/pyproject.toml" "$version" <<'PYCODE' +from __future__ import annotations + +import pathlib +import re +import sys +import tomllib + +path = pathlib.Path(sys.argv[1]) +new_version = sys.argv[2] +text = path.read_text() +data = tomllib.loads(text) +project = data["project"] +name = project["name"] +old_version = project["version"] + +text, version_count = re.subn( + r'(?m)^version\s*=\s*["\'][^"\']+["\']\s*$', + f'version = "{new_version}"', + text, + count=1, +) +if version_count != 1: + raise SystemExit(f"could not update project.version in {path}") + +if name != "govoplan-core": + text, dependency_count = re.subn( + r'govoplan-core>=\d+\.\d+\.\d+', + f'govoplan-core>={new_version}', + text, + ) + if dependency_count < 1: + raise SystemExit(f"could not update govoplan-core dependency in {path}") + +path.write_text(text) +print(f"{name}: {old_version} -> {new_version}") +PYCODE +} + +print_command() { + printf '+' + printf ' %q' "$@" + printf '\n' +} + +run() { + print_command "$@" + if [[ "$DRY_RUN" -eq 0 ]]; then + "$@" + fi +} + +prompt_for_bump() { + local answer="" + + if [[ -n "$BUMP" || -n "$TARGET_VERSION" ]]; then + return + fi + + if [[ ! -t 0 ]]; then + fail "pass --bump major|minor|subversion or --version x.y.z for non-interactive use" + fi + + echo "Select version increase:" + echo " 1) major X.Y.Z -> (X+1).0.0" + echo " 2) minor X.Y.Z -> X.(Y+1).0" + echo " 3) subversion X.Y.Z -> X.Y.(Z+1)" + printf 'Choice [subversion]: ' + read -r answer + + case "${answer,,}" in + ""|3|subversion|patch|sub|sub-version) + BUMP="subversion" + ;; + 1|major) + BUMP="major" + ;; + 2|minor) + BUMP="minor" + ;; + *) + fail "unknown version bump: $answer" + ;; + esac +} + +confirm_release() { + local answer="" + + if [[ "$DRY_RUN" -eq 1 || "$YES" -eq 1 ]]; then + return + fi + + if [[ ! -t 0 ]]; then + fail "refusing to commit, tag, and push without confirmation; pass --yes for non-interactive use" + fi + + printf 'Commit all changes, tag all repos, and push to %s? [y/N] ' "$REMOTE" + read -r answer + case "${answer,,}" in + y|yes) + ;; + *) + echo "Aborted." + exit 1 + ;; + esac +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --bump) + [[ $# -ge 2 ]] || fail "missing value for $1" + BUMP="$(normalize_bump "$2")" || fail "unknown bump kind: $2" + shift 2 + ;; + --version) + [[ $# -ge 2 ]] || fail "missing value for $1" + TARGET_VERSION="$2" + shift 2 + ;; + -r|--remote) + [[ $# -ge 2 ]] || fail "missing value for $1" + REMOTE="$2" + shift 2 + ;; + -b|--branch) + [[ $# -ge 2 ]] || fail "missing value for $1" + BRANCH="$2" + shift 2 + ;; + -m|--message) + [[ $# -ge 2 ]] || fail "missing value for $1" + COMMIT_MESSAGE="$2" + shift 2 + ;; + --tag-message) + [[ $# -ge 2 ]] || fail "missing value for $1" + TAG_MESSAGE="$2" + shift 2 + ;; + -y|--yes) + YES=1 + shift + ;; + -n|--dry-run) + DRY_RUN=1 + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unknown argument: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +if [[ -n "$BUMP" && -n "$TARGET_VERSION" ]]; then + fail "use either --bump or --version, not both" +fi + +prompt_for_bump + +if ! command -v "$PYTHON" >/dev/null 2>&1; then + fail "Python interpreter not found: $PYTHON" +fi + +declare -A PROJECT_NAMES +declare -A OLD_VERSIONS +declare -A BRANCHES + +for repo in "${REPOS[@]}"; do + [[ -d "$repo/.git" ]] || fail "not a git repo: $repo" + [[ -f "$repo/pyproject.toml" ]] || fail "missing pyproject.toml: $repo" + + PROJECT_NAMES["$repo"]="$(get_project_name "$repo")" + OLD_VERSIONS["$repo"]="$(get_project_version "$repo")" + + validate_version "${OLD_VERSIONS[$repo]}" || fail "unsupported version ${OLD_VERSIONS[$repo]} in $repo" + + if [[ -n "$BRANCH" ]]; then + BRANCHES["$repo"]="$BRANCH" + else + BRANCHES["$repo"]="$(git -C "$repo" symbolic-ref --quiet --short HEAD || true)" + fi + + [[ -n "${BRANCHES[$repo]}" ]] || fail "cannot infer branch for $repo; pass --branch " + git -C "$repo" check-ref-format "refs/heads/${BRANCHES[$repo]}" || fail "invalid branch name ${BRANCHES[$repo]} for $repo" + git -C "$repo" remote get-url "$REMOTE" >/dev/null || fail "remote $REMOTE not found in $repo" + + upstream="$(git -C "$repo" rev-parse --abbrev-ref --symbolic-full-name '@{u}' 2>/dev/null || true)" + if [[ -n "$upstream" ]]; then + behind_count="$(git -C "$repo" rev-list --count "HEAD..$upstream")" + [[ "$behind_count" == "0" ]] || fail "$repo is behind $upstream by $behind_count commit(s); pull/rebase first" + fi +done + +if [[ -z "$TARGET_VERSION" ]]; then + BASE_VERSION="${OLD_VERSIONS[$ROOT]}" + for repo in "${REPOS[@]}"; do + if [[ "${OLD_VERSIONS[$repo]}" != "$BASE_VERSION" ]]; then + fail "repo versions differ; pass --version x.y.z to choose an explicit target" + fi + done + TARGET_VERSION="$(bump_version "$BASE_VERSION" "$BUMP")" +fi + +validate_version "$TARGET_VERSION" || fail "target version must be x.y.z: $TARGET_VERSION" + +for repo in "${REPOS[@]}"; do + if ! version_gt "$TARGET_VERSION" "${OLD_VERSIONS[$repo]}"; then + fail "target version $TARGET_VERSION must be greater than ${OLD_VERSIONS[$repo]} for ${PROJECT_NAMES[$repo]}" + fi +done + +TAG="v$TARGET_VERSION" +COMMIT_MESSAGE="${COMMIT_MESSAGE:-Release $TAG}" +TAG_MESSAGE="${TAG_MESSAGE:-$COMMIT_MESSAGE}" + +git -C "$ROOT" check-ref-format "refs/tags/$TAG" || fail "invalid tag name: $TAG" + +for repo in "${REPOS[@]}"; do + if git -C "$repo" rev-parse --quiet --verify "refs/tags/$TAG" >/dev/null; then + fail "local tag already exists in ${PROJECT_NAMES[$repo]}: $TAG" + fi + + set +e + git -C "$repo" ls-remote --exit-code --tags "$REMOTE" "refs/tags/$TAG" >/dev/null + ls_remote_status=$? + set -e + + if [[ "$ls_remote_status" -eq 0 ]]; then + fail "remote tag already exists for ${PROJECT_NAMES[$repo]} on $REMOTE: $TAG" + elif [[ "$ls_remote_status" -ne 2 ]]; then + fail "could not check remote tags for ${PROJECT_NAMES[$repo]} on $REMOTE" + fi +done + +echo "Release plan:" +echo " version: $TARGET_VERSION" +echo " tag: $TAG" +echo " remote: $REMOTE" +echo " commit message: $COMMIT_MESSAGE" +echo + +for repo in "${REPOS[@]}"; do + echo "${PROJECT_NAMES[$repo]}: ${OLD_VERSIONS[$repo]} -> $TARGET_VERSION on ${BRANCHES[$repo]}" + git -C "$repo" status --short + echo +done + +confirm_release + +for repo in "${REPOS[@]}"; do + if [[ "$DRY_RUN" -eq 1 ]]; then + echo "Would update $repo/pyproject.toml to $TARGET_VERSION" + else + update_pyproject "$repo" "$TARGET_VERSION" + fi +done + +for repo in "${REPOS[@]}"; do + run git -C "$repo" add -A + + if [[ "$DRY_RUN" -eq 0 ]]; then + if git -C "$repo" diff --cached --quiet; then + fail "no staged changes for ${PROJECT_NAMES[$repo]} after version bump" + fi + fi + + run git -C "$repo" commit -m "$COMMIT_MESSAGE" + run git -C "$repo" tag -a "$TAG" -m "$TAG_MESSAGE" +done + +for repo in "${REPOS[@]}"; do + run git -C "$repo" push --atomic "$REMOTE" "HEAD:refs/heads/${BRANCHES[$repo]}" "refs/tags/$TAG" +done + +if [[ "$DRY_RUN" -eq 1 ]]; then + echo "Dry run complete for $TAG across all GovOPlaN repos." +else + echo "Released $TAG across all GovOPlaN repos." +fi diff --git a/src/govoplan_core/admin/governance.py b/src/govoplan_core/admin/governance.py index 290325c..0d91bb0 100644 --- a/src/govoplan_core/admin/governance.py +++ b/src/govoplan_core/admin/governance.py @@ -5,6 +5,7 @@ from dataclasses import dataclass from sqlalchemy.orm import Session from govoplan_core.admin.service import AdminConflictError, AdminValidationError, slugify +from govoplan_core.core.optional import reraise_unless_missing_package from govoplan_core.db.models import ( ApiKey, GovernanceTemplate, @@ -17,7 +18,6 @@ from govoplan_core.db.models import ( UserGroupMembership, UserRoleAssignment, ) -from govoplan_files.backend.db.models import FileAsset, FileFolder, FileShare from govoplan_core.security.permissions import validate_tenant_permissions SYSTEM_SETTINGS_ID = "global" @@ -25,6 +25,15 @@ TEMPLATE_KINDS = {"group", "role"} ASSIGNMENT_MODES = {"available", "required"} +def _file_models(): + try: + from govoplan_files.backend.db.models import FileAsset, FileFolder, FileShare + except ModuleNotFoundError as exc: + reraise_unless_missing_package(exc, "govoplan_files") + return None + return FileAsset, FileFolder, FileShare + + @dataclass(frozen=True) class EffectiveTenantGovernance: allow_custom_groups: bool @@ -246,11 +255,17 @@ def _remove_materialized_template(session: Session, item: GovernanceTemplate, te return membership_count = session.query(UserGroupMembership).filter(UserGroupMembership.group_id == group.id).count() role_count = session.query(GroupRoleAssignment).filter(GroupRoleAssignment.group_id == group.id).count() - owned_asset_count = session.query(FileAsset).filter(FileAsset.owner_group_id == group.id).count() - owned_folder_count = session.query(FileFolder).filter(FileFolder.owner_group_id == group.id).count() - shared_asset_count = session.query(FileShare).filter( - FileShare.target_type == "group", FileShare.target_id == group.id - ).count() + file_models = _file_models() + owned_asset_count = 0 + owned_folder_count = 0 + shared_asset_count = 0 + if file_models is not None: + FileAsset, FileFolder, FileShare = file_models + owned_asset_count = session.query(FileAsset).filter(FileAsset.owner_group_id == group.id).count() + owned_folder_count = session.query(FileFolder).filter(FileFolder.owner_group_id == group.id).count() + shared_asset_count = session.query(FileShare).filter( + FileShare.target_type == "group", FileShare.target_id == group.id + ).count() if membership_count or role_count or owned_asset_count or owned_folder_count or shared_asset_count: raise AdminConflictError( f"Cannot remove {item.name!r} from the tenant while its managed group has members, roles, files, folders or shares." diff --git a/src/govoplan_core/admin/service.py b/src/govoplan_core/admin/service.py index 57c3787..e48ae96 100644 --- a/src/govoplan_core/admin/service.py +++ b/src/govoplan_core/admin/service.py @@ -9,6 +9,7 @@ from dataclasses import dataclass from sqlalchemy import func from sqlalchemy.orm import Session +from govoplan_core.core.optional import reraise_unless_missing_package from govoplan_core.db.models import ( Account, ApiKey, @@ -548,16 +549,25 @@ def role_assignment_counts(session: Session, role_id: str) -> tuple[int, int]: ) +def _optional_model(module_name: str, model_name: str): + try: + module = __import__(module_name, fromlist=[model_name]) + except ModuleNotFoundError as exc: + reraise_unless_missing_package(exc, module_name.split(".", 1)[0]) + return None + return getattr(module, model_name) + + def tenant_counts(session: Session, tenant_id: str) -> dict[str, int]: - from govoplan_campaign.backend.db.models import Campaign - from govoplan_files.backend.db.models import FileAsset + Campaign = _optional_model("govoplan_campaign.backend.db.models", "Campaign") + FileAsset = _optional_model("govoplan_files.backend.db.models", "FileAsset") return { "users": session.query(User).filter(User.tenant_id == tenant_id).count(), "active_users": session.query(User).filter(User.tenant_id == tenant_id, User.is_active.is_(True)).count(), "groups": session.query(Group).filter(Group.tenant_id == tenant_id).count(), - "campaigns": session.query(Campaign).filter(Campaign.tenant_id == tenant_id).count(), - "files": session.query(FileAsset).filter(FileAsset.tenant_id == tenant_id).count(), + "campaigns": session.query(Campaign).filter(Campaign.tenant_id == tenant_id).count() if Campaign is not None else 0, + "files": session.query(FileAsset).filter(FileAsset.tenant_id == tenant_id).count() if FileAsset is not None else 0, "api_keys": session.query(ApiKey).filter(ApiKey.tenant_id == tenant_id, ApiKey.revoked_at.is_(None)).count(), } diff --git a/src/govoplan_core/api/v1/admin_schemas.py b/src/govoplan_core/api/v1/admin_schemas.py index 44f5a9f..80b0936 100644 --- a/src/govoplan_core/api/v1/admin_schemas.py +++ b/src/govoplan_core/api/v1/admin_schemas.py @@ -317,6 +317,7 @@ class PolicySourceStepItem(BaseModel): scope_id: str | None = None label: str applied_fields: list[str] = Field(default_factory=list) + policy: dict[str, Any] = Field(default_factory=dict) class PrivacyRetentionPolicyItem(BaseModel): diff --git a/src/govoplan_core/api/v1/system.py b/src/govoplan_core/api/v1/system.py index 9e2ec35..fd81e4a 100644 --- a/src/govoplan_core/api/v1/system.py +++ b/src/govoplan_core/api/v1/system.py @@ -5,6 +5,7 @@ from typing import Any from fastapi import APIRouter, Depends, HTTPException, status from govoplan_core.auth.dependencies import ApiPrincipal, require_any_scope +from govoplan_core.core.optional import module_not_found_for router = APIRouter(prefix="/schemas", tags=["schemas"]) @@ -13,12 +14,22 @@ def _load_campaign_schema() -> dict[str, Any]: try: from govoplan_campaign.backend.campaign.loader import load_campaign_schema except ModuleNotFoundError as exc: - if exc.name == "govoplan_campaign" or exc.name.startswith("govoplan_campaign."): + if module_not_found_for(exc, "govoplan_campaign"): raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Campaign module is not installed") from exc raise return load_campaign_schema() +def _load_campaign_schema_ui() -> dict[str, Any]: + try: + from govoplan_campaign.backend.campaign.loader import load_campaign_schema_ui + except ModuleNotFoundError as exc: + if module_not_found_for(exc, "govoplan_campaign"): + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Campaign module is not installed") from exc + raise + return load_campaign_schema_ui() + + @router.get("/campaign") def get_campaign_schema( principal: ApiPrincipal = Depends(require_any_scope("campaigns:campaign:read", "campaigns:campaign:create", "admin:roles:read")), @@ -27,3 +38,14 @@ def get_campaign_schema( del principal return _load_campaign_schema() + + +@router.get("/campaign/ui") +def get_campaign_schema_ui( + principal: ApiPrincipal = Depends(require_any_scope("campaigns:campaign:read", "campaigns:campaign:create", "admin:roles:read")), +) -> dict[str, Any]: + """Return UI metadata paired with the campaign JSON Schema.""" + + del principal + return _load_campaign_schema_ui() + diff --git a/src/govoplan_core/core/discovery.py b/src/govoplan_core/core/discovery.py index c766ecc..6c9e7f2 100644 --- a/src/govoplan_core/core/discovery.py +++ b/src/govoplan_core/core/discovery.py @@ -35,6 +35,8 @@ def discover_module_manifests( enabled = set(enabled_modules) if enabled_modules is not None else None manifests: list[ModuleManifest] = [] for entry_point in iter_module_entry_points(group): + if enabled is not None and entry_point.name not in enabled: + continue manifest = _load_manifest(entry_point) if enabled is not None and manifest.id not in enabled: continue diff --git a/src/govoplan_core/core/modules.py b/src/govoplan_core/core/modules.py index f8ca625..76b7a92 100644 --- a/src/govoplan_core/core/modules.py +++ b/src/govoplan_core/core/modules.py @@ -104,6 +104,7 @@ class ResourceAclProvider(Protocol): TenantSummaryProvider = Callable[[object, str], Mapping[str, int]] DeleteVetoProvider = Callable[[object, str, str], None] RouteFactory = Callable[[ModuleContext], "APIRouter"] +CapabilityFactory = Callable[[ModuleContext], object] @dataclass(frozen=True, slots=True) @@ -122,4 +123,5 @@ class ModuleManifest: resource_acl_providers: tuple[ResourceAclProvider, ...] = () tenant_summary_providers: tuple[TenantSummaryProvider, ...] = () delete_veto_providers: Mapping[str, Sequence[DeleteVetoProvider]] = field(default_factory=dict) + capability_factories: Mapping[str, CapabilityFactory] = field(default_factory=dict) diff --git a/src/govoplan_core/core/optional.py b/src/govoplan_core/core/optional.py new file mode 100644 index 0000000..15dc66c --- /dev/null +++ b/src/govoplan_core/core/optional.py @@ -0,0 +1,13 @@ +from __future__ import annotations + + +def module_not_found_for(exc: ModuleNotFoundError, package: str) -> bool: + """Return true when a ModuleNotFoundError belongs to an optional package.""" + + missing = exc.name or "" + return missing == package or missing.startswith(f"{package}.") + + +def reraise_unless_missing_package(exc: ModuleNotFoundError, package: str) -> None: + if not module_not_found_for(exc, package): + raise exc diff --git a/src/govoplan_core/core/registry.py b/src/govoplan_core/core/registry.py index 38d7909..6679ed5 100644 --- a/src/govoplan_core/core/registry.py +++ b/src/govoplan_core/core/registry.py @@ -6,7 +6,9 @@ from collections.abc import Iterable, Mapping from dataclasses import dataclass from govoplan_core.core.modules import ( + CapabilityFactory, DeleteVetoProvider, + ModuleContext, ModuleManifest, NavItem, PermissionDefinition, @@ -36,6 +38,9 @@ class PlatformRegistry: self._manifests: dict[str, ModuleManifest] = {} self._tenant_summary_providers: dict[str, TenantSummaryProvider] = {} self._delete_veto_providers: dict[str, list[DeleteVetoProvider]] = defaultdict(list) + self._capability_factories: dict[str, CapabilityFactory] = {} + self._capabilities: dict[str, object] = {} + self._capability_context: ModuleContext | None = None def register(self, manifest: ModuleManifest) -> ModuleManifest: if manifest.id in self._manifests: @@ -46,6 +51,8 @@ class PlatformRegistry: for resource_type, providers in manifest.delete_veto_providers.items(): for provider in providers: self.register_delete_veto(resource_type, provider) + for name, factory in manifest.capability_factories.items(): + self.register_capability_factory(manifest.id, name, factory) return manifest def get(self, module_id: str) -> ModuleManifest | None: @@ -79,6 +86,36 @@ class PlatformRegistry: def resource_acl_providers(self) -> tuple[ResourceAclProvider, ...]: return tuple(provider for manifest in self.manifests() for provider in manifest.resource_acl_providers) + def configure_capability_context(self, context: ModuleContext) -> None: + self._capability_context = context + self._capabilities.clear() + + def register_capability_factory(self, module_id: str, name: str, factory: CapabilityFactory) -> None: + if name in self._capability_factories: + raise RegistryError(f"Duplicate capability: {name}") + self._capability_factories[name] = factory + + def has_capability(self, name: str) -> bool: + return name in self._capability_factories + + def capability(self, name: str) -> object | None: + if name in self._capabilities: + return self._capabilities[name] + factory = self._capability_factories.get(name) + if factory is None: + return None + if self._capability_context is None: + raise RegistryError(f"Capability context is not configured: {name}") + capability = factory(self._capability_context) + self._capabilities[name] = capability + return capability + + def require_capability(self, name: str) -> object: + capability = self.capability(name) + if capability is None: + raise RegistryError(f"Required capability is not available: {name}") + return capability + def register_tenant_summary_provider(self, module_id: str, provider: TenantSummaryProvider) -> None: self._tenant_summary_providers[module_id] = provider diff --git a/src/govoplan_core/db/bootstrap.py b/src/govoplan_core/db/bootstrap.py index 7056df2..fa3c8e4 100644 --- a/src/govoplan_core/db/bootstrap.py +++ b/src/govoplan_core/db/bootstrap.py @@ -5,6 +5,7 @@ from dataclasses import dataclass from sqlalchemy.orm import Session from govoplan_core.admin.service import ensure_default_roles, get_or_create_account +from govoplan_core.core.optional import reraise_unless_missing_package from govoplan_core.db.base import Base from govoplan_core.db.models import Role, SystemRoleAssignment, Tenant, User, UserRoleAssignment from govoplan_core.db.session import get_database @@ -21,15 +22,26 @@ class BootstrapResult: created_api_key: CreatedApiKey | None +def _import_optional_models(module_name: str) -> None: + try: + __import__(module_name) + except ModuleNotFoundError as exc: + reraise_unless_missing_package(exc, module_name.split(".", 1)[0]) + + def create_all_tables() -> None: - # Import models so SQLAlchemy sees all tables. + # Import models so SQLAlchemy sees all tables from installed modules. from govoplan_core.db import models # noqa: F401 from govoplan_core.access.db import models as access_models # noqa: F401 - from govoplan_files.backend.db import models as file_models # noqa: F401 - from govoplan_mail.backend.db import models as mail_models # noqa: F401 - from govoplan_campaign.backend.db import models as campaign_models # noqa: F401 from govoplan_core.access.db.base import AccessBase + for module_name in ( + "govoplan_files.backend.db.models", + "govoplan_mail.backend.db.models", + "govoplan_campaign.backend.db.models", + ): + _import_optional_models(module_name) + engine = get_database().engine Base.metadata.create_all(bind=engine) AccessBase.metadata.create_all(bind=engine) diff --git a/src/govoplan_core/db/migrations.py b/src/govoplan_core/db/migrations.py index 871b51c..9e02409 100644 --- a/src/govoplan_core/db/migrations.py +++ b/src/govoplan_core/db/migrations.py @@ -7,7 +7,7 @@ from pathlib import Path from alembic import command from alembic.config import Config from alembic.runtime.migration import MigrationContext -from sqlalchemy import create_engine, inspect +from sqlalchemy import create_engine, inspect, text from govoplan_core.core.migrations import MigrationMetadataPlan, migration_metadata_plan from govoplan_core.server.default_config import get_server_config @@ -23,6 +23,7 @@ from govoplan_core.settings import settings REVISION_AUTH_RBAC = "2c3d4e5f6a7b" REVISION_FILE_STORAGE = "3d4e5f6a7b8c" REVISION_FILE_FOLDERS = "4e5f6a7b8c9d" +REVISION_HIERARCHICAL_SETTINGS = "f5a6b7c8d9e0" _FILE_STORAGE_TABLES = { "file_blobs", @@ -69,6 +70,61 @@ _FILE_STORAGE_COLUMNS = { "file_folders": {"id", "tenant_id", "owner_type", "path"}, } +_CREATE_ALL_THROUGH_HIERARCHICAL_TABLES = { + "accounts", + "auth_sessions", + "audit_log", + "campaign_versions", + "campaign_jobs", + "send_attempts", + "system_role_assignments", + "system_settings", + "governance_templates", + "governance_template_assignments", + "mail_server_profiles", +} + +_CREATE_ALL_THROUGH_HIERARCHICAL_COLUMNS = { + "auth_sessions": {"account_id", "csrf_token_hash"}, + "audit_log": {"scope", "tenant_id", "user_id", "api_key_id"}, + "campaign_versions": { + "workflow_state", + "current_flow", + "user_lock_state", + "user_locked_at", + "user_locked_by_user_id", + "execution_snapshot", + "execution_snapshot_hash", + "execution_snapshot_at", + }, + "campaign_jobs": {"claimed_at", "claim_token", "smtp_started_at", "outcome_unknown_at", "eml_sha256"}, + "send_attempts": {"status", "claim_token"}, + "users": {"account_id", "settings", "mail_profile_policy"}, + "groups": {"system_template_id", "system_required", "settings", "mail_profile_policy"}, + "roles": {"system_template_id", "system_required"}, + "tenants": {"settings", "allow_custom_groups", "allow_custom_roles", "allow_api_keys"}, + "campaigns": {"settings", "mail_profile_policy"}, + "mail_server_profiles": {"scope_type", "scope_id"}, + "system_settings": {"settings", "allow_tenant_custom_groups", "allow_tenant_custom_roles", "allow_tenant_api_keys"}, +} + +_CREATE_ALL_THROUGH_HIERARCHICAL_INDEXES = { + "file_folders": {"uq_file_folders_active_user_path", "uq_file_folders_active_group_path"}, + "campaign_versions": { + "ix_campaign_versions_user_lock_state", + "ix_campaign_versions_user_locked_by_user_id", + "ix_campaign_versions_execution_snapshot_hash", + }, + "campaign_jobs": {"ix_campaign_jobs_claim_token", "ix_campaign_jobs_eml_sha256"}, + "send_attempts": {"ix_send_attempts_status", "ix_send_attempts_claim_token"}, + "audit_log": {"ix_audit_log_scope_created_at", "ix_audit_log_tenant_scope_created_at"}, + "auth_sessions": {"ix_auth_sessions_account_id"}, + "users": {"ix_users_account_id"}, + "groups": {"ix_groups_system_template_id"}, + "roles": {"ix_roles_system_template_id"}, + "mail_server_profiles": {"ix_mail_server_profiles_scope_type", "ix_mail_server_profiles_scope_id", "ix_mail_server_profiles_scope"}, +} + @dataclass(frozen=True, slots=True) class MigrationResult: @@ -123,8 +179,44 @@ def _has_columns(inspector, table_name: str, required: set[str]) -> bool: return required.issubset(actual) +def _has_indexes(inspector, table_name: str, required: set[str]) -> bool: + try: + actual = {index["name"] for index in inspector.get_indexes(table_name)} + except Exception: + return False + return required.issubset(actual) + + +def _has_create_all_schema_through_hierarchical_settings(inspector, tables: set[str]) -> bool: + if not _CREATE_ALL_THROUGH_HIERARCHICAL_TABLES.issubset(tables): + return False + for table_name, required_columns in _CREATE_ALL_THROUGH_HIERARCHICAL_COLUMNS.items(): + if table_name not in tables or not _has_columns(inspector, table_name, required_columns): + return False + for table_name, required_indexes in _CREATE_ALL_THROUGH_HIERARCHICAL_INDEXES.items(): + if table_name not in tables or not _has_indexes(inspector, table_name, required_indexes): + return False + return True + + +def _backfill_user_lock_state_for_create_all_schema(database_url: str) -> None: + engine = create_engine(database_url) + try: + with engine.begin() as connection: + connection.execute(text(""" + UPDATE campaign_versions + SET user_lock_state = 'permanent', + user_locked_at = published_at, + user_locked_by_user_id = NULL + WHERE published_at IS NOT NULL + AND user_lock_state IS NULL + """)) + finally: + engine.dispose() + + def reconcile_legacy_create_all_schema(database_url: str | None = None) -> str | None: - """Repair the known Alembic/create_all drift without modifying table data. + """Repair the known Alembic/create_all drift without modifying application data. Returns the revision stamped during reconciliation, or ``None`` when no repair was necessary. A partial/unknown schema is intentionally left alone @@ -148,6 +240,7 @@ def reconcile_legacy_create_all_schema(database_url: str | None = None) -> str | "file_folders", _FILE_STORAGE_COLUMNS["file_folders"], ) + has_create_all_hierarchical_schema = _has_create_all_schema_through_hierarchical_settings(schema, tables) finally: engine.dispose() @@ -158,6 +251,17 @@ def reconcile_legacy_create_all_schema(database_url: str | None = None) -> str | target = REVISION_FILE_STORAGE elif current == REVISION_FILE_STORAGE and has_file_folders: target = REVISION_FILE_FOLDERS + elif current == REVISION_FILE_FOLDERS and has_create_all_hierarchical_schema: + # Development DBs may be stamped at 4e after earlier create_all + # reconciliation even though later tables/columns were already created + # by a newer model set. Skip only when that complete known schema is + # present, then let newer migrations such as mail credential usernames + # run normally. + _backfill_user_lock_state_for_create_all_schema(url) + target = REVISION_HIERARCHICAL_SETTINGS + elif current is None and has_create_all_hierarchical_schema: + _backfill_user_lock_state_for_create_all_schema(url) + target = REVISION_HIERARCHICAL_SETTINGS elif current is None and has_file_storage and has_file_folders: # This is the other create_all-only development shape. The strict # column checks above ensure that we only stamp a complete known schema. diff --git a/src/govoplan_core/devserver.py b/src/govoplan_core/devserver.py index 93dd20e..ca80911 100644 --- a/src/govoplan_core/devserver.py +++ b/src/govoplan_core/devserver.py @@ -2,6 +2,7 @@ from __future__ import annotations import argparse import importlib.util +from dataclasses import dataclass import inspect import os import sys @@ -19,6 +20,17 @@ DEFAULT_APP = "govoplan_core.server.app:app" DEFAULT_CONFIG = "govoplan_core.server.default_config:get_server_config" +@dataclass(slots=True) +class DevserverState: + config_path: str | None + runtime_root: Path | None + database_url: str + bootstrap_db_path: Path | None + config: GovoplanServerConfig + registry: PlatformRegistry + reload_dirs: list[str] + + def _config_module_runtime_root(config_path: str | None) -> Path | None: object_path = config_path or os.getenv("GOVOPLAN_SERVER_CONFIG") if not object_path: @@ -42,12 +54,13 @@ def _config_module_runtime_root(config_path: str | None) -> Path | None: def apply_runtime_defaults(config_path: str | None) -> Path | None: - runtime_root = _config_module_runtime_root(config_path) - if runtime_root is None: - return None + runtime_root = _config_module_runtime_root(config_path) or Path.cwd().resolve() + (runtime_root / "runtime").mkdir(parents=True, exist_ok=True) + (runtime_root / "runtime" / "files").mkdir(parents=True, exist_ok=True) + (runtime_root / "runtime" / "mock-mailbox").mkdir(parents=True, exist_ok=True) defaults = { - "DATABASE_URL": f"sqlite:///{runtime_root / 'multimailer-dev.db'}", + "DATABASE_URL": f"sqlite:///{runtime_root / 'runtime' / 'multimailer-dev.db'}", "FILE_STORAGE_LOCAL_ROOT": str(runtime_root / "runtime" / "files"), "MOCK_MAILBOX_DIR": str(runtime_root / "runtime" / "mock-mailbox"), } @@ -56,6 +69,55 @@ def apply_runtime_defaults(config_path: str | None) -> Path | None: return runtime_root +def _sqlite_database_path(database_url: str) -> Path | None: + if not database_url.startswith("sqlite:///"): + return None + raw_path = database_url[len("sqlite:///"):] + if not raw_path or raw_path == ":memory:": + return None + path = Path(raw_path).expanduser() + return path if path.is_absolute() else Path.cwd() / path + + +def validate_sqlite_database_url(database_url: str) -> None: + db_path = _sqlite_database_path(database_url) + if db_path is None: + return + if db_path.name.startswith(".fuse"): + raise SystemExit( + "DATABASE_URL points at a FUSE-hidden SQLite file. " + "Stop the process still holding the deleted database, unset DATABASE_URL, " + "or set DATABASE_URL to a normal .db path under the core runtime directory." + ) + if not db_path.parent.exists(): + raise SystemExit( + f"DATABASE_URL points to {db_path}, but its parent directory does not exist. " + "Unset stale DATABASE_URL or set it to " + f"sqlite:///{Path.cwd().resolve() / 'runtime' / 'multimailer-dev.db'}." + ) + + +def _env_truthy(value: str | None) -> bool: + return value is not None and value.strip().lower() in {"1", "true", "yes", "on"} + + +def enable_dev_bootstrap_for_missing_sqlite(database_url: str) -> Path | None: + db_path = _sqlite_database_path(database_url) + if db_path is None or db_path.name.startswith(".fuse") or not db_path.parent.exists(): + return None + if db_path.exists() and db_path.stat().st_size > 0: + return None + os.environ.setdefault("DEV_BOOTSTRAP_ENABLED", "true") + return db_path + + +def apply_detected_dev_bootstrap(config: GovoplanServerConfig, bootstrap_db_path: Path | None) -> None: + if bootstrap_db_path is None or not _env_truthy(os.getenv("DEV_BOOTSTRAP_ENABLED")): + return + if config.settings is not None and hasattr(config.settings, "dev_bootstrap_enabled"): + setattr(config.settings, "dev_bootstrap_enabled", True) + + def _source_root_for_path(path: str | os.PathLike[str]) -> Path | None: source_path = Path(path).resolve() directory = source_path if source_path.is_dir() else source_path.parent @@ -177,31 +239,86 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: parser.add_argument("--port", type=int, default=8000, help="Port to bind. Default: 8000.") parser.add_argument("--no-reload", action="store_true", help="Disable uvicorn reload.") parser.add_argument("--reload-dir", action="append", default=[], help="Additional directory to watch. May be passed multiple times.") + parser.add_argument("--smoke", action="store_true", help="Prepare runtime paths, run app startup, print effective paths, and exit without uvicorn.") return parser.parse_args(argv) +def prepare_devserver(config_path: str | None, *, extra_reload_dirs: Sequence[str] = ()) -> DevserverState: + runtime_root = apply_runtime_defaults(config_path) + database_url = os.getenv("DATABASE_URL", "") + validate_sqlite_database_url(database_url) + bootstrap_db_path = enable_dev_bootstrap_for_missing_sqlite(database_url) + config = load_server_config(config_path) + database_url = str(getattr(config.settings, "database_url", database_url) or "") + validate_sqlite_database_url(database_url) + if bootstrap_db_path is None: + bootstrap_db_path = enable_dev_bootstrap_for_missing_sqlite(database_url) + apply_detected_dev_bootstrap(config, bootstrap_db_path) + registry = build_platform_registry(config.enabled_modules, manifest_factories=config.manifest_factories) + reload_dirs = build_reload_dirs(config, config_path=config_path, registry=registry, extra_dirs=extra_reload_dirs) + return DevserverState( + config_path=config_path, + runtime_root=runtime_root, + database_url=database_url, + bootstrap_db_path=bootstrap_db_path, + config=config, + registry=registry, + reload_dirs=reload_dirs, + ) + + +def print_devserver_summary(state: DevserverState, *, app: str, no_reload: bool) -> None: + print(f"Starting GovOPlaN dev server: {app}") + print(f"Config: {state.config_path or DEFAULT_CONFIG}") + print(f"Runtime root: {state.runtime_root}") + if state.database_url: + print(f"Database: {state.database_url}") + if state.bootstrap_db_path is not None: + bootstrap_state = "enabled" if getattr(state.config.settings, "dev_bootstrap_enabled", False) else "disabled by DEV_BOOTSTRAP_ENABLED" + print(f"Dev bootstrap for missing SQLite DB: {bootstrap_state} ({state.bootstrap_db_path})") + print("Modules: " + ", ".join(manifest.id for manifest in state.registry.manifests())) + if no_reload: + print("Reload: disabled") + else: + print("Reload dirs:") + for directory in state.reload_dirs: + print(f" - {directory}") + + +def run_smoke_check(state: DevserverState) -> None: + try: + from fastapi.testclient import TestClient + except ModuleNotFoundError as exc: + raise SystemExit("fastapi.testclient is not available. Install govoplan-core dev/server requirements before running --smoke.") from exc + + from govoplan_core.server.app import create_app + + print("Smoke check: creating ASGI app and running startup") + app = create_app(state.config) + with TestClient(app) as client: + response = client.get("/health") + if response.status_code != 200: + raise SystemExit(f"Smoke check failed: /health returned {response.status_code}: {response.text}") + + db_path = _sqlite_database_path(state.database_url) + if state.bootstrap_db_path is not None: + if db_path is None or not db_path.exists() or db_path.stat().st_size <= 0: + raise SystemExit(f"Smoke check failed: expected initialized SQLite DB at {state.bootstrap_db_path}") + print("Smoke check: OK") + + def main(argv: Sequence[str] | None = None) -> int: args = parse_args(argv) if args.config: os.environ["GOVOPLAN_SERVER_CONFIG"] = args.config config_path = args.config or os.getenv("GOVOPLAN_SERVER_CONFIG") - runtime_root = apply_runtime_defaults(config_path) - config = load_server_config(config_path) - registry = build_platform_registry(config.enabled_modules, manifest_factories=config.manifest_factories) - reload_dirs = build_reload_dirs(config, config_path=config_path, registry=registry, extra_dirs=args.reload_dir) + state = prepare_devserver(config_path, extra_reload_dirs=args.reload_dir) + print_devserver_summary(state, app=args.app, no_reload=args.no_reload) - print(f"Starting GovOPlaN dev server: {args.app}") - print(f"Config: {config_path or DEFAULT_CONFIG}") - if runtime_root is not None: - print(f"Runtime root: {runtime_root}") - print("Modules: " + ", ".join(manifest.id for manifest in registry.manifests())) - if args.no_reload: - print("Reload: disabled") - else: - print("Reload dirs:") - for directory in reload_dirs: - print(f" - {directory}") + if args.smoke: + run_smoke_check(state) + return 0 try: import uvicorn @@ -213,7 +330,7 @@ def main(argv: Sequence[str] | None = None) -> int: host=args.host, port=args.port, reload=not args.no_reload, - reload_dirs=reload_dirs if not args.no_reload else None, + reload_dirs=state.reload_dirs if not args.no_reload else None, ) return 0 diff --git a/src/govoplan_core/mail/__init__.py b/src/govoplan_core/mail/__init__.py new file mode 100644 index 0000000..2124ddb --- /dev/null +++ b/src/govoplan_core/mail/__init__.py @@ -0,0 +1,3 @@ +from govoplan_core.mail.config import ImapConfig, SmtpConfig, TransportSecurity + +__all__ = ["ImapConfig", "SmtpConfig", "TransportSecurity"] diff --git a/src/govoplan_core/mail/config.py b/src/govoplan_core/mail/config.py new file mode 100644 index 0000000..5110403 --- /dev/null +++ b/src/govoplan_core/mail/config.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +from enum import StrEnum +from typing import Any + +from pydantic import BaseModel, ConfigDict, Field, model_validator + + +class StrictModel(BaseModel): + model_config = ConfigDict(extra="forbid", populate_by_name=True) + + +class TransportSecurity(StrEnum): + PLAIN = "plain" + TLS = "tls" + STARTTLS = "starttls" + + +class TransportCredentials(StrictModel): + username: str | None = None + password: str | None = None + + +class SmtpServerConfig(StrictModel): + host: str | None = None + port: int | None = Field(default=None, ge=1, le=65535) + security: TransportSecurity = TransportSecurity.STARTTLS + timeout_seconds: int = Field(default=30, ge=1) + + @model_validator(mode="after") + def apply_default_port(self) -> "SmtpServerConfig": + if self.port is None: + self.port = default_smtp_port(self.security) + return self + + +class ImapServerConfig(StrictModel): + host: str | None = None + port: int | None = Field(default=None, ge=1, le=65535) + security: TransportSecurity = TransportSecurity.TLS + sent_folder: str = "auto" + timeout_seconds: int = Field(default=30, ge=1) + + @model_validator(mode="before") + @classmethod + def discard_legacy_enabled(cls, value: Any) -> Any: + if isinstance(value, dict) and "enabled" in value: + data = dict(value) + data.pop("enabled", None) + return data + return value + + @model_validator(mode="after") + def apply_default_port(self) -> "ImapServerConfig": + if self.port is None: + self.port = default_imap_port(self.security) + return self + + +class SmtpConfig(SmtpServerConfig): + username: str | None = None + password: str | None = None + + +class ImapConfig(ImapServerConfig): + username: str | None = None + password: str | None = None + + +def default_smtp_port(security: TransportSecurity | str | None) -> int: + if security == TransportSecurity.TLS or security == "tls": + return 465 + if security == TransportSecurity.PLAIN or security == "plain": + return 25 + return 587 + + +def default_imap_port(security: TransportSecurity | str | None) -> int: + if security == TransportSecurity.TLS or security == "tls": + return 993 + return 143 diff --git a/src/govoplan_core/privacy/retention.py b/src/govoplan_core/privacy/retention.py index 071c071..479aeb4 100644 --- a/src/govoplan_core/privacy/retention.py +++ b/src/govoplan_core/privacy/retention.py @@ -11,8 +11,8 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator from sqlalchemy.orm import Session from govoplan_core.admin.governance import get_system_settings +from govoplan_core.core.optional import reraise_unless_missing_package from govoplan_core.db.models import AuditLog, Group, SystemSettings, Tenant, User -from govoplan_campaign.backend.db.models import Campaign, CampaignJob, CampaignVersion, JobImapStatus, JobQueueStatus, JobSendStatus from govoplan_core.settings import settings PRIVACY_POLICY_SETTINGS_KEY = "privacy_retention_policy" @@ -57,11 +57,11 @@ FINAL_VERSION_STATES = { "archived", } FINAL_EML_SEND_STATUSES = { - JobSendStatus.SMTP_ACCEPTED.value, - JobSendStatus.SENT.value, - JobSendStatus.FAILED_PERMANENT.value, - JobSendStatus.CANCELLED.value, - JobSendStatus.OUTCOME_UNKNOWN.value, + "smtp_accepted", + "sent", + "failed_permanent", + "cancelled", + "outcome_unknown", } AUDIT_DETAIL_REDACTION_KEYS = { "attachments", @@ -148,6 +148,22 @@ class PrivacyPolicyError(RuntimeError): pass +def _campaign_models(): + try: + from govoplan_campaign.backend.db.models import Campaign, CampaignJob, CampaignVersion, JobImapStatus, JobQueueStatus + except ModuleNotFoundError as exc: + reraise_unless_missing_package(exc, "govoplan_campaign") + raise PrivacyPolicyError("Campaign module is not installed") from exc + return Campaign, CampaignJob, CampaignVersion, JobImapStatus, JobQueueStatus + + +def _campaign_models_or_none(): + try: + return _campaign_models() + except PrivacyPolicyError: + return None + + def _utcnow() -> datetime: return datetime.now(timezone.utc) @@ -251,8 +267,9 @@ def effective_privacy_policy( campaign_id: str | None = None, ) -> PrivacyRetentionPolicy: policy = privacy_policy_from_session(session) - campaign: Campaign | None = None + campaign: Any | None = None if campaign_id: + Campaign, _, _, _, _ = _campaign_models() campaign = session.get(Campaign, campaign_id) if campaign is None: raise PrivacyPolicyError("Campaign not found for privacy policy") @@ -290,6 +307,7 @@ def parent_privacy_policy(session: Session, *, tenant_id: str, scope_type: str, return policy if clean_scope != "campaign" or not scope_id: return policy + Campaign, _, _, _, _ = _campaign_models() campaign = session.get(Campaign, scope_id) if campaign is None or campaign.tenant_id != tenant_id: raise PrivacyPolicyError("Campaign not found for privacy policy") @@ -318,11 +336,13 @@ def _retention_policy_source_fields(patch: dict[str, Any], *, baseline: bool = F def _retention_policy_source_step(scope_type: str, label: str, scope_id: str | None, patch: dict[str, Any], *, baseline: bool = False) -> dict[str, Any]: + source_policy = PrivacyRetentionPolicy.model_validate(patch).model_dump(mode="json") if baseline else dict(patch) return { "scope_type": scope_type, "scope_id": scope_id, "label": label, "applied_fields": _retention_policy_source_fields(patch, baseline=baseline), + "policy": source_policy, } @@ -336,8 +356,9 @@ def effective_privacy_policy_sources( ) -> list[dict[str, Any]]: system_settings = get_system_settings(session) sources = [_retention_policy_source_step("system", "System", None, _privacy_policy_patch_from_settings(system_settings.settings or {}), baseline=True)] - campaign: Campaign | None = None + campaign: Any | None = None if campaign_id: + Campaign, _, _, _, _ = _campaign_models() campaign = session.get(Campaign, campaign_id) if campaign is None: raise PrivacyPolicyError("Campaign not found for privacy policy") @@ -378,6 +399,7 @@ def parent_privacy_policy_sources(session: Session, *, tenant_id: str, scope_typ return sources if clean_scope != "campaign" or not scope_id: return sources + Campaign, _, _, _, _ = _campaign_models() campaign = session.get(Campaign, scope_id) if campaign is None or campaign.tenant_id != tenant_id: raise PrivacyPolicyError("Campaign not found for privacy policy") @@ -413,6 +435,7 @@ def get_privacy_policy_for_scope(session: Session, *, tenant_id: str, scope_type raise PrivacyPolicyError("Group privacy policy not found") return _privacy_policy_patch_from_settings(group.settings or {}) if clean_scope == "campaign": + Campaign, _, _, _, _ = _campaign_models() campaign = session.get(Campaign, scope_id) if campaign is None or campaign.tenant_id != tenant_id: raise PrivacyPolicyError("Campaign privacy policy not found") @@ -460,6 +483,7 @@ def set_privacy_policy_for_scope( session.add(group) return patch if clean_scope == "campaign": + Campaign, _, _, _, _ = _campaign_models() campaign = session.get(Campaign, scope_id) if campaign is None or campaign.tenant_id != tenant_id: raise PrivacyPolicyError("Campaign privacy policy not found") @@ -542,6 +566,10 @@ def _campaign_policy_for_id(session: Session, campaign_id: str | None, cache: di def _apply_raw_json_retention(session: Session, *, dry_run: bool, now: datetime) -> dict[str, int]: result = {"eligible": 0, "redacted": 0, "skipped_not_final": 0, "already_redacted": 0} + models = _campaign_models_or_none() + if models is None: + return result + _, _, CampaignVersion, _, _ = models policy_cache: dict[str, PrivacyRetentionPolicy] = {} versions = ( session.query(CampaignVersion) @@ -579,6 +607,10 @@ def _apply_raw_json_retention(session: Session, *, dry_run: bool, now: datetime) def _apply_eml_retention(session: Session, *, dry_run: bool, now: datetime) -> dict[str, int]: result = {"eligible": 0, "metadata_cleared": 0, "files_deleted": 0, "files_missing": 0, "skipped_not_final": 0} + models = _campaign_models_or_none() + if models is None: + return result + _, CampaignJob, _, JobImapStatus, JobQueueStatus = models policy_cache: dict[str, PrivacyRetentionPolicy] = {} jobs = ( session.query(CampaignJob) @@ -616,6 +648,10 @@ def _apply_eml_retention(session: Session, *, dry_run: bool, now: datetime) -> d def _apply_report_detail_retention(session: Session, *, dry_run: bool, now: datetime) -> dict[str, int]: result = {"eligible_versions": 0, "summaries_redacted": 0, "already_redacted": 0} + models = _campaign_models_or_none() + if models is None: + return result + _, _, CampaignVersion, _, _ = models policy_cache: dict[str, PrivacyRetentionPolicy] = {} versions = ( session.query(CampaignVersion) @@ -708,9 +744,12 @@ def _parse_datetime(value: Any) -> datetime | None: def _privacy_policy_for_audit_item(session: Session, item: AuditLog, campaign_cache: dict[str, PrivacyRetentionPolicy], tenant_cache: dict[str, PrivacyRetentionPolicy]) -> PrivacyRetentionPolicy: if item.object_type == "campaign" and item.object_id: - campaign = session.get(Campaign, str(item.object_id)) - if campaign is not None: - return _campaign_policy_for_id(session, campaign.id, campaign_cache) + models = _campaign_models_or_none() + if models is not None: + Campaign, _, _, _, _ = models + campaign = session.get(Campaign, str(item.object_id)) + if campaign is not None: + return _campaign_policy_for_id(session, campaign.id, campaign_cache) if item.tenant_id: if item.tenant_id not in tenant_cache: tenant_cache[item.tenant_id] = effective_privacy_policy(session, tenant_id=item.tenant_id) diff --git a/src/govoplan_core/server/app.py b/src/govoplan_core/server/app.py index 6ac8df4..98eb8e1 100644 --- a/src/govoplan_core/server/app.py +++ b/src/govoplan_core/server/app.py @@ -27,11 +27,12 @@ def create_app(config: GovoplanServerConfig | str | None = None): api_router.include_router(router) module_context = ModuleContext(registry=registry, settings=server_config.settings, data=server_config.module_context_data) + registry.configure_capability_context(module_context) for manifest in registry.manifests(): if manifest.route_factory is not None: api_router.include_router(manifest.route_factory(module_context)) - api_router.include_router(create_platform_router()) + api_router.include_router(create_platform_router(settings=server_config.settings)) for router in server_config.post_module_routers: api_router.include_router(router) diff --git a/src/govoplan_core/server/platform.py b/src/govoplan_core/server/platform.py index d8f3652..d26d907 100644 --- a/src/govoplan_core/server/platform.py +++ b/src/govoplan_core/server/platform.py @@ -48,7 +48,16 @@ def _frontend_payload(frontend: FrontendModule | None) -> dict[str, object] | No } -def create_platform_router() -> APIRouter: +def _runtime_ui_capabilities(manifest_id: str, settings: object | None, registry: PlatformRegistry) -> list[str]: + capabilities: list[str] = [] + if manifest_id == "mail": + app_env = str(getattr(settings, "app_env", "dev")).lower() + if app_env == "dev" and bool(getattr(settings, "dev_mailbox_api_enabled", False)) and registry.has_module("mail"): + capabilities.append("mail.devMailbox") + return capabilities + + +def create_platform_router(settings: object | None = None) -> APIRouter: router = APIRouter(prefix="/platform", tags=["platform"]) @router.get("/modules") @@ -63,6 +72,7 @@ def create_platform_router() -> APIRouter: "dependencies": list(manifest.dependencies), "optional_dependencies": list(manifest.optional_dependencies), "enabled": True, + "runtime_ui_capabilities": _runtime_ui_capabilities(manifest.id, settings, registry), "nav": [_nav_item_payload(item) for item in manifest.nav_items], "frontend": _frontend_payload(manifest.frontend), } diff --git a/src/govoplan_core/server/registry.py b/src/govoplan_core/server/registry.py index 0565c41..d4e668b 100644 --- a/src/govoplan_core/server/registry.py +++ b/src/govoplan_core/server/registry.py @@ -18,8 +18,8 @@ def parse_enabled_modules(value: str | Iterable[str]) -> list[str]: return [item for item in items if item] -def available_module_manifests(manifest_factories: Sequence[ManifestFactory] = ()) -> dict[str, ModuleManifest]: - manifests = {manifest.id: manifest for manifest in discover_module_manifests()} +def available_module_manifests(manifest_factories: Sequence[ManifestFactory] = (), *, enabled_modules: Iterable[str] | None = None) -> dict[str, ModuleManifest]: + manifests = {manifest.id: manifest for manifest in discover_module_manifests(enabled_modules=enabled_modules)} manifests.setdefault("access", get_access_manifest()) for factory in manifest_factories: manifest = factory() @@ -32,7 +32,7 @@ def build_platform_registry(enabled_modules: str | Iterable[str], *, manifest_fa if "access" not in requested: requested.insert(0, "access") - available = available_module_manifests(manifest_factories) + available = available_module_manifests(manifest_factories, enabled_modules=requested) registry = PlatformRegistry() for module_id in requested: manifest = available.get(module_id) diff --git a/src/govoplan_core/settings.py b/src/govoplan_core/settings.py index 8e55651..e041f4b 100644 --- a/src/govoplan_core/settings.py +++ b/src/govoplan_core/settings.py @@ -8,7 +8,7 @@ class Settings(BaseSettings): app_env: str = Field(default="dev", alias="APP_ENV") database_url: str = Field( - default="sqlite:///./multimailer-dev.db", + default="sqlite:///./runtime/multimailer-dev.db", alias="DATABASE_URL", ) access_database_url: str | None = Field(default=None, alias="ACCESS_DATABASE_URL") diff --git a/tests/test_api_smoke.py b/tests/test_api_smoke.py index 87a1d21..c103e52 100644 --- a/tests/test_api_smoke.py +++ b/tests/test_api_smoke.py @@ -215,19 +215,18 @@ class ApiSmokeTests(unittest.TestCase): "smtp": { "host": "mock.smtp", "port": 2525, - "username": "sender@example.org", - "password": "smtp-secret", "security": "starttls", }, "imap": { - "enabled": True, "host": "mock.imap", "port": 993, - "username": "sender@example.org", - "password": "imap-secret", "security": "tls", "sent_folder": "Sent", }, + "credentials": { + "smtp": {"username": "sender@example.org", "password": "smtp-secret"}, + "imap": {"username": "sender@example.org", "password": "imap-secret"}, + }, }, ) self.assertEqual(created_profile.status_code, 201, created_profile.text) @@ -264,25 +263,28 @@ class ApiSmokeTests(unittest.TestCase): "smtp": { "host": "mock.smtp", "port": 2525, - "username": "sender@example.org", - "password": "smtp-secret", "security": "starttls", }, "imap": { - "enabled": True, "host": "mock.imap", "port": 993, - "username": "sender@example.org", - "password": "imap-secret", "security": "tls", "sent_folder": "Sent", }, + "credentials": { + "smtp": {"username": "sender@example.org", "password": "smtp-secret"}, + "imap": {"username": "sender@example.org", "password": "imap-secret"}, + }, }, ) self.assertEqual(created_profile.status_code, 201, created_profile.text) profile_payload = created_profile.json() self.assertNotIn("password", profile_payload["smtp"]) + self.assertNotIn("username", profile_payload["smtp"]) self.assertNotIn("password", profile_payload["imap"]) + self.assertNotIn("username", profile_payload["imap"]) + self.assertEqual(profile_payload["credentials"]["smtp"]["username"], "sender@example.org") + self.assertEqual(profile_payload["credentials"]["imap"]["username"], "sender@example.org") profile_id = profile_payload["id"] campaign_json = { @@ -325,8 +327,12 @@ class ApiSmokeTests(unittest.TestCase): assert profile is not None self.assertNotEqual(profile.smtp_password_encrypted, "smtp-secret") self.assertNotEqual(profile.imap_password_encrypted, "imap-secret") + self.assertEqual(profile.smtp_username, "sender@example.org") + self.assertEqual(profile.imap_username, "sender@example.org") self.assertNotIn("password", profile.smtp_config) + self.assertNotIn("username", profile.smtp_config) self.assertNotIn("password", profile.imap_config or {}) + self.assertNotIn("username", profile.imap_config or {}) version = session.get(CampaignVersion, version_id) self.assertIsNotNone(version) @@ -346,19 +352,18 @@ class ApiSmokeTests(unittest.TestCase): "smtp": { "host": "mock.smtp", "port": 2525, - "username": "profile-smtp@example.org", - "password": "profile-smtp-secret", "security": "starttls", }, "imap": { - "enabled": True, "host": "mock.imap", "port": 993, - "username": "profile-imap@example.org", - "password": "profile-imap-secret", "security": "tls", "sent_folder": "Sent", }, + "credentials": { + "smtp": {"username": "profile-smtp@example.org", "password": "profile-smtp-secret"}, + "imap": {"username": "profile-imap@example.org", "password": "profile-imap-secret"}, + }, }, ) self.assertEqual(created_profile.status_code, 201, created_profile.text) @@ -373,8 +378,10 @@ class ApiSmokeTests(unittest.TestCase): "mail_profile_id": profile_id, "inherit_smtp_credentials": False, "inherit_imap_credentials": False, - "smtp": {"username": "campaign-smtp@example.org", "password": "campaign-smtp-secret"}, - "imap": {"username": "campaign-imap@example.org", "password": "campaign-imap-secret"}, + "credentials": { + "smtp": {"username": "campaign-smtp@example.org", "password": "campaign-smtp-secret"}, + "imap": {"username": "campaign-imap@example.org", "password": "campaign-imap-secret"}, + }, }, "recipients": { "from": {"email": "sender@example.org", "name": "Sender", "type": "to"}, @@ -2713,10 +2720,11 @@ class ApiSmokeTests(unittest.TestCase): "smtp": { "host": "mock.smtp", "port": 2525, - "username": "profile-smtp@example.org", - "password": "profile-smtp-secret", "security": "starttls", }, + "credentials": { + "smtp": {"username": "profile-smtp@example.org", "password": "profile-smtp-secret"}, + }, }, ) self.assertEqual(profile.status_code, 201, profile.text) @@ -2725,10 +2733,14 @@ class ApiSmokeTests(unittest.TestCase): policy = self.client.put( "/api/v1/mail/policies/tenant", headers=headers, - json={"policy": {"smtp_credentials": {"inherit": True, "allow_override": False}}}, + json={"policy": { + "smtp_credentials": {"inherit": True}, + "allow_lower_level_limits": {"smtp_credentials.inherit": False}, + }}, ) self.assertEqual(policy.status_code, 200, policy.text) - self.assertTrue(policy.json()["effective_policy"] is None) + self.assertEqual(policy.json()["effective_policy"]["smtp_credentials"]["inherit"], True) + self.assertEqual(policy.json()["effective_policy"]["allow_lower_level_limits"]["smtp_credentials.inherit"], False) campaign_json = { "version": "1.0", @@ -2738,7 +2750,9 @@ class ApiSmokeTests(unittest.TestCase): "server": { "mail_profile_id": profile_id, "inherit_smtp_credentials": False, - "smtp": {"username": "campaign-smtp@example.org", "password": "campaign-smtp-secret"}, + "credentials": { + "smtp": {"username": "campaign-smtp@example.org", "password": "campaign-smtp-secret"}, + }, }, "recipients": { "from": {"email": "sender@example.org", "name": "Sender", "type": "to"}, diff --git a/tests/test_database_migrations.py b/tests/test_database_migrations.py index 9e66024..d18b057 100644 --- a/tests/test_database_migrations.py +++ b/tests/test_database_migrations.py @@ -12,6 +12,7 @@ from govoplan_core.db.base import Base from govoplan_core.db.migrations import ( REVISION_AUTH_RBAC, REVISION_FILE_FOLDERS, + REVISION_HIERARCHICAL_SETTINGS, alembic_config, migrate_database, ) @@ -151,6 +152,85 @@ class DatabaseMigrationTests(unittest.TestCase): self.assertEqual(system_owner_count, 1 if user_count else 0) finally: engine.dispose() + + def test_repairs_current_schema_stamped_at_file_folders(self) -> None: + with tempfile.TemporaryDirectory(prefix="msm-current-schema-marker-test-") as directory: + database = Path(directory) / "current-schema-marker.db" + url = f"sqlite:///{database}" + command.upgrade(alembic_config(database_url=url), REVISION_HIERARCHICAL_SETTINGS) + + engine = create_engine(url) + try: + with engine.begin() as connection: + connection.execute( + text( + """ + INSERT INTO mail_server_profiles + (id, tenant_id, scope_type, scope_id, name, slug, description, is_active, + smtp_config, smtp_password_encrypted, imap_config, imap_password_encrypted, + created_by_user_id, updated_by_user_id, created_at, updated_at) + VALUES + ('profile-1', NULL, 'system', 'system', 'System Mail', 'system-mail', NULL, 1, + :smtp_config, NULL, :imap_config, NULL, + NULL, NULL, '2026-06-25 15:00:00', '2026-06-25 15:00:00') + """ + ), + { + "smtp_config": '{"host":"smtp.example.test","port":587,"security":"starttls","username":"smtp-user","enabled":true}', + "imap_config": '{"host":"imap.example.test","port":993,"security":"ssl","username":"imap-user","enabled":true}', + }, + ) + connection.execute( + text("UPDATE alembic_version SET version_num = :revision"), + {"revision": REVISION_FILE_FOLDERS}, + ) + with engine.connect() as connection: + self.assertEqual( + MigrationContext.configure(connection).get_current_revision(), + REVISION_FILE_FOLDERS, + ) + profile_columns = { + column["name"] + for column in inspect(connection).get_columns("mail_server_profiles") + } + self.assertNotIn("smtp_username", profile_columns) + self.assertNotIn("imap_username", profile_columns) + finally: + engine.dispose() + + result = migrate_database(database_url=url) + self.assertEqual(result.previous_revision, REVISION_FILE_FOLDERS) + self.assertEqual(result.reconciled_revision, REVISION_HIERARCHICAL_SETTINGS) + + engine = create_engine(url) + try: + with engine.connect() as connection: + self.assertEqual( + MigrationContext.configure(connection).get_current_revision(), + result.current_revision, + ) + profile_columns = { + column["name"] + for column in inspect(connection).get_columns("mail_server_profiles") + } + profile = connection.execute(text( + """ + SELECT smtp_username, smtp_config, imap_username, imap_config + FROM mail_server_profiles + WHERE id = 'profile-1' + """ + )).mappings().one() + self.assertIn("smtp_username", profile_columns) + self.assertIn("imap_username", profile_columns) + self.assertEqual(profile["smtp_username"], "smtp-user") + self.assertEqual(profile["imap_username"], "imap-user") + self.assertNotIn("username", profile["smtp_config"]) + self.assertNotIn("enabled", profile["smtp_config"]) + self.assertNotIn("username", profile["imap_config"]) + self.assertNotIn("enabled", profile["imap_config"]) + finally: + engine.dispose() + def test_migrates_legacy_login_identity_and_bootstraps_system_owner(self) -> None: with tempfile.TemporaryDirectory(prefix="msm-account-migration-test-") as directory: database = Path(directory) / "accounts.db" diff --git a/tests/test_devserver.py b/tests/test_devserver.py new file mode 100644 index 0000000..3dbbde6 --- /dev/null +++ b/tests/test_devserver.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +import os +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + + +class DevserverSmokeTests(unittest.TestCase): + def test_smoke_mode_bootstraps_missing_local_sqlite_database(self) -> None: + repo_root = Path(__file__).resolve().parents[1] + src_root = repo_root / "src" + with tempfile.TemporaryDirectory(prefix="govoplan-devserver-smoke-") as directory: + runtime_root = Path(directory) + env = os.environ.copy() + for key in ( + "DATABASE_URL", + "DEV_BOOTSTRAP_ENABLED", + "GOVOPLAN_SERVER_CONFIG", + "FILE_STORAGE_LOCAL_ROOT", + "MOCK_MAILBOX_DIR", + ): + env.pop(key, None) + env["APP_ENV"] = "dev" + env["CELERY_ENABLED"] = "false" + env["ENABLED_MODULES"] = "access" + env["PYTHONPATH"] = str(src_root) + (os.pathsep + env["PYTHONPATH"] if env.get("PYTHONPATH") else "") + + result = subprocess.run( + [sys.executable, "-m", "govoplan_core.devserver", "--smoke", "--no-reload"], + cwd=runtime_root, + env=env, + text=True, + capture_output=True, + check=False, + ) + + self.assertEqual(result.returncode, 0, result.stderr or result.stdout) + db_path = runtime_root / "runtime" / "multimailer-dev.db" + self.assertTrue(db_path.exists(), result.stdout) + self.assertGreater(db_path.stat().st_size, 0, result.stdout) + self.assertIn(f"Runtime root: {runtime_root}", result.stdout) + self.assertIn(f"Database: sqlite:///{db_path}", result.stdout) + self.assertIn("Dev bootstrap for missing SQLite DB: enabled", result.stdout) + self.assertIn("Smoke check: OK", result.stdout) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_module_system.py b/tests/test_module_system.py new file mode 100644 index 0000000..63f064c --- /dev/null +++ b/tests/test_module_system.py @@ -0,0 +1,243 @@ +from __future__ import annotations + +import os +import shutil +import subprocess +import sys +import tempfile +import textwrap +import unittest +from pathlib import Path +from types import SimpleNamespace + +# Keep the default app import side effect from bootstrapping a development DB. +_TEST_ROOT = Path(tempfile.mkdtemp(prefix="govoplan-module-tests-")) +os.environ.setdefault("APP_ENV", "test") +os.environ.setdefault("DATABASE_URL", f"sqlite:///{_TEST_ROOT / 'module-test.db'}") +os.environ.setdefault("DEV_BOOTSTRAP_ENABLED", "false") +os.environ.setdefault("CELERY_ENABLED", "false") + +from fastapi.testclient import TestClient + +from govoplan_core.core.migrations import migration_metadata_plan +from govoplan_core.server.app import create_app +from govoplan_core.server.config import GovoplanServerConfig +from govoplan_core.server.registry import available_module_manifests, build_platform_registry +from govoplan_files.backend.storage.backends import LocalFilesystemStorageBackend, StorageBackendError + + +def _settings(root: Path) -> SimpleNamespace: + return SimpleNamespace( + app_env="test", + database_url=f"sqlite:///{root / 'test.db'}", + dev_mailbox_api_enabled=False, + file_storage_backend="local", + file_storage_local_root=str(root / "files"), + file_storage_local_fallback_roots="", + file_storage_s3_endpoint_url=None, + file_storage_s3_region=None, + file_storage_s3_bucket=None, + file_storage_s3_access_key_id=None, + file_storage_s3_secret_access_key=None, + s3_endpoint_url=None, + s3_region=None, + s3_bucket=None, + s3_access_key_id=None, + s3_secret_access_key=None, + file_upload_max_bytes=1024 * 1024, + file_upload_zip_max_bytes=10 * 1024 * 1024, + celery_enabled=False, + redis_url="redis://127.0.0.1:6379/15", + mock_mailbox_dir=str(root / "mock-mailbox"), + ) + + +class ModuleSystemTests(unittest.TestCase): + @classmethod + def tearDownClass(cls) -> None: + shutil.rmtree(_TEST_ROOT, ignore_errors=True) + + def _app_for_modules(self, modules: tuple[str, ...]): + root = Path(tempfile.mkdtemp(prefix="govoplan-module-app-", dir=_TEST_ROOT)) + settings = _settings(root) + config = GovoplanServerConfig( + title="GovOPlaN Module Test", + version="test", + settings=settings, + enabled_modules=modules, + base_routers=(), + post_module_routers=(), + extra_routers=(), + lifespan=None, + app_configurators=(), + ) + return create_app(config), settings + + def test_discovers_installed_core_and_product_module_manifests(self) -> None: + manifests = available_module_manifests() + self.assertTrue({"access", "files", "mail", "campaigns"}.issubset(manifests)) + self.assertEqual(manifests["campaigns"].dependencies, ("access",)) + self.assertEqual(manifests["campaigns"].optional_dependencies, ("files", "mail")) + + def test_enabled_module_permutations_register_expected_routes(self) -> None: + cases = ( + ("core_only", (), {"access"}, set()), + ("files_only", ("files",), {"access", "files"}, {"/api/v1/files"}), + ("mail_only", ("mail",), {"access", "mail"}, {"/api/v1/mail"}), + ("campaign_without_files_or_mail", ("campaigns",), {"access", "campaigns"}, {"/api/v1/campaigns"}), + ("campaign_without_files", ("campaigns", "mail"), {"access", "campaigns", "mail"}, {"/api/v1/campaigns", "/api/v1/mail"}), + ("campaign_with_files_but_no_mail", ("campaigns", "files"), {"access", "campaigns", "files"}, {"/api/v1/campaigns", "/api/v1/files"}), + ("full_product", ("campaigns", "files", "mail"), {"access", "campaigns", "files", "mail"}, {"/api/v1/campaigns", "/api/v1/files", "/api/v1/mail"}), + ) + for name, enabled_modules, expected_modules, expected_prefixes in cases: + with self.subTest(name=name): + app, _settings_obj = self._app_for_modules(enabled_modules) + route_paths = {getattr(route, "path", "") for route in app.routes} + self.assertIn("/api/v1/platform/modules", route_paths) + for prefix in ("/api/v1/campaigns", "/api/v1/files", "/api/v1/mail"): + has_prefix = any(path == prefix or path.startswith(f"{prefix}/") for path in route_paths) + self.assertEqual(prefix in expected_prefixes, has_prefix, f"{name}: {prefix}") + + with TestClient(app) as client: + response = client.get("/api/v1/platform/modules") + self.assertEqual(response.status_code, 200, response.text) + payload_modules = {item["id"] for item in response.json()["modules"]} + self.assertEqual(expected_modules, payload_modules) + + + def _run_physical_absence_probe(self, *, enabled_modules: tuple[str, ...], blocked_modules: tuple[str, ...]) -> None: + root = Path(tempfile.mkdtemp(prefix="govoplan-absence-probe-", dir=_TEST_ROOT)) + enabled = ",".join(("access", *enabled_modules)) if enabled_modules else "access" + script = f""" +import importlib.abc +import json +import os +import sys +from pathlib import Path + +os.environ["APP_ENV"] = "test" +os.environ["DATABASE_URL"] = {f"sqlite:///{root / 'probe.db'}"!r} +os.environ["DEV_BOOTSTRAP_ENABLED"] = "false" +os.environ["CELERY_ENABLED"] = "false" +os.environ["ENABLED_MODULES"] = {enabled!r} + +BLOCKED = {blocked_modules!r} + +class Blocker(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + for prefix in BLOCKED: + if fullname == prefix or fullname.startswith(prefix + "."): + raise ModuleNotFoundError(f"No module named {{prefix!r}}", name=prefix) + return None + +sys.meta_path.insert(0, Blocker()) + +from govoplan_core.server.app import create_app +from govoplan_core.server.config import GovoplanServerConfig +from govoplan_core.server.registry import build_platform_registry + +registry = build_platform_registry({enabled_modules!r}) +config = GovoplanServerConfig( + title="absence probe", + version="test", + enabled_modules={enabled_modules!r}, + base_routers=(), + post_module_routers=(), + extra_routers=(), + lifespan=None, + app_configurators=(), +) +app = create_app(config) +route_paths = [getattr(route, "path", "") for route in app.routes] +print(json.dumps({{"modules": [item.id for item in registry.manifests()], "routes": route_paths}}, sort_keys=True)) +""" + result = subprocess.run( + [sys.executable, "-c", script], + cwd=str(Path(__file__).resolve().parents[1]), + env=os.environ.copy(), + text=True, + capture_output=True, + check=False, + ) + if result.returncode != 0: + self.fail(result.stderr or result.stdout) + + def test_campaign_optional_integrations_follow_enabled_modules(self) -> None: + cases = ( + (("campaigns",), False, False), + (("campaigns", "mail"), False, True), + (("campaigns", "files"), True, False), + (("campaigns", "files", "mail"), True, True), + ) + for enabled_modules, files_enabled, mail_enabled in cases: + with self.subTest(enabled_modules=enabled_modules): + registry = build_platform_registry(enabled_modules) + self.assertTrue(registry.has_module("campaigns")) + self.assertEqual(files_enabled, registry.integration_enabled("campaigns", "files")) + self.assertEqual(mail_enabled, registry.integration_enabled("campaigns", "mail")) + + + def test_module_permutations_start_when_absent_modules_are_physically_unavailable(self) -> None: + cases = ( + ((), ("govoplan_files", "govoplan_mail", "govoplan_campaign")), + (("files",), ("govoplan_mail", "govoplan_campaign")), + (("mail",), ("govoplan_files", "govoplan_campaign")), + (("campaigns",), ("govoplan_files", "govoplan_mail")), + (("campaigns", "files"), ("govoplan_mail",)), + (("campaigns", "mail"), ("govoplan_files",)), + (("campaigns", "files", "mail"), ()), + ) + for enabled_modules, blocked_modules in cases: + with self.subTest(enabled_modules=enabled_modules, blocked_modules=blocked_modules): + self._run_physical_absence_probe(enabled_modules=enabled_modules, blocked_modules=blocked_modules) + + def test_module_route_factories_receive_runtime_settings(self) -> None: + app, settings = self._app_for_modules(("files", "mail")) + self.assertIsNotNone(app) + + from govoplan_files.backend.runtime import get_settings as get_files_settings + from govoplan_mail.backend.runtime import get_settings as get_mail_settings + + self.assertIs(settings, get_files_settings()) + self.assertIs(settings, get_mail_settings()) + + def test_module_migrations_are_registered_only_for_enabled_modules(self) -> None: + core_plan = migration_metadata_plan(build_platform_registry(())) + self.assertEqual((), core_plan.script_locations) + self.assertEqual(1, len(core_plan.metadata)) + + files_plan = migration_metadata_plan(build_platform_registry(("files",))) + self.assertEqual(1, len(files_plan.script_locations)) + self.assertTrue(files_plan.script_locations[0].endswith("govoplan_files/backend/migrations/versions")) + + full_plan = migration_metadata_plan(build_platform_registry(("campaigns", "files", "mail"))) + locations = "\n".join(full_plan.script_locations) + self.assertIn("govoplan_campaign/backend/migrations/versions", locations) + self.assertIn("govoplan_files/backend/migrations/versions", locations) + self.assertIn("govoplan_mail/backend/migrations/versions", locations) + + def test_local_storage_backend_reads_from_fallback_roots_without_copying(self) -> None: + with tempfile.TemporaryDirectory(prefix="govoplan-storage-fallback-") as directory: + root = Path(directory) / "primary" + fallback = Path(directory) / "fallback" + key = "tenant-a/files/demo.txt" + (fallback / key).parent.mkdir(parents=True) + (fallback / key).write_bytes(b"fallback-data") + + backend = LocalFilesystemStorageBackend(root=root, fallback_roots=(fallback,)) + + self.assertTrue(backend.exists(key)) + self.assertEqual(b"fallback-data", backend.get_bytes(key)) + self.assertEqual([b"fallback", b"-data"], list(backend.iter_bytes(key, chunk_size=8))) + self.assertFalse((root / key).exists()) + + backend.delete(key) + self.assertTrue((fallback / key).exists()) + self.assertEqual(b"fallback-data", backend.get_bytes(key)) + + with self.assertRaises(StorageBackendError): + backend.get_bytes("../escape.txt") + + +if __name__ == "__main__": + unittest.main() diff --git a/webui/package.json b/webui/package.json index 5d5a1ff..baebcc1 100644 --- a/webui/package.json +++ b/webui/package.json @@ -19,7 +19,10 @@ "scripts": { "dev": "vite --host 127.0.0.1 --port 5173", "build": "tsc && vite build", - "preview": "vite preview --host 127.0.0.1 --port 4173" + "preview": "vite preview --host 127.0.0.1 --port 4173", + "test:module-capabilities": "rm -rf .module-test-build && mkdir -p .module-test-build && printf '{\"type\":\"commonjs\"}\n' > .module-test-build/package.json && tsc -p tsconfig.module-tests.json && node .module-test-build/tests/module-capabilities.test.js", + "test:module-permutations": "node scripts/test-module-permutations.mjs", + "test:mail-components": "rm -rf .component-test-build && mkdir -p .component-test-build && printf '{\"type\":\"commonjs\"}\\n' > .component-test-build/package.json && tsc -p tsconfig.component-tests.json && node .component-test-build/tests/mail-components.test.js" }, "dependencies": { "@govoplan/files-webui": "file:../../govoplan-files/webui", diff --git a/webui/scripts/test-module-permutations.mjs b/webui/scripts/test-module-permutations.mjs new file mode 100644 index 0000000..79206aa --- /dev/null +++ b/webui/scripts/test-module-permutations.mjs @@ -0,0 +1,37 @@ +import { spawnSync } from "node:child_process"; + +const packageByModule = { + campaigns: "@govoplan/campaign-webui", + files: "@govoplan/files-webui", + mail: "@govoplan/mail-webui" +}; + +const cases = [ + { name: "core-only", modules: [] }, + { name: "files-only", modules: ["files"] }, + { name: "mail-only", modules: ["mail"] }, + { name: "campaign-only", modules: ["campaigns"] }, + { name: "campaign-with-files-no-mail", modules: ["campaigns", "files"] }, + { name: "campaign-with-mail-no-files", modules: ["campaigns", "mail"] }, + { name: "full-product", modules: ["campaigns", "files", "mail"] } +]; + +const npmExec = process.env.npm_execpath; +const command = npmExec ? process.execPath : (process.platform === "win32" ? "npm.cmd" : "npm"); +const baseArgs = npmExec ? [npmExec, "run", "build"] : ["run", "build"]; + +for (const testCase of cases) { + const packages = testCase.modules.map((moduleId) => packageByModule[moduleId]).join(","); + console.log(`\n== WebUI module permutation: ${testCase.name} ==`); + const result = spawnSync(command, baseArgs, { + cwd: new URL("..", import.meta.url), + env: { + ...process.env, + GOVOPLAN_WEBUI_MODULE_PACKAGES: packages + }, + stdio: "inherit" + }); + if (result.status !== 0) { + process.exit(result.status ?? 1); + } +} diff --git a/webui/src/App.tsx b/webui/src/App.tsx index a2b1b4a..b6c60d0 100644 --- a/webui/src/App.tsx +++ b/webui/src/App.tsx @@ -2,13 +2,16 @@ import { Navigate, Route, Routes } from "react-router-dom"; import { lazy, Suspense, useEffect, useMemo, useState } from "react"; import { fetchMe } from "./api/auth"; import { fetchPlatformModules } from "./api/platform"; -import { loadApiSettings, saveApiSettings } from "./api/client"; +import { AUTH_REQUIRED_EVENT, loadApiSettings, saveApiSettings, type AuthRequiredEventDetail } from "./api/client"; import type { ApiSettings, AuthInfo, LoginResponse, PlatformModuleInfo } from "./types"; import AppShell from "./layout/AppShell"; import PublicLandingPage from "./features/auth/PublicLandingPage"; +import LoginModal from "./features/auth/LoginModal"; import { PermissionBoundary } from "./components/AccessBoundary"; import { adminReadScopes } from "./utils/permissions"; import { firstAccessibleRoute, navItemsForModules, resolveInstalledWebModules, routeContributionsForModules } from "./platform/modules"; +import { PlatformModulesProvider } from "./platform/ModuleContext"; +import { UnsavedChangesProvider } from "./components/UnsavedChangesGuard"; const DashboardPage = lazy(() => import("./features/dashboard/DashboardPage")); const SettingsPage = lazy(() => import("./features/settings/SettingsPage")); @@ -19,6 +22,7 @@ export default function App() { const [auth, setAuth] = useState(null); const [checkingSession, setCheckingSession] = useState(true); const [platformModules, setPlatformModules] = useState(null); + const [reloginMessage, setReloginMessage] = useState(""); const webModules = useMemo(() => resolveInstalledWebModules(platformModules), [platformModules]); const navItems = useMemo(() => navItemsForModules(webModules), [webModules]); @@ -38,22 +42,39 @@ export default function App() { } } - function handlePublicLogin(response: LoginResponse) { + function authFromLoginResponse(response: LoginResponse): AuthInfo { const active = response.active_tenant ?? response.tenant; - updateAuth( - { - user: response.user, - tenant: active, - active_tenant: active, - tenants: response.tenants ?? [active], - scopes: response.scopes, - roles: response.roles, - groups: response.groups - }, - "" - ); + return { + user: response.user, + tenant: active, + active_tenant: active, + tenants: response.tenants ?? [active], + scopes: response.scopes, + roles: response.roles, + groups: response.groups + }; } + function handlePublicLogin(response: LoginResponse) { + updateAuth(authFromLoginResponse(response), ""); + } + + function handleRelogin(response: LoginResponse) { + updateAuth(authFromLoginResponse(response), ""); + setReloginMessage(""); + } + + useEffect(() => { + function handleAuthRequired(event: Event) { + if (!auth) return; + const detail = (event as CustomEvent).detail; + setReloginMessage(detail?.message || "Your session has expired. Sign in again to continue."); + } + + window.addEventListener(AUTH_REQUIRED_EVENT, handleAuthRequired); + return () => window.removeEventListener(AUTH_REQUIRED_EVENT, handleAuthRequired); + }, [auth]); + useEffect(() => { let cancelled = false; setCheckingSession(true); @@ -117,7 +138,9 @@ export default function App() { if (checkingSession) { return ( - + + +
GovOPlaN
@@ -125,26 +148,34 @@ export default function App() {

Please wait while the local session is verified.

-
+
+ + ); } if (!auth) { return ( - - - + + + + + + + ); } const defaultRoute = firstAccessibleRoute(auth, webModules); return ( - + + +

Loading module...

}> } /> - } /> + } /> {moduleRoutes.map((route) => ( } />
-
+ {reloginMessage && ( + setReloginMessage("")} + onLogin={handleRelogin} + /> + )} +
+ + ); } diff --git a/webui/src/api/admin.ts b/webui/src/api/admin.ts index 00bb44a..e44f021 100644 --- a/webui/src/api/admin.ts +++ b/webui/src/api/admin.ts @@ -152,6 +152,7 @@ export type PolicySourceStep = { scope_id?: string | null; label: string; applied_fields?: string[]; + policy?: PrivacyRetentionPolicyPatch | PrivacyRetentionPolicy | null; }; export type PrivacyRetentionPolicyScopeResponse = { diff --git a/webui/src/api/client.ts b/webui/src/api/client.ts index 1d77db6..2909ea5 100644 --- a/webui/src/api/client.ts +++ b/webui/src/api/client.ts @@ -6,6 +6,14 @@ const CSRF_COOKIE_NAME = import.meta.env.VITE_CSRF_COOKIE_NAME ?? "msm_csrf"; const RECENT_SAFE_REQUEST_TTL_MS = 750; const MAX_RECENT_SAFE_REQUESTS = 100; +export const AUTH_REQUIRED_EVENT = "govoplan:auth-required"; + +export type AuthRequiredEventDetail = { + path: string; + status: number; + message: string; +}; + type RecentSafeRequest = { value: unknown; expiresAt: number; @@ -169,6 +177,25 @@ export function authHeaders(settings: ApiSettings): Headers { return headers; } +function shouldNotifyAuthRequired(path: string): boolean { + const normalized = path.split("?", 1)[0].replace(/\/+$/, ""); + return !normalized.endsWith("/auth/login"); +} + +function notifyAuthRequired(path: string): void { + if (typeof window === "undefined" || !shouldNotifyAuthRequired(path)) return; + const detail: AuthRequiredEventDetail = { + path, + status: 401, + message: "Your session has expired. Sign in again to continue." + }; + window.dispatchEvent(new CustomEvent(AUTH_REQUIRED_EVENT, { detail })); +} + +function authExpiredError(statusText: string): ApiError { + return new ApiError(401, statusText || "Unauthorized", "Your session has expired. Sign in again to continue."); +} + export async function apiFetch(settings: ApiSettings, path: string, init?: RequestInit): Promise { const headers = new Headers(init?.headers || {}); const method = (init?.method || "GET").toUpperCase(); @@ -200,6 +227,10 @@ export async function apiFetch(settings: ApiSettings, path: string, init?: Re if (!response.ok) { const text = await response.text(); + if (response.status === 401 && shouldNotifyAuthRequired(path)) { + notifyAuthRequired(path); + throw authExpiredError(response.statusText); + } throw new ApiError(response.status, response.statusText, text); } @@ -252,6 +283,10 @@ export async function apiDownload(settings: ApiSettings, path: string, filename: const response = await fetch(apiUrl(settings, path), { headers: authHeaders(settings), credentials: "include" }); if (!response.ok) { const text = await response.text(); + if (response.status === 401 && shouldNotifyAuthRequired(path)) { + notifyAuthRequired(path); + throw authExpiredError(response.statusText); + } throw new ApiError(response.status, response.statusText, text); } const blob = await response.blob(); diff --git a/webui/src/app.ts b/webui/src/app.ts index 5ce8158..42c705e 100644 --- a/webui/src/app.ts +++ b/webui/src/app.ts @@ -7,7 +7,4 @@ import "./styles/components.css"; import "./styles/dialogs.css"; import "./styles/retention-policies.css"; import "./styles/auth-gate.css"; -import "@govoplan/campaign-webui/styles/campaign-workspace.css"; -import "@govoplan/files-webui/styles/file-manager.css"; -import "@govoplan/mail-webui/styles/mail-profiles.css"; export { default, default as GovoplanApp } from "./App"; diff --git a/webui/src/components/Card.tsx b/webui/src/components/Card.tsx index 22fbc8c..2da2529 100644 --- a/webui/src/components/Card.tsx +++ b/webui/src/components/Card.tsx @@ -1,19 +1,60 @@ -import { useState } from "react"; +import { useEffect, useState, type ReactNode } from "react"; import { ChevronDown } from "lucide-react"; type CardProps = { - title?: React.ReactNode; - children: React.ReactNode; - actions?: React.ReactNode; + title?: ReactNode; + children: ReactNode; + actions?: ReactNode; collapsible?: boolean; + collapseKey?: string; + persistCollapse?: boolean; }; -export default function Card({ title, children, actions, collapsible = false }: CardProps) { - const [collapsed, setCollapsed] = useState(false); +function resolveCollapseStorageKey(collapsible: boolean, persistCollapse: boolean, collapseKey: string | undefined, title: ReactNode): string | null { + if (!collapsible || !persistCollapse) return null; + if (collapseKey) return `govoplan.card.collapsed:${collapseKey}`; + if (typeof window === "undefined" || typeof title !== "string") return null; + const normalizedTitle = title.trim(); + if (!normalizedTitle) return null; + return `govoplan.card.collapsed:${window.location.pathname}:${normalizedTitle}`; +} + +function readCollapseState(storageKey: string | null): boolean { + if (!storageKey || typeof window === "undefined") return false; + try { + return window.localStorage.getItem(storageKey) === "1"; + } catch { + return false; + } +} + +function writeCollapseState(storageKey: string | null, collapsed: boolean): void { + if (!storageKey || typeof window === "undefined") return; + try { + window.localStorage.setItem(storageKey, collapsed ? "1" : "0"); + } catch { + // localStorage may be unavailable in private or restricted contexts. + } +} + +export default function Card({ title, children, actions, collapsible = false, collapseKey, persistCollapse = true }: CardProps) { + const storageKey = resolveCollapseStorageKey(collapsible, persistCollapse, collapseKey, title); + const [collapseState, setCollapseState] = useState(() => ({ storageKey, collapsed: readCollapseState(storageKey) })); + const collapsed = collapseState.storageKey === storageKey ? collapseState.collapsed : readCollapseState(storageKey); const hasHeader = Boolean(title || actions || collapsible); const body =
{children}
; const shouldRenderBody = !collapsible || !collapsed; + useEffect(() => { + setCollapseState({ storageKey, collapsed: readCollapseState(storageKey) }); + }, [storageKey]); + + function toggleCollapsed() { + const nextCollapsed = !collapsed; + writeCollapseState(storageKey, nextCollapsed); + setCollapseState({ storageKey, collapsed: nextCollapsed }); + } + return (
{hasHeader && ( @@ -29,7 +70,7 @@ export default function Card({ title, children, actions, collapsible = false }: aria-label={collapsed ? "Show content" : "Show header only"} aria-expanded={!collapsed} title={collapsed ? "Show content" : "Show header only"} - onClick={() => setCollapsed((value) => !value)} + onClick={toggleCollapsed} >