GovOPlaN/govoplan-core
1
0
Fork 0
Code Issues 43 Pull Requests Actions Packages Projects Releases Wiki Activity

Release v0.1.8

Browse Source
This commit is contained in:
Albrecht Degering
2026-07-11 17:00:37 +02:00
parent a00ef54821
commit 9a0c467d55
102 changed files with 2150 additions and 9272 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
src/govoplan_core/audit/logging.py
View File

@@ -20,30 +20,13 @@ from govoplan_core.core.events import (
)
from govoplan_core.core.runtime import get_registry
from govoplan_core.privacy.retention import sanitize_audit_details_for_policy
from govoplan_core.security.redaction import redact_secret_values
AUDIT_MODULE_ID = "audit"
AUDIT_TENANT_EVENTS_COLLECTION = "audit.admin.tenant_events"
AUDIT_SYSTEM_EVENTS_COLLECTION = "audit.admin.system_events"
SENSITIVE_DETAIL_KEYS = {
"password",
"smtp_password",
"imap_password",
"secret",
"api_key",
"token",
"access_token",
"refresh_token",
"authorization",
"cookie",
"credentials",
"credential",
"private_key",
"claim_token",
"build_token",
}
_ACTION_MODULE_PREFIXES = {
"api_key": "access",
"configuration_change": "access",
@@ -90,17 +73,7 @@ def _compact_trace(trace: Mapping[str, object] | None) -> dict[str, str]:
def _sanitize_details(value: Any) -> Any:
if isinstance(value, dict):
sanitized: dict[str, Any] = {}
for key, item in value.items():
if str(key).lower() in SENSITIVE_DETAIL_KEYS:
sanitized[key] = "<redacted>"
else:
sanitized[key] = _sanitize_details(item)
return sanitized
if isinstance(value, list):
return [_sanitize_details(item) for item in value]
return value
return redact_secret_values(value)
def audit_operation_context(

7
src/govoplan_core/commands/init_db.py
View File

@@ -19,6 +19,12 @@ from govoplan_core.settings import settings
def main() -> None:
parser = argparse.ArgumentParser(description="Initialize the GovOPlaN database")
parser.add_argument("--database-url", default=settings.database_url, help="Database URL to migrate")
parser.add_argument(
"--migration-track",
default=settings.migration_track,
choices=("release", "dev"),
help="Alembic migration track to load: release uses squashed release shortcuts; dev uses detailed development chains.",
)
parser.add_argument("--enabled-module", action="append", default=[], help="Target enabled module id used to discover module migrations; may be repeated.")
parser.add_argument("--migration-module", action="append", default=[], help="Module id whose migration heads should be upgraded in this order before final heads.")
parser.add_argument("--migration-task-record-output", type=Path, help="Write executed module migration task records to this JSON file.")
@@ -42,6 +48,7 @@ def main() -> None:
database_url=args.database_url,
enabled_modules=enabled_modules,
migration_module_order=migration_order,
migration_track=args.migration_track,
)
_run_migration_tasks(
task_records,

13
src/govoplan_core/core/configuration_control.py
View File

@@ -12,6 +12,7 @@ from govoplan_core.admin.settings import get_system_settings
from govoplan_core.core.change_sequence import record_change
from govoplan_core.core.configuration_safety import classify_configuration_field, plan_configuration_change
from govoplan_core.core.maintenance import saved_maintenance_mode
from govoplan_core.security.redaction import redact_secret_values
from govoplan_core.security.time import utc_now
@@ -377,17 +378,7 @@ def _sanitize_value(key: str, value: object) -> object:
def _redact_secrets(value: object) -> object:
if isinstance(value, dict):
result: dict[str, object] = {}
for key, item in value.items():
if str(key).strip().casefold() in {"password", "token", "secret", "api_key", "access_key", "secret_key"}:
result[str(key)] = "<redacted>"
else:
result[str(key)] = _redact_secrets(item)
return result
if isinstance(value, list):
return [_redact_secrets(item) for item in value]
return value
return redact_secret_values(value)
def _trim_requests(requests: list[dict[str, Any]]) -> list[dict[str, Any]]:

12
src/govoplan_core/core/configuration_safety.py
View File

@@ -5,6 +5,7 @@ from collections.abc import Mapping
from typing import Any, Literal
from govoplan_core.security.permissions import scopes_grant
from govoplan_core.security.redaction import contains_plain_secret
ConfigurationScope = Literal["system", "tenant", "user", "group", "campaign"]
@@ -411,13 +412,4 @@ def _policy_explanation(field: ConfigurationFieldSafety) -> str:
def _contains_plain_secret(value: object) -> bool:
if not isinstance(value, Mapping):
return False
secret_keys = {"password", "token", "secret", "api_key", "access_key", "secret_key"}
for key, item in value.items():
name = str(key).strip().casefold()
if name in secret_keys and item not in (None, "", {"secret_ref": ""}):
return True
if isinstance(item, Mapping) and _contains_plain_secret(item):
return True
return False
return contains_plain_secret(value)

153
src/govoplan_core/core/module_installer.py
View File

@@ -21,6 +21,7 @@ import time
import urllib.error
import urllib.request
from sqlalchemy.engine import make_url
from sqlalchemy.orm import Session
from govoplan_core.core.maintenance import saved_maintenance_mode
@@ -474,6 +475,8 @@ def run_module_install_plan(
database_restore_command=database_restore_command,
database_restore_check_command=database_restore_check_command,
)
record_redactions = _installer_secret_redactions(database_url)
result_commands = _command_displays(commands, redactions=record_redactions)
record: dict[str, Any] = {
"run_id": run_id,
@@ -482,7 +485,7 @@ def run_module_install_plan(
"plan": [item.as_dict() for item in plan.items],
"preflight": preflight.as_dict(),
"snapshot": snapshot,
"commands": [_command_record(command) for command in commands],
"commands": [_command_record(command, redactions=record_redactions) for command in commands],
"build_webui": build_webui,
"migrate_database": migrate_database,
"activate_installed_modules": activate_installed_modules,
@@ -493,7 +496,7 @@ def run_module_install_plan(
_write_json(record_path, record)
if dry_run:
return ModuleInstallerRunResult(run_id=run_id, status="dry-run", record_path=record_path, commands=tuple(command["display"] for command in commands))
return ModuleInstallerRunResult(run_id=run_id, status="dry-run", record_path=record_path, commands=result_commands)
executed: list[dict[str, object]] = []
failed_error: str | None = None
@@ -507,10 +510,10 @@ def run_module_install_plan(
for command in commands:
result = subprocess.run(command["argv"], cwd=command.get("cwd"), text=True, capture_output=True, check=False)
executed.append({
**_command_record(command),
**_command_record(command, redactions=record_redactions),
"return_code": result.returncode,
"stdout": result.stdout[-8000:],
"stderr": result.stderr[-8000:],
"stdout": _redact_installer_text(result.stdout[-8000:], redactions=record_redactions),
"stderr": _redact_installer_text(result.stderr[-8000:], redactions=record_redactions),
})
migration_task_record_path = command.get("migration_task_record_path")
if migration_task_record_path:
@@ -521,9 +524,9 @@ def run_module_install_plan(
record["commands"] = executed
_write_json(record_path, record)
if result.returncode != 0:
raise ModuleInstallerError(f"Command failed ({result.returncode}): {command['display']}")
raise ModuleInstallerError(f"Command failed ({result.returncode}): {_redact_installer_text(str(command['display']), redactions=record_redactions)}")
except Exception as exc:
failed_error = str(exc)
failed_error = _redact_installer_text(str(exc), redactions=record_redactions)
try:
session.rollback()
except Exception:
@@ -551,12 +554,12 @@ def run_module_install_plan(
run_id=run_id,
status="rolled-back" if rollback.return_code == 0 else "failed",
record_path=record_path,
commands=tuple(command["display"] for command in commands),
commands=result_commands,
return_code=1,
error=failed_error,
rollback=rollback.as_dict(),
)
return ModuleInstallerRunResult(run_id=run_id, status="failed", record_path=record_path, commands=tuple(command["display"] for command in commands), return_code=1, error=failed_error)
return ModuleInstallerRunResult(run_id=run_id, status="failed", record_path=record_path, commands=result_commands, return_code=1, error=failed_error)
applied_items = tuple(_mark_applied(item) for item in plan.items)
save_module_install_plan(session, applied_items)
@@ -576,7 +579,7 @@ def run_module_install_plan(
"commands": executed,
})
_write_json(record_path, record)
return ModuleInstallerRunResult(run_id=run_id, status="applied", record_path=record_path, commands=tuple(command["display"] for command in commands))
return ModuleInstallerRunResult(run_id=run_id, status="applied", record_path=record_path, commands=result_commands)
def supervise_module_install_plan(
@@ -720,6 +723,8 @@ def rollback_module_install_run(
raise ModuleInstallerError(f"Run record does not exist: {record_path}")
record = json.loads(record_path.read_text(encoding="utf-8"))
snapshot = record.get("snapshot") if isinstance(record.get("snapshot"), dict) else {}
raw_database_backup = snapshot.get("database_backup") if isinstance(snapshot.get("database_backup"), Mapping) else {}
rollback_redactions = _installer_secret_redactions(database_url or _database_url_from_external_snapshot(run_dir, raw_database_backup))
commands: list[dict[str, Any]] = []
for package in _planned_python_install_packages(record):
commands.append(_structured_command([sys.executable, "-m", "pip", "uninstall", "-y", package]))
@@ -741,15 +746,15 @@ def rollback_module_install_run(
for command in commands:
result = subprocess.run(command["argv"], cwd=command.get("cwd"), text=True, capture_output=True, check=False)
executed.append({
**_command_record(command),
**_command_record(command, redactions=rollback_redactions),
"return_code": result.returncode,
"stdout": result.stdout[-8000:],
"stderr": result.stderr[-8000:],
"stdout": _redact_installer_text(result.stdout[-8000:], redactions=rollback_redactions),
"stderr": _redact_installer_text(result.stderr[-8000:], redactions=rollback_redactions),
})
if result.returncode != 0:
record.update({"rollback_status": "failed", "rollback_commands": executed})
_write_json(record_path, record)
return ModuleInstallerRunResult(run_id=run_id, status="failed", record_path=record_path, commands=tuple(command["display"] for command in commands), return_code=1)
return ModuleInstallerRunResult(run_id=run_id, status="failed", record_path=record_path, commands=_command_displays(commands, redactions=rollback_redactions), return_code=1)
database_restore = _restore_database_snapshot(
run_dir,
snapshot,
@@ -763,7 +768,7 @@ def rollback_module_install_run(
"rollback_database_restore": database_restore,
})
_write_json(record_path, record)
return ModuleInstallerRunResult(run_id=run_id, status="failed", record_path=record_path, commands=tuple(command["display"] for command in commands), return_code=1, error=str(database_restore.get("error") or "Database restore failed."))
return ModuleInstallerRunResult(run_id=run_id, status="failed", record_path=record_path, commands=_command_displays(commands, redactions=rollback_redactions), return_code=1, error=str(database_restore.get("error") or "Database restore failed."))
record.update({
"rollback_status": "rolled-back",
@@ -773,7 +778,7 @@ def rollback_module_install_run(
if database_restore is not None:
record["rollback_database_restore"] = database_restore
_write_json(record_path, record)
return ModuleInstallerRunResult(run_id=run_id, status="rolled-back", record_path=record_path, commands=tuple(command["display"] for command in commands))
return ModuleInstallerRunResult(run_id=run_id, status="rolled-back", record_path=record_path, commands=_command_displays(commands, redactions=rollback_redactions))
def list_module_installer_runs(*, runtime_dir: Path, limit: int = 25) -> tuple[dict[str, object], ...]:
@@ -3007,6 +3012,8 @@ def _snapshot_external_database(
) -> dict[str, object]:
backup_path = run_dir / "database.external.backup"
metadata_path = run_dir / "database-backup-metadata.json"
redactions = _installer_secret_redactions(database_url)
database_url_secret = _write_installer_secret(run_dir / "database-url.secret", database_url) if database_url else None
result = _run_database_hook(
backup_command,
run_dir=run_dir,
@@ -3016,18 +3023,20 @@ def _snapshot_external_database(
)
payload: dict[str, object] = {
"type": "external",
"database_url": database_url or "",
"backup_command": backup_command,
"restore_command": restore_command,
"restore_check_command": restore_check_command,
"database_url": _redact_installer_text(database_url or "", redactions=redactions),
"backup_command": _redact_installer_text(backup_command, redactions=redactions),
"restore_command": _redact_installer_text(restore_command, redactions=redactions),
"restore_check_command": _redact_installer_text(restore_check_command, redactions=redactions) if restore_check_command else None,
"backup_path": backup_path.name,
"metadata_path": metadata_path.name,
"return_code": result.returncode,
"stdout": result.stdout[-8000:],
"stderr": result.stderr[-8000:],
"stdout": _redact_installer_text(result.stdout[-8000:], redactions=redactions),
"stderr": _redact_installer_text(result.stderr[-8000:], redactions=redactions),
}
if database_url_secret:
payload["database_url_secret"] = database_url_secret
if result.returncode != 0:
raise ModuleInstallerError(f"Database backup command failed ({result.returncode}): {backup_command}")
raise ModuleInstallerError(f"Database backup command failed ({result.returncode}): {_redact_installer_text(backup_command, redactions=redactions)}")
if restore_check_command:
restore_check = _run_database_hook(
restore_check_command,
@@ -3037,18 +3046,18 @@ def _snapshot_external_database(
metadata_path=metadata_path,
)
payload["restore_check"] = {
"command": restore_check_command,
"command": _redact_installer_text(restore_check_command, redactions=redactions),
"return_code": restore_check.returncode,
"stdout": restore_check.stdout[-8000:],
"stderr": restore_check.stderr[-8000:],
"stdout": _redact_installer_text(restore_check.stdout[-8000:], redactions=redactions),
"stderr": _redact_installer_text(restore_check.stderr[-8000:], redactions=redactions),
}
if restore_check.returncode != 0:
raise ModuleInstallerError(f"Database restore-check command failed ({restore_check.returncode}): {restore_check_command}")
raise ModuleInstallerError(f"Database restore-check command failed ({restore_check.returncode}): {_redact_installer_text(restore_check_command, redactions=redactions)}")
if metadata_path.exists():
try:
metadata = json.loads(metadata_path.read_text(encoding="utf-8"))
if isinstance(metadata, dict):
payload["metadata"] = metadata
payload["metadata"] = _redact_installer_value(metadata, redactions=redactions)
except json.JSONDecodeError as exc:
raise ModuleInstallerError(f"Database backup metadata is not valid JSON: {metadata_path}") from exc
return payload
@@ -3102,29 +3111,92 @@ def _restore_external_database_snapshot(
return {"restored": False, "type": "external", "error": "No database restore command is available for this run."}
backup_name = raw.get("backup_path")
metadata_name = raw.get("metadata_path")
raw_database_url = raw.get("database_url")
raw_database_url = _database_url_from_external_snapshot(run_dir, raw)
backup_path = run_dir / backup_name if isinstance(backup_name, str) else run_dir / "database.external.backup"
metadata_path = run_dir / metadata_name if isinstance(metadata_name, str) else run_dir / "database-backup-metadata.json"
effective_database_url = database_url or raw_database_url
redactions = _installer_secret_redactions(effective_database_url)
result = _run_database_hook(
command,
run_dir=run_dir,
database_url=database_url or (raw_database_url if isinstance(raw_database_url, str) else None),
database_url=effective_database_url,
backup_path=backup_path,
metadata_path=metadata_path,
)
payload: dict[str, object] = {
"restored": result.returncode == 0,
"type": "external",
"command": command,
"command": _redact_installer_text(command, redactions=redactions),
"return_code": result.returncode,
"stdout": result.stdout[-8000:],
"stderr": result.stderr[-8000:],
"stdout": _redact_installer_text(result.stdout[-8000:], redactions=redactions),
"stderr": _redact_installer_text(result.stderr[-8000:], redactions=redactions),
}
if result.returncode != 0:
payload["error"] = f"Database restore command failed ({result.returncode})."
return payload
def _write_installer_secret(path: Path, value: str | None) -> str | None:
if not value:
return None
path.write_text(value, encoding="utf-8")
try:
path.chmod(0o600)
except OSError:
pass
return path.name
def _database_url_from_external_snapshot(run_dir: Path, raw: Mapping[str, object]) -> str | None:
secret_name = raw.get("database_url_secret")
if isinstance(secret_name, str) and secret_name:
secret_path = run_dir / Path(secret_name).name
try:
value = secret_path.read_text(encoding="utf-8").strip()
except OSError:
value = ""
if value:
return value
value = raw.get("database_url")
if isinstance(value, str) and "<redacted>" not in value and "***" not in value:
return value
return None
def _installer_secret_redactions(database_url: str | None) -> dict[str, str]:
redactions: dict[str, str] = {}
for value in (database_url, _database_url_for_pgtools(database_url or "") if database_url else None):
if not value:
continue
redacted = _redact_url_credentials(value)
redactions[value] = redacted
return redactions
def _redact_installer_value(value: object, *, redactions: Mapping[str, str]) -> object:
if isinstance(value, str):
return _redact_installer_text(value, redactions=redactions)
if isinstance(value, Mapping):
return {str(key): _redact_installer_value(item, redactions=redactions) for key, item in value.items()}
if isinstance(value, list):
return [_redact_installer_value(item, redactions=redactions) for item in value]
return value
def _redact_installer_text(value: str, *, redactions: Mapping[str, str]) -> str:
result = value
for raw, redacted in sorted(redactions.items(), key=lambda item: len(item[0]), reverse=True):
result = result.replace(raw, redacted)
return _redact_url_credentials(result)
def _redact_url_credentials(value: str) -> str:
try:
return make_url(value).render_as_string(hide_password=True)
except Exception:
return re.sub(r"([A-Za-z][A-Za-z0-9+.-]*://[^\s/@:]+:)([^@\s]+)(@)", r"\1***\3", value)
def _run_database_hook(
command: str,
*,
@@ -3229,15 +3301,20 @@ def _quote_arg(value: str) -> str:
return shlex.quote(value)
def _command_record(command: Mapping[str, Any]) -> dict[str, object]:
def _command_displays(commands: Iterable[Mapping[str, Any]], *, redactions: Mapping[str, str]) -> tuple[str, ...]:
return tuple(_redact_installer_text(str(command["display"]), redactions=redactions) for command in commands)
def _command_record(command: Mapping[str, Any], *, redactions: Mapping[str, str] | None = None) -> dict[str, object]:
effective_redactions = redactions or {}
payload: dict[str, object] = {
"display": str(command["display"]),
"argv": [str(item) for item in command["argv"]],
"display": _redact_installer_text(str(command["display"]), redactions=effective_redactions),
"argv": [_redact_installer_text(str(item), redactions=effective_redactions) for item in command["argv"]],
}
if command.get("cwd"):
payload["cwd"] = str(command["cwd"])
payload["cwd"] = _redact_installer_text(str(command["cwd"]), redactions=effective_redactions)
if command.get("migration_task_record_path"):
payload["migration_task_record_path"] = str(command["migration_task_record_path"])
payload["migration_task_record_path"] = _redact_installer_text(str(command["migration_task_record_path"]), redactions=effective_redactions)
return payload

153
src/govoplan_core/db/migrations.py
View File

@@ -1,7 +1,7 @@
from __future__ import annotations
from collections.abc import Mapping
from dataclasses import dataclass
from dataclasses import dataclass, replace
import json
import os
from pathlib import Path
@@ -41,6 +41,9 @@ CORE_SCOPE_TABLE = "core_scopes"
PRE_MIGRATION_TASK_PHASES = ("pre_migration_check", "pre_migration_prepare")
POST_MIGRATION_TASK_PHASES = ("post_migration_backfill", "post_migration_verify")
MIGRATION_TASK_PHASES = (*PRE_MIGRATION_TASK_PHASES, *POST_MIGRATION_TASK_PHASES)
MIGRATION_TRACK_RELEASE = "release"
MIGRATION_TRACK_DEV = "dev"
MIGRATION_TRACKS = (MIGRATION_TRACK_RELEASE, MIGRATION_TRACK_DEV)
_NAMESPACE_TABLE_RENAMES = (
("tenants", "tenancy_tenants"),
@@ -179,12 +182,14 @@ def registered_module_migration_plan(
*,
enabled_modules: tuple[str, ...] | list[str] | None = None,
manifest_factories: tuple[ManifestFactory, ...] = (),
migration_track: str | None = None,
) -> MigrationMetadataPlan:
return migration_metadata_plan(_registered_module_registry(
plan = migration_metadata_plan(_registered_module_registry(
database_url=database_url,
enabled_modules=enabled_modules,
manifest_factories=manifest_factories,
))
return _migration_plan_for_track(plan, _normalize_migration_track(migration_track))
def _registered_module_registry(
@@ -357,13 +362,76 @@ def _repo_root() -> Path:
return packaged_root
def _normalize_migration_track(value: str | None = None) -> str:
raw_value = value if value is not None else settings.migration_track
track = str(raw_value or MIGRATION_TRACK_RELEASE).strip().lower()
aliases = {
"baseline": MIGRATION_TRACK_RELEASE,
"prod": MIGRATION_TRACK_RELEASE,
"production": MIGRATION_TRACK_RELEASE,
"releases": MIGRATION_TRACK_RELEASE,
"detailed": MIGRATION_TRACK_DEV,
"development": MIGRATION_TRACK_DEV,
}
track = aliases.get(track, track)
if track not in MIGRATION_TRACKS:
raise ValueError(
"Unsupported migration track "
f"{raw_value!r}; expected one of: {', '.join(MIGRATION_TRACKS)}"
)
return track
def _core_version_location(root: Path, migration_track: str) -> Path:
if migration_track == MIGRATION_TRACK_DEV:
dev_versions = root / "alembic" / "dev_versions"
if dev_versions.is_dir():
return dev_versions
return root / "alembic" / "versions"
def _script_location_for_track(location: str | None, migration_track: str) -> str | None:
if not location or migration_track != MIGRATION_TRACK_DEV:
return location
path = Path(location)
if path.name != "versions":
return location
dev_location = path.with_name("dev_versions")
if dev_location.is_dir():
return str(dev_location)
return location
def _migration_plan_for_track(plan: MigrationMetadataPlan, migration_track: str) -> MigrationMetadataPlan:
if migration_track == MIGRATION_TRACK_RELEASE:
return plan
script_locations = tuple(
location
for location in (
_script_location_for_track(location, migration_track)
for location in plan.script_locations
)
if location
)
modules = tuple(
replace(
item,
script_location=_script_location_for_track(item.script_location, migration_track),
)
for item in plan.modules
)
return replace(plan, script_locations=script_locations, modules=modules)
def alembic_config(
*,
database_url: str | None = None,
enabled_modules: tuple[str, ...] | list[str] | None = None,
manifest_factories: tuple[ManifestFactory, ...] = (),
migration_track: str | None = None,
) -> Config:
root = _repo_root()
active_migration_track = _normalize_migration_track(migration_track)
config = Config(str(root / "alembic.ini"))
config.set_main_option("script_location", str(root / "alembic"))
@@ -372,13 +440,15 @@ def alembic_config(
active_database_url,
enabled_modules=enabled_modules,
manifest_factories=manifest_factories,
migration_track=active_migration_track,
)
version_locations = [str(root / "alembic" / "versions")]
version_locations = [str(_core_version_location(root, active_migration_track))]
version_locations.extend(location for location in plan.script_locations if location)
config.set_main_option("version_locations", os.pathsep.join(dict.fromkeys(version_locations)))
config.set_main_option("version_path_separator", "os")
config.set_main_option("path_separator", "os")
config.attributes["database_url"] = active_database_url
config.attributes["migration_track"] = active_migration_track
if enabled_modules is not None:
config.attributes["enabled_modules"] = tuple(enabled_modules)
if manifest_factories:
@@ -602,7 +672,67 @@ def reconcile_change_sequence_retention_floor_drift(database_url: str | None = N
return changed
def reconcile_legacy_create_all_schema(database_url: str | None = None) -> str | None:
def reconcile_covered_alembic_dependency_heads(
database_url: str | None = None,
*,
config: Config | None = None,
) -> tuple[str, ...]:
"""Remove stale parent/dependency rows from ``alembic_version``.
Alembic's version table represents current graph heads, not every owner
head that contributed to the schema. After the v0.1.7 baseline squash,
development databases may still carry dependency parents such as the core
baseline beside child module heads. That shape is semantically already
upgraded, but Alembic rejects it as an overlapping current revision set.
"""
url = database_url or settings.database_url
active_config = config or alembic_config(database_url=url)
script = ScriptDirectory.from_config(active_config)
engine = create_engine(url)
removed: set[str] = set()
try:
with engine.begin() as connection:
tables = set(inspect(connection).get_table_names())
if "alembic_version" not in tables:
return ()
current = {
str(row[0])
for row in connection.execute(text("SELECT version_num FROM alembic_version")).all()
if row[0]
}
if len(current) < 2:
return ()
covered: set[str] = set()
for revision_id in current:
try:
revision = script.get_revision(revision_id)
except Exception:
continue
ancestors = {
item.revision
for item in script.revision_map._get_ancestor_nodes( # noqa: SLF001 - Alembic has no public dependency-ancestor API.
[revision],
include_dependencies=True,
)
}
covered.update(ancestors - {revision_id})
removed = current.intersection(covered)
for revision_id in sorted(removed):
connection.execute(
text("DELETE FROM alembic_version WHERE version_num = :revision"),
{"revision": revision_id},
)
finally:
engine.dispose()
return tuple(sorted(removed))
def reconcile_legacy_create_all_schema(
database_url: str | None = None,
*,
migration_track: str | None = None,
) -> str | None:
"""Repair the known Alembic/create_all drift without modifying application data.
Returns the revision stamped during reconciliation, or ``None`` when no
@@ -659,7 +789,7 @@ def reconcile_legacy_create_all_schema(database_url: str | None = None) -> str |
if target is None:
return None
command.stamp(alembic_config(database_url=url), target)
command.stamp(alembic_config(database_url=url, migration_track=migration_track), target)
return target
@@ -670,23 +800,32 @@ def migrate_database(
enabled_modules: tuple[str, ...] | list[str] | None = None,
migration_module_order: tuple[str, ...] | list[str] | None = None,
manifest_factories: tuple[ManifestFactory, ...] = (),
migration_track: str | None = None,
) -> MigrationResult:
url = database_url or settings.database_url
active_migration_track = _normalize_migration_track(migration_track)
if reconcile_legacy_schema:
reconcile_namespace_table_drift(url)
reconcile_change_sequence_retention_floor_drift(url)
previous = database_revision(url)
reconciled = reconcile_legacy_create_all_schema(url) if reconcile_legacy_schema else None
reconciled = (
reconcile_legacy_create_all_schema(url, migration_track=active_migration_track)
if reconcile_legacy_schema
else None
)
config = alembic_config(
database_url=url,
enabled_modules=enabled_modules,
manifest_factories=manifest_factories,
migration_track=active_migration_track,
)
reconcile_covered_alembic_dependency_heads(url, config=config)
if migration_module_order:
plan = registered_module_migration_plan(
url,
enabled_modules=enabled_modules,
manifest_factories=manifest_factories,
migration_track=active_migration_track,
)
_upgrade_ordered_module_heads(config, plan, tuple(migration_module_order))
command.upgrade(config, "heads")

4
src/govoplan_core/devserver.py
View File

@@ -22,7 +22,7 @@ DEFAULT_APP = "govoplan_core.server.app:app"
DEFAULT_CONFIG = "govoplan_core.server.default_config:get_server_config"
DEFAULT_DEV_DATABASE_URL = "postgresql+psycopg://govoplan_dev@127.0.0.1:5432/govoplan_dev"
DEFAULT_DEV_DATABASE_URL_PGTOOLS = "postgresql://govoplan_dev@127.0.0.1:5432/govoplan_dev"
DEFAULT_SQLITE_DATABASE_NAME = "multimailer-dev.db"
DEFAULT_SQLITE_DATABASE_NAME = "govoplan-dev.db"
@dataclass(slots=True)
@@ -352,7 +352,7 @@ def main(argv: Sequence[str] | None = None) -> int:
try:
import uvicorn
except ModuleNotFoundError as exc:
raise SystemExit("uvicorn is not installed. Install govoplan-core with the server extra or requirements-dev.txt.") from exc
raise SystemExit("uvicorn is not installed. Install govoplan-core with the server extra or the meta requirements-dev.txt.") from exc
uvicorn.run(
args.app,

47
src/govoplan_core/security/redaction.py Normal file
View File

@@ -0,0 +1,47 @@
from __future__ import annotations
import re
from collections.abc import Mapping
from typing import Any
def sensitive_key_tokens(key: object) -> set[str]:
value = str(key).strip()
value = re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", value)
return {item for item in re.split(r"[^A-Za-z0-9]+", value.lower()) if item}
def is_sensitive_key(key: object) -> bool:
tokens = sensitive_key_tokens(key)
if not tokens:
return False
if tokens & {"authorization", "cookie", "credential", "credentials", "password", "passwd", "secret", "token"}:
return True
return bool({"api", "key"} <= tokens or {"access", "key"} <= tokens or {"private", "key"} <= tokens)
def redact_secret_values(value: object, *, placeholder: str = "<redacted>") -> object:
if isinstance(value, Mapping):
result: dict[str, object] = {}
for key, item in value.items():
if is_sensitive_key(key):
result[str(key)] = placeholder
else:
result[str(key)] = redact_secret_values(item, placeholder=placeholder)
return result
if isinstance(value, list):
return [redact_secret_values(item, placeholder=placeholder) for item in value]
return value
def contains_plain_secret(value: object) -> bool:
if not isinstance(value, Mapping):
return False
for key, item in value.items():
normalized_key = str(key).strip().casefold().replace("-", "_")
if normalized_key in {"credential_ref", "secret_ref", "secret_reference"}:
continue
if is_sensitive_key(key) and item not in (None, "", {"secret_ref": ""}):
return True
if isinstance(item, Mapping) and contains_plain_secret(item):
return True
return False

4
src/govoplan_core/server/fastapi.py
View File

@@ -43,9 +43,11 @@ def create_govoplan_app(
origins = [item.strip() for item in cors_origins if item.strip()]
if origins:
if "*" in origins:
raise ValueError("CORS wildcard origin '*' cannot be used with credentialed requests; configure explicit origins.")
app.add_middleware(
CORSMiddleware,
allow_origins=["*"] if "*" in origins else origins,
allow_origins=origins,
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],

1
src/govoplan_core/settings.py
View File

@@ -18,6 +18,7 @@ class Settings(BaseSettings):
tenant_db_url_template: str | None = Field(default=None, alias="TENANT_DB_URL_TEMPLATE")
tenant_schema_template: str | None = Field(default=None, alias="TENANT_SCHEMA_TEMPLATE")
enabled_modules: str = Field(default="tenancy,organizations,identity,access,admin,dashboard,policy,audit,campaigns,files,mail,calendar,docs,ops", alias="ENABLED_MODULES")
migration_track: str = Field(default="release", alias="GOVOPLAN_MIGRATION_TRACK")
redis_url: str = Field(default="redis://redis:6379/0", alias="REDIS_URL")
celery_enabled: bool = Field(default=False, alias="CELERY_ENABLED")