Release v0.1.8

src/govoplan_core/core/configuration_control.py

@@ -12,6 +12,7 @@ from govoplan_core.admin.settings import get_system_settings
 from govoplan_core.core.change_sequence import record_change
 from govoplan_core.core.configuration_safety import classify_configuration_field, plan_configuration_change
 from govoplan_core.core.maintenance import saved_maintenance_mode
+from govoplan_core.security.redaction import redact_secret_values
 from govoplan_core.security.time import utc_now
 
 
@@ -377,17 +378,7 @@ def _sanitize_value(key: str, value: object) -> object:
 
 
 def _redact_secrets(value: object) -> object:
-    if isinstance(value, dict):
-        result: dict[str, object] = {}
-        for key, item in value.items():
-            if str(key).strip().casefold() in {"password", "token", "secret", "api_key", "access_key", "secret_key"}:
-                result[str(key)] = "<redacted>"
-            else:
-                result[str(key)] = _redact_secrets(item)
-        return result
-    if isinstance(value, list):
-        return [_redact_secrets(item) for item in value]
-    return value
+    return redact_secret_values(value)
 
 
 def _trim_requests(requests: list[dict[str, Any]]) -> list[dict[str, Any]]:

src/govoplan_core/core/configuration_safety.py

@@ -5,6 +5,7 @@ from collections.abc import Mapping
 from typing import Any, Literal
 
 from govoplan_core.security.permissions import scopes_grant
+from govoplan_core.security.redaction import contains_plain_secret
 
 
 ConfigurationScope = Literal["system", "tenant", "user", "group", "campaign"]
@@ -411,13 +412,4 @@ def _policy_explanation(field: ConfigurationFieldSafety) -> str:
 
 
 def _contains_plain_secret(value: object) -> bool:
-    if not isinstance(value, Mapping):
-        return False
-    secret_keys = {"password", "token", "secret", "api_key", "access_key", "secret_key"}
-    for key, item in value.items():
-        name = str(key).strip().casefold()
-        if name in secret_keys and item not in (None, "", {"secret_ref": ""}):
-            return True
-        if isinstance(item, Mapping) and _contains_plain_secret(item):
-            return True
-    return False
+    return contains_plain_secret(value)

src/govoplan_core/core/module_installer.py

@@ -21,6 +21,7 @@ import time
 import urllib.error
 import urllib.request
 
+from sqlalchemy.engine import make_url
 from sqlalchemy.orm import Session
 
 from govoplan_core.core.maintenance import saved_maintenance_mode
@@ -474,6 +475,8 @@ def run_module_install_plan(
         database_restore_command=database_restore_command,
         database_restore_check_command=database_restore_check_command,
     )
+    record_redactions = _installer_secret_redactions(database_url)
+    result_commands = _command_displays(commands, redactions=record_redactions)
 
     record: dict[str, Any] = {
         "run_id": run_id,
@@ -482,7 +485,7 @@ def run_module_install_plan(
         "plan": [item.as_dict() for item in plan.items],
         "preflight": preflight.as_dict(),
         "snapshot": snapshot,
-        "commands": [_command_record(command) for command in commands],
+        "commands": [_command_record(command, redactions=record_redactions) for command in commands],
         "build_webui": build_webui,
         "migrate_database": migrate_database,
         "activate_installed_modules": activate_installed_modules,
@@ -493,7 +496,7 @@ def run_module_install_plan(
     _write_json(record_path, record)
 
     if dry_run:
-        return ModuleInstallerRunResult(run_id=run_id, status="dry-run", record_path=record_path, commands=tuple(command["display"] for command in commands))
+        return ModuleInstallerRunResult(run_id=run_id, status="dry-run", record_path=record_path, commands=result_commands)
 
     executed: list[dict[str, object]] = []
     failed_error: str | None = None
@@ -507,10 +510,10 @@ def run_module_install_plan(
             for command in commands:
                 result = subprocess.run(command["argv"], cwd=command.get("cwd"), text=True, capture_output=True, check=False)
                 executed.append({
-                    **_command_record(command),
+                    **_command_record(command, redactions=record_redactions),
                     "return_code": result.returncode,
-                    "stdout": result.stdout[-8000:],
-                    "stderr": result.stderr[-8000:],
+                    "stdout": _redact_installer_text(result.stdout[-8000:], redactions=record_redactions),
+                    "stderr": _redact_installer_text(result.stderr[-8000:], redactions=record_redactions),
                 })
                 migration_task_record_path = command.get("migration_task_record_path")
                 if migration_task_record_path:
@@ -521,9 +524,9 @@ def run_module_install_plan(
                 record["commands"] = executed
                 _write_json(record_path, record)
                 if result.returncode != 0:
-                    raise ModuleInstallerError(f"Command failed ({result.returncode}): {command['display']}")
+                    raise ModuleInstallerError(f"Command failed ({result.returncode}): {_redact_installer_text(str(command['display']), redactions=record_redactions)}")
         except Exception as exc:
-            failed_error = str(exc)
+            failed_error = _redact_installer_text(str(exc), redactions=record_redactions)
             try:
                 session.rollback()
             except Exception:
@@ -551,12 +554,12 @@ def run_module_install_plan(
                 run_id=run_id,
                 status="rolled-back" if rollback.return_code == 0 else "failed",
                 record_path=record_path,
-                commands=tuple(command["display"] for command in commands),
+                commands=result_commands,
                 return_code=1,
                 error=failed_error,
                 rollback=rollback.as_dict(),
             )
-        return ModuleInstallerRunResult(run_id=run_id, status="failed", record_path=record_path, commands=tuple(command["display"] for command in commands), return_code=1, error=failed_error)
+        return ModuleInstallerRunResult(run_id=run_id, status="failed", record_path=record_path, commands=result_commands, return_code=1, error=failed_error)
 
     applied_items = tuple(_mark_applied(item) for item in plan.items)
     save_module_install_plan(session, applied_items)
@@ -576,7 +579,7 @@ def run_module_install_plan(
         "commands": executed,
     })
     _write_json(record_path, record)
-    return ModuleInstallerRunResult(run_id=run_id, status="applied", record_path=record_path, commands=tuple(command["display"] for command in commands))
+    return ModuleInstallerRunResult(run_id=run_id, status="applied", record_path=record_path, commands=result_commands)
 
 
 def supervise_module_install_plan(
@@ -720,6 +723,8 @@ def rollback_module_install_run(
         raise ModuleInstallerError(f"Run record does not exist: {record_path}")
     record = json.loads(record_path.read_text(encoding="utf-8"))
     snapshot = record.get("snapshot") if isinstance(record.get("snapshot"), dict) else {}
+    raw_database_backup = snapshot.get("database_backup") if isinstance(snapshot.get("database_backup"), Mapping) else {}
+    rollback_redactions = _installer_secret_redactions(database_url or _database_url_from_external_snapshot(run_dir, raw_database_backup))
     commands: list[dict[str, Any]] = []
     for package in _planned_python_install_packages(record):
         commands.append(_structured_command([sys.executable, "-m", "pip", "uninstall", "-y", package]))
@@ -741,15 +746,15 @@ def rollback_module_install_run(
         for command in commands:
             result = subprocess.run(command["argv"], cwd=command.get("cwd"), text=True, capture_output=True, check=False)
             executed.append({
-                **_command_record(command),
+                **_command_record(command, redactions=rollback_redactions),
                 "return_code": result.returncode,
-                "stdout": result.stdout[-8000:],
-                "stderr": result.stderr[-8000:],
+                "stdout": _redact_installer_text(result.stdout[-8000:], redactions=rollback_redactions),
+                "stderr": _redact_installer_text(result.stderr[-8000:], redactions=rollback_redactions),
             })
             if result.returncode != 0:
                 record.update({"rollback_status": "failed", "rollback_commands": executed})
                 _write_json(record_path, record)
-                return ModuleInstallerRunResult(run_id=run_id, status="failed", record_path=record_path, commands=tuple(command["display"] for command in commands), return_code=1)
+                return ModuleInstallerRunResult(run_id=run_id, status="failed", record_path=record_path, commands=_command_displays(commands, redactions=rollback_redactions), return_code=1)
         database_restore = _restore_database_snapshot(
             run_dir,
             snapshot,
@@ -763,7 +768,7 @@ def rollback_module_install_run(
                 "rollback_database_restore": database_restore,
             })
             _write_json(record_path, record)
-            return ModuleInstallerRunResult(run_id=run_id, status="failed", record_path=record_path, commands=tuple(command["display"] for command in commands), return_code=1, error=str(database_restore.get("error") or "Database restore failed."))
+            return ModuleInstallerRunResult(run_id=run_id, status="failed", record_path=record_path, commands=_command_displays(commands, redactions=rollback_redactions), return_code=1, error=str(database_restore.get("error") or "Database restore failed."))
 
     record.update({
         "rollback_status": "rolled-back",
@@ -773,7 +778,7 @@ def rollback_module_install_run(
     if database_restore is not None:
         record["rollback_database_restore"] = database_restore
     _write_json(record_path, record)
-    return ModuleInstallerRunResult(run_id=run_id, status="rolled-back", record_path=record_path, commands=tuple(command["display"] for command in commands))
+    return ModuleInstallerRunResult(run_id=run_id, status="rolled-back", record_path=record_path, commands=_command_displays(commands, redactions=rollback_redactions))
 
 
 def list_module_installer_runs(*, runtime_dir: Path, limit: int = 25) -> tuple[dict[str, object], ...]:
@@ -3007,6 +3012,8 @@ def _snapshot_external_database(
 ) -> dict[str, object]:
     backup_path = run_dir / "database.external.backup"
     metadata_path = run_dir / "database-backup-metadata.json"
+    redactions = _installer_secret_redactions(database_url)
+    database_url_secret = _write_installer_secret(run_dir / "database-url.secret", database_url) if database_url else None
     result = _run_database_hook(
         backup_command,
         run_dir=run_dir,
@@ -3016,18 +3023,20 @@ def _snapshot_external_database(
     )
     payload: dict[str, object] = {
         "type": "external",
-        "database_url": database_url or "",
-        "backup_command": backup_command,
-        "restore_command": restore_command,
-        "restore_check_command": restore_check_command,
+        "database_url": _redact_installer_text(database_url or "", redactions=redactions),
+        "backup_command": _redact_installer_text(backup_command, redactions=redactions),
+        "restore_command": _redact_installer_text(restore_command, redactions=redactions),
+        "restore_check_command": _redact_installer_text(restore_check_command, redactions=redactions) if restore_check_command else None,
         "backup_path": backup_path.name,
         "metadata_path": metadata_path.name,
         "return_code": result.returncode,
-        "stdout": result.stdout[-8000:],
-        "stderr": result.stderr[-8000:],
+        "stdout": _redact_installer_text(result.stdout[-8000:], redactions=redactions),
+        "stderr": _redact_installer_text(result.stderr[-8000:], redactions=redactions),
     }
+    if database_url_secret:
+        payload["database_url_secret"] = database_url_secret
     if result.returncode != 0:
-        raise ModuleInstallerError(f"Database backup command failed ({result.returncode}): {backup_command}")
+        raise ModuleInstallerError(f"Database backup command failed ({result.returncode}): {_redact_installer_text(backup_command, redactions=redactions)}")
     if restore_check_command:
         restore_check = _run_database_hook(
             restore_check_command,
@@ -3037,18 +3046,18 @@ def _snapshot_external_database(
             metadata_path=metadata_path,
         )
         payload["restore_check"] = {
-            "command": restore_check_command,
+            "command": _redact_installer_text(restore_check_command, redactions=redactions),
             "return_code": restore_check.returncode,
-            "stdout": restore_check.stdout[-8000:],
-            "stderr": restore_check.stderr[-8000:],
+            "stdout": _redact_installer_text(restore_check.stdout[-8000:], redactions=redactions),
+            "stderr": _redact_installer_text(restore_check.stderr[-8000:], redactions=redactions),
         }
         if restore_check.returncode != 0:
-            raise ModuleInstallerError(f"Database restore-check command failed ({restore_check.returncode}): {restore_check_command}")
+            raise ModuleInstallerError(f"Database restore-check command failed ({restore_check.returncode}): {_redact_installer_text(restore_check_command, redactions=redactions)}")
     if metadata_path.exists():
         try:
             metadata = json.loads(metadata_path.read_text(encoding="utf-8"))
             if isinstance(metadata, dict):
-                payload["metadata"] = metadata
+                payload["metadata"] = _redact_installer_value(metadata, redactions=redactions)
         except json.JSONDecodeError as exc:
             raise ModuleInstallerError(f"Database backup metadata is not valid JSON: {metadata_path}") from exc
     return payload
@@ -3102,29 +3111,92 @@ def _restore_external_database_snapshot(
         return {"restored": False, "type": "external", "error": "No database restore command is available for this run."}
     backup_name = raw.get("backup_path")
     metadata_name = raw.get("metadata_path")
-    raw_database_url = raw.get("database_url")
+    raw_database_url = _database_url_from_external_snapshot(run_dir, raw)
     backup_path = run_dir / backup_name if isinstance(backup_name, str) else run_dir / "database.external.backup"
     metadata_path = run_dir / metadata_name if isinstance(metadata_name, str) else run_dir / "database-backup-metadata.json"
+    effective_database_url = database_url or raw_database_url
+    redactions = _installer_secret_redactions(effective_database_url)
     result = _run_database_hook(
         command,
         run_dir=run_dir,
-        database_url=database_url or (raw_database_url if isinstance(raw_database_url, str) else None),
+        database_url=effective_database_url,
         backup_path=backup_path,
         metadata_path=metadata_path,
     )
     payload: dict[str, object] = {
         "restored": result.returncode == 0,
         "type": "external",
-        "command": command,
+        "command": _redact_installer_text(command, redactions=redactions),
         "return_code": result.returncode,
-        "stdout": result.stdout[-8000:],
-        "stderr": result.stderr[-8000:],
+        "stdout": _redact_installer_text(result.stdout[-8000:], redactions=redactions),
+        "stderr": _redact_installer_text(result.stderr[-8000:], redactions=redactions),
     }
     if result.returncode != 0:
         payload["error"] = f"Database restore command failed ({result.returncode})."
     return payload
 
 
+def _write_installer_secret(path: Path, value: str | None) -> str | None:
+    if not value:
+        return None
+    path.write_text(value, encoding="utf-8")
+    try:
+        path.chmod(0o600)
+    except OSError:
+        pass
+    return path.name
+
+
+def _database_url_from_external_snapshot(run_dir: Path, raw: Mapping[str, object]) -> str | None:
+    secret_name = raw.get("database_url_secret")
+    if isinstance(secret_name, str) and secret_name:
+        secret_path = run_dir / Path(secret_name).name
+        try:
+            value = secret_path.read_text(encoding="utf-8").strip()
+        except OSError:
+            value = ""
+        if value:
+            return value
+    value = raw.get("database_url")
+    if isinstance(value, str) and "<redacted>" not in value and "***" not in value:
+        return value
+    return None
+
+
+def _installer_secret_redactions(database_url: str | None) -> dict[str, str]:
+    redactions: dict[str, str] = {}
+    for value in (database_url, _database_url_for_pgtools(database_url or "") if database_url else None):
+        if not value:
+            continue
+        redacted = _redact_url_credentials(value)
+        redactions[value] = redacted
+    return redactions
+
+
+def _redact_installer_value(value: object, *, redactions: Mapping[str, str]) -> object:
+    if isinstance(value, str):
+        return _redact_installer_text(value, redactions=redactions)
+    if isinstance(value, Mapping):
+        return {str(key): _redact_installer_value(item, redactions=redactions) for key, item in value.items()}
+    if isinstance(value, list):
+        return [_redact_installer_value(item, redactions=redactions) for item in value]
+    return value
+
+
+def _redact_installer_text(value: str, *, redactions: Mapping[str, str]) -> str:
+    result = value
+    for raw, redacted in sorted(redactions.items(), key=lambda item: len(item[0]), reverse=True):
+        result = result.replace(raw, redacted)
+    return _redact_url_credentials(result)
+
+
+def _redact_url_credentials(value: str) -> str:
+    try:
+        return make_url(value).render_as_string(hide_password=True)
+    except Exception:
+        return re.sub(r"([A-Za-z][A-Za-z0-9+.-]*://[^\s/@:]+:)([^@\s]+)(@)", r"\1***\3", value)
+
+
 def _run_database_hook(
     command: str,
     *,
@@ -3229,15 +3301,20 @@ def _quote_arg(value: str) -> str:
     return shlex.quote(value)
 
 
-def _command_record(command: Mapping[str, Any]) -> dict[str, object]:
+def _command_displays(commands: Iterable[Mapping[str, Any]], *, redactions: Mapping[str, str]) -> tuple[str, ...]:
+    return tuple(_redact_installer_text(str(command["display"]), redactions=redactions) for command in commands)
+
+
+def _command_record(command: Mapping[str, Any], *, redactions: Mapping[str, str] | None = None) -> dict[str, object]:
+    effective_redactions = redactions or {}
     payload: dict[str, object] = {
-        "display": str(command["display"]),
-        "argv": [str(item) for item in command["argv"]],
+        "display": _redact_installer_text(str(command["display"]), redactions=effective_redactions),
+        "argv": [_redact_installer_text(str(item), redactions=effective_redactions) for item in command["argv"]],
     }
     if command.get("cwd"):
-        payload["cwd"] = str(command["cwd"])
+        payload["cwd"] = _redact_installer_text(str(command["cwd"]), redactions=effective_redactions)
     if command.get("migration_task_record_path"):
-        payload["migration_task_record_path"] = str(command["migration_task_record_path"])
+        payload["migration_task_record_path"] = _redact_installer_text(str(command["migration_task_record_path"]), redactions=effective_redactions)
     return payload