chore: consolidate platform split checks
Some checks failed
Dependency Audit / dependency-audit (push) Has been cancelled
Module Matrix / module-matrix (push) Has been cancelled

This commit is contained in:
2026-07-10 12:51:19 +02:00
parent 150b720f12
commit 635d25c74c
216 changed files with 23336 additions and 4077 deletions

View File

@@ -0,0 +1,37 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
PYTHON="${PYTHON:-$ROOT/.venv/bin/python}"
NPM="${NPM:-/home/zemion/.nvm/versions/node/v22.22.3/bin/npm}"
NODE_BIN="$(dirname "$NPM")"
if [[ ! -x "$PYTHON" ]]; then
PYTHON=python
fi
if [[ ! -x "$NPM" ]]; then
NPM=npm
NODE_BIN="$(dirname "$(command -v "$NPM")")"
fi
cd "$ROOT"
CHECK_TESTCLIENT_DEPRECATIONS=auto bash "$ROOT/scripts/check-dependency-hygiene.sh"
if ! "$PYTHON" -m pip_audit --version >/dev/null 2>&1; then
echo "pip-audit is not installed. Install dev requirements first:" >&2
echo " $PYTHON -m pip install -r $ROOT/requirements-dev.txt" >&2
exit 127
fi
python_status=0
npm_status=0
"$PYTHON" -m pip_audit --progress-spinner off || python_status=$?
cd "$ROOT/webui"
PATH="$ROOT/webui/node_modules/.bin:$NODE_BIN:$PATH" "$NPM" audit --omit=dev || npm_status=$?
if [[ "$python_status" -ne 0 || "$npm_status" -ne 0 ]]; then
echo "Dependency audit failed: python=$python_status npm=$npm_status" >&2
exit 1
fi

View File

@@ -0,0 +1,158 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
PYTHON="${PYTHON:-$ROOT/.venv/bin/python}"
CHECK_TESTCLIENT_DEPRECATIONS="${CHECK_TESTCLIENT_DEPRECATIONS:-auto}"
if [[ ! -x "$PYTHON" ]]; then
PYTHON=python
fi
cd "$ROOT"
"$PYTHON" - <<'PY'
import importlib.util
import sys
if importlib.util.find_spec("pip") is None:
print("Dependency hygiene failed: this Python environment cannot import pip.", file=sys.stderr)
print("Recreate or repair the venv, then reinstall requirements:", file=sys.stderr)
print(" python -m venv .venv", file=sys.stderr)
print(" ./.venv/bin/python -m pip install --upgrade pip", file=sys.stderr)
print(" ./.venv/bin/python -m pip install -r requirements-dev.txt", file=sys.stderr)
raise SystemExit(127)
PY
"$PYTHON" -m pip check
"$PYTHON" - <<'PY'
from __future__ import annotations
import importlib.metadata
import pathlib
import sys
prefix = pathlib.Path(sys.prefix)
legacy_patterns = (
"__editable__.govoplan_module_multimailer-*.pth",
"__editable___govoplan_module_multimailer_*_finder.py",
"govoplan_module_multimailer-*.dist-info",
)
problems: list[pathlib.Path] = []
for site_packages in (prefix / "lib").glob("python*/site-packages"):
for pattern in legacy_patterns:
problems.extend(site_packages.glob(pattern))
for dist in importlib.metadata.distributions():
if dist.metadata.get("Name", "").lower() == "govoplan-module-multimailer":
problems.append(pathlib.Path(str(dist.locate_file(""))))
if problems:
print("Dependency hygiene failed: stale legacy multimailer install metadata found.", file=sys.stderr)
for path in sorted(set(problems)):
print(f" {path}", file=sys.stderr)
print("Remove the stale files or recreate the venv. This package pins obsolete dependencies.", file=sys.stderr)
raise SystemExit(1)
print("No stale legacy multimailer package metadata found.")
PY
"$PYTHON" - <<'PY'
from __future__ import annotations
import pathlib
import sys
root = pathlib.Path.cwd()
scan_roots = [
root / "src",
root / "tests",
root.parent / "govoplan-access" / "src",
root.parent / "govoplan-admin" / "src",
root.parent / "govoplan-audit" / "src",
root.parent / "govoplan-calendar" / "src",
root.parent / "govoplan-campaign" / "src",
root.parent / "govoplan-dashboard" / "src",
root.parent / "govoplan-files" / "src",
root.parent / "govoplan-identity" / "src",
root.parent / "govoplan-mail" / "src",
root.parent / "govoplan-ops" / "src",
root.parent / "govoplan-organizations" / "src",
root.parent / "govoplan-policy" / "src",
root.parent / "govoplan-tenancy" / "src",
]
deprecated_tokens = {
"HTTP_422_UNPROCESSABLE_ENTITY": "Use HTTP_422_UNPROCESSABLE_CONTENT; the numeric status remains 422.",
}
hits: list[str] = []
for scan_root in scan_roots:
if not scan_root.exists():
continue
for path in scan_root.rglob("*.py"):
try:
text = path.read_text(encoding="utf-8")
except UnicodeDecodeError:
continue
for token, replacement in deprecated_tokens.items():
if token in text:
hits.append(f"{path}: deprecated {token}. {replacement}")
if hits:
print("Dependency hygiene failed: deprecated framework constants are still used.", file=sys.stderr)
print("\n".join(hits), file=sys.stderr)
raise SystemExit(1)
print("Deprecated framework constant scan passed.")
PY
RUN_TESTCLIENT_DEPRECATION_SMOKE=0
case "$CHECK_TESTCLIENT_DEPRECATIONS" in
0|false|False|no|No)
echo "TestClient deprecation smoke skipped by CHECK_TESTCLIENT_DEPRECATIONS=$CHECK_TESTCLIENT_DEPRECATIONS"
;;
auto)
if "$PYTHON" - <<'PY'
import importlib.util
raise SystemExit(0 if importlib.util.find_spec("fastapi") and importlib.util.find_spec("starlette") and importlib.util.find_spec("httpx2") else 1)
PY
then
RUN_TESTCLIENT_DEPRECATION_SMOKE=1
else
echo "TestClient deprecation smoke skipped; install dev requirements to enable it."
fi
;;
*)
RUN_TESTCLIENT_DEPRECATION_SMOKE=1
;;
esac
if [[ "$RUN_TESTCLIENT_DEPRECATION_SMOKE" -eq 1 ]]; then
"$PYTHON" - <<'PY'
import warnings
from starlette.exceptions import StarletteDeprecationWarning
warnings.simplefilter("error", StarletteDeprecationWarning)
from fastapi import FastAPI
from fastapi.testclient import TestClient
app = FastAPI()
@app.get("/dependency-hygiene-ping")
def ping() -> dict[str, bool]:
return {"ok": True}
with TestClient(app) as client:
response = client.get("/dependency-hygiene-ping")
assert response.status_code == 200
assert response.json() == {"ok": True}
print("TestClient deprecation smoke passed.")
PY
fi
echo "Dependency hygiene check passed."

View File

@@ -5,11 +5,19 @@ ROOT="/mnt/DATA/git/govoplan-core"
NODE="/home/zemion/.nvm/versions/node/v22.22.3/bin"
NPM="$NODE/npm"
WEBUI_BIN="$ROOT/webui/node_modules/.bin"
NPM_USERCONFIG="$(mktemp "${TMPDIR:-/tmp}/govoplan-npmrc.XXXXXXXX")"
trap 'rm -f "$NPM_USERCONFIG"' EXIT
export PATH="$WEBUI_BIN:$NODE:$PATH"
export NPM_CONFIG_USERCONFIG="$NPM_USERCONFIG"
export GOVOPLAN_NPM_USERCONFIG="$NPM_USERCONFIG"
unset npm_config_tmp NPM_CONFIG_TMP
cd "$ROOT"
CHECK_TESTCLIENT_DEPRECATIONS=1 bash scripts/check-dependency-hygiene.sh
./.venv/bin/python - <<'PY'
import ast
import pathlib
@@ -20,6 +28,8 @@ roots = [
pathlib.Path("/mnt/DATA/git/govoplan-access/src"),
pathlib.Path("/mnt/DATA/git/govoplan-admin/src"),
pathlib.Path("/mnt/DATA/git/govoplan-tenancy/src"),
pathlib.Path("/mnt/DATA/git/govoplan-organizations/src"),
pathlib.Path("/mnt/DATA/git/govoplan-identity/src"),
pathlib.Path("/mnt/DATA/git/govoplan-policy/src"),
pathlib.Path("/mnt/DATA/git/govoplan-audit/src"),
pathlib.Path("/mnt/DATA/git/govoplan-mail/src"),
@@ -47,7 +57,7 @@ if errors:
print(f"AST syntax check passed for {count} Python files")
PY
./.venv/bin/python -c 'import govoplan_core.db.bootstrap; import govoplan_core.admin.service; import govoplan_files.backend.router; import govoplan_mail.backend.sending.imap; print("targeted backend imports passed")'
./.venv/bin/python -c 'import govoplan_core.db.bootstrap; import govoplan_access.backend.admin.service; import govoplan_files.backend.router; import govoplan_mail.backend.sending.imap; print("targeted backend imports passed")'
./.venv/bin/python scripts/check_dependency_boundaries.py
./.venv/bin/python -m unittest tests.test_module_system
./.venv/bin/python -m unittest discover -s /mnt/DATA/git/govoplan-mail/tests

View File

@@ -0,0 +1,23 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
PYTHON="${PYTHON:-$ROOT/.venv/bin/python}"
NPM="${NPM:-/home/zemion/.nvm/versions/node/v22.22.3/bin/npm}"
NODE_BIN="$(dirname "$NPM")"
if [[ ! -x "$PYTHON" ]]; then
PYTHON=python
fi
if [[ ! -x "$NPM" ]]; then
NPM=npm
NODE_BIN="$(dirname "$(command -v "$NPM")")"
fi
cd "$ROOT"
"$PYTHON" -m unittest tests.test_module_system tests.test_access_contracts
"$PYTHON" scripts/check_dependency_boundaries.py
cd "$ROOT/webui"
PATH="$ROOT/webui/node_modules/.bin:$NODE_BIN:$PATH" "$NPM" run test:module-capabilities
PATH="$ROOT/webui/node_modules/.bin:$NODE_BIN:$PATH" "$NPM" run test:module-permutations

View File

@@ -11,8 +11,11 @@ REPOS = {
"access": ROOT.parent / "govoplan-access" / "src" / "govoplan_access",
"admin": ROOT.parent / "govoplan-admin" / "src" / "govoplan_admin",
"tenancy": ROOT.parent / "govoplan-tenancy" / "src" / "govoplan_tenancy",
"organizations": ROOT.parent / "govoplan-organizations" / "src" / "govoplan_organizations",
"identity": ROOT.parent / "govoplan-identity" / "src" / "govoplan_identity",
"policy": ROOT.parent / "govoplan-policy" / "src" / "govoplan_policy",
"audit": ROOT.parent / "govoplan-audit" / "src" / "govoplan_audit",
"dashboard": ROOT.parent / "govoplan-dashboard" / "src" / "govoplan_dashboard",
"files": ROOT.parent / "govoplan-files" / "src" / "govoplan_files",
"mail": ROOT.parent / "govoplan-mail" / "src" / "govoplan_mail",
"campaign": ROOT.parent / "govoplan-campaign" / "src" / "govoplan_campaign",
@@ -22,15 +25,18 @@ PREFIXES = {
"access": "govoplan_access",
"admin": "govoplan_admin",
"tenancy": "govoplan_tenancy",
"organizations": "govoplan_organizations",
"identity": "govoplan_identity",
"policy": "govoplan_policy",
"audit": "govoplan_audit",
"dashboard": "govoplan_dashboard",
"files": "govoplan_files",
"mail": "govoplan_mail",
"campaign": "govoplan_campaign",
}
FEATURE_OWNERS = ("files", "mail", "campaign")
PLATFORM_OWNERS = ("admin", "tenancy", "policy", "audit")
PUBLIC_ACCESS_IMPORTS = ("govoplan_access.backend.auth.dependencies",)
PLATFORM_OWNERS = ("admin", "tenancy", "organizations", "identity", "policy", "audit", "dashboard")
PUBLIC_ACCESS_IMPORTS = ("govoplan_access.auth",)
@dataclass(frozen=True)

View File

@@ -38,6 +38,22 @@ CATALOG_MODULES = (
description="Tenant registry, tenant settings, and tenant resolution platform module.",
tags=("official", "platform-module"),
),
CatalogModule(
module_id="organizations",
repo="govoplan-organizations",
python_package="govoplan-organizations",
name="Organizations",
description="Organization units, functions, and account-held function assignments.",
tags=("official", "platform-module"),
),
CatalogModule(
module_id="identity",
repo="govoplan-identity",
python_package="govoplan-identity",
name="Identity",
description="Canonical identities and links between identities and platform accounts.",
tags=("official", "platform-module"),
),
CatalogModule(
module_id="access",
repo="govoplan-access",
@@ -72,6 +88,15 @@ CATALOG_MODULES = (
description="Audit-log storage and audit administration routes.",
tags=("official", "platform-module"),
),
CatalogModule(
module_id="dashboard",
repo="govoplan-dashboard",
python_package="govoplan-dashboard",
name="Dashboard",
description="Configurable user home assembled from module-provided dashboard widgets.",
tags=("official", "platform-module"),
webui_package="@govoplan/dashboard-webui",
),
CatalogModule(
module_id="files",
repo="govoplan-files",
@@ -108,6 +133,24 @@ CATALOG_MODULES = (
tags=("official", "service-module"),
webui_package="@govoplan/calendar-webui",
),
CatalogModule(
module_id="docs",
repo="govoplan-docs",
python_package="govoplan-docs",
name="Docs",
description="Configured-system documentation and evidence-aware help surfaces.",
tags=("official", "platform-module"),
webui_package="@govoplan/docs-webui",
),
CatalogModule(
module_id="ops",
repo="govoplan-ops",
python_package="govoplan-ops",
name="Ops",
description="Runtime health, deployment profile, worker split, and sizing visibility.",
tags=("official", "platform-module"),
webui_package="@govoplan/ops-webui",
),
)

View File

@@ -12,11 +12,10 @@ import re
import sys
from typing import Any, Iterable
from gitea_common import GiteaClient, GiteaError, infer_target, load_dotenv, repo_path, repo_root, require_token
from gitea_common import GiteaClient, GiteaError, infer_target, label_ids_by_name, load_dotenv, repo_path, repo_root, require_token
DEFAULT_PRODUCT_BACKLOG = pathlib.Path("/mnt/DATA/Nextcloud/ADD ideas UG/Products/govoplan/backlog.md")
DEFAULT_ACCESS_PLAN = pathlib.Path("/mnt/DATA/git/govoplan-core/docs/ACCESS_EXTRACTION_PLAN.md")
DEFAULT_WEB_TODO = pathlib.Path("/mnt/DATA/git/govoplan-web/TODO.md")
REPO_ROOTS = {
@@ -62,7 +61,7 @@ def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--env-file", type=pathlib.Path, help="dotenv file to read before using GITEA_* values")
parser.add_argument("--product-backlog", type=pathlib.Path, default=DEFAULT_PRODUCT_BACKLOG)
parser.add_argument("--access-plan", type=pathlib.Path, default=DEFAULT_ACCESS_PLAN)
parser.add_argument("--access-plan", type=pathlib.Path, help="optional legacy access extraction plan to import")
parser.add_argument("--web-todo", type=pathlib.Path, default=DEFAULT_WEB_TODO)
parser.add_argument("--apply", action="store_true", help="create missing Gitea issues")
args = parser.parse_args()
@@ -128,7 +127,7 @@ def main() -> int:
def build_candidates(args: argparse.Namespace) -> Iterable[Candidate]:
if args.product_backlog.exists():
yield from parse_product_backlog(args.product_backlog)
if args.access_plan.exists():
if args.access_plan and args.access_plan.exists():
yield from parse_access_plan(args.access_plan)
if args.web_todo.exists():
yield from parse_simple_todo(args.web_todo)
@@ -539,8 +538,7 @@ def load_repo_states(token: str, repo_keys: list[str]) -> dict[str, RepoState]:
root = repo_root(REPO_ROOTS[repo_key])
target = infer_target(root)
client = GiteaClient(target, token)
labels_payload = client.paginate(repo_path(target.owner, target.repo, "/labels"))
label_ids = {str(label["name"]): int(label["id"]) for label in labels_payload if "name" in label and "id" in label}
label_ids = label_ids_by_name(client, target.owner, target.repo)
issues = client.paginate(repo_path(target.owner, target.repo, "/issues"), query={"state": "all"}, limit=100)
fingerprints: set[str] = set()
titles: set[str] = set()

View File

@@ -14,7 +14,7 @@ import subprocess
import sys
from typing import Any, Iterable
from gitea_common import GiteaClient, GiteaError, infer_target, load_dotenv, repo_path, require_token
from gitea_common import GiteaClient, GiteaError, infer_target, label_ids_by_name, load_dotenv, repo_path, require_token
GIT_ROOT = pathlib.Path("/mnt/DATA/git")
@@ -246,9 +246,7 @@ def is_excluded_repo_file(path: pathlib.Path) -> bool:
return True
if text.endswith("/docs/GITEA_ISSUES.md"):
return True
if text.endswith("/docs/GOVOPLAN_MODULE_ROADMAP.md"):
return True
if text.endswith("/docs/ACCESS_EXTRACTION_PLAN.md"):
if text.endswith("/docs/GOVOPLAN_MASTER_ROADMAP.md"):
return True
if text.endswith("/govoplan-web/TODO.md"):
return True
@@ -537,8 +535,7 @@ def grouped_counts(candidates: list[Candidate]) -> dict[str, int]:
def load_repo_state(repo: RepoInfo, token: str, *, apply: bool) -> RepoState:
target = infer_target(repo.root)
client = GiteaClient(target, token)
labels = client.paginate(repo_path(target.owner, target.repo, "/labels"), limit=50)
label_ids = {str(label["name"]): int(label["id"]) for label in labels}
label_ids = label_ids_by_name(client, target.owner, target.repo)
if apply:
for name, color, description in GENERIC_LABELS:
if name in label_ids:

View File

@@ -0,0 +1,250 @@
#!/usr/bin/env python3
"""Move issue labels from repository-local labels to organization labels."""
from __future__ import annotations
import argparse
import os
import pathlib
import re
import sys
from typing import Any
from gitea_common import (
GiteaClient,
GiteaError,
infer_target,
load_dotenv,
org_path,
repo_path,
repo_root,
require_token,
)
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--root", type=pathlib.Path, default=pathlib.Path.cwd(), help="repository root used for Gitea URL inference")
parser.add_argument("--remote", default="origin", help="git remote to use for target inference")
parser.add_argument("--env-file", type=pathlib.Path, default=pathlib.Path("/home/zemion/.config/gitea/gitea.env"))
parser.add_argument("--org", help="organization owner; defaults to inferred owner")
parser.add_argument("--repo", action="append", default=[], help="repository name to process; may be repeated")
parser.add_argument("--repo-regex", default=".*", help="only process repositories whose name matches this regex")
parser.add_argument(
"--issue-mode",
choices=("replace", "delete-add"),
default="replace",
help="replace labels in one request, or delete local ids before adding org ids",
)
parser.add_argument("--delete-repo-labels", action="store_true", help="delete matching repository-local labels after issue and PR migration")
parser.add_argument("--apply", action="store_true", help="apply changes; omit for dry-run")
args = parser.parse_args()
try:
root = repo_root(args.root)
load_dotenv(args.env_file or root / ".env")
target = infer_target(root, args.remote)
owner = args.org or target.owner
client = GiteaClient(target, require_token() if args.apply else os.environ.get("GITEA_TOKEN"))
if client.token is None:
raise GiteaError("GITEA_TOKEN is required for dry-run and apply because migration inspects live issue labels.")
repo_filter = re.compile(args.repo_regex)
org_labels = _labels_by_name(client.paginate(org_path(owner, "/labels"), limit=100))
if not org_labels:
raise GiteaError(f"{owner} has no organization labels")
repos = _selected_repositories(client, owner, args.repo, repo_filter)
totals = Totals()
print(f"Organization: {owner}")
print(f"Repositories: {len(repos)}")
print(f"Mode: {'apply' if args.apply else 'dry-run'}")
print(f"Delete repository labels: {args.delete_repo_labels}")
for index, repo in enumerate(repos, start=1):
repo_result = migrate_repository(
client,
owner=owner,
repo=repo,
org_labels=org_labels,
issue_mode=args.issue_mode,
apply=args.apply,
delete_repo_labels=args.delete_repo_labels,
)
totals.add(repo_result)
if repo_result.has_work:
print(
f"[{index}/{len(repos)}] {repo}: "
f"issue-labels={repo_result.issue_label_migrations}, "
f"delete-labels={repo_result.repo_label_deletions}, "
f"errors={len(repo_result.errors)}"
)
for error in repo_result.errors:
print(f" error: {error}")
else:
print(f"[{index}/{len(repos)}] {repo}: no matching local labels")
print("Summary:")
print(f" repositories processed: {totals.repositories}")
print(f" repositories with matching local labels: {totals.repositories_with_work}")
print(f" issue/PR local labels migrated: {totals.issue_label_migrations}")
print(f" repository labels deleted: {totals.repo_label_deletions}")
print(f" errors: {totals.errors}")
if totals.errors:
return 1
return 0
except GiteaError as exc:
print(f"error: {exc}", file=sys.stderr)
return 1
class RepoResult:
def __init__(self) -> None:
self.issue_label_migrations = 0
self.repo_label_deletions = 0
self.errors: list[str] = []
self.matched_local_labels = 0
@property
def has_work(self) -> bool:
return bool(self.matched_local_labels or self.issue_label_migrations or self.repo_label_deletions or self.errors)
class Totals:
def __init__(self) -> None:
self.repositories = 0
self.repositories_with_work = 0
self.issue_label_migrations = 0
self.repo_label_deletions = 0
self.errors = 0
def add(self, result: RepoResult) -> None:
self.repositories += 1
if result.has_work:
self.repositories_with_work += 1
self.issue_label_migrations += result.issue_label_migrations
self.repo_label_deletions += result.repo_label_deletions
self.errors += len(result.errors)
def migrate_repository(
client: GiteaClient,
*,
owner: str,
repo: str,
org_labels: dict[str, dict[str, Any]],
issue_mode: str,
apply: bool,
delete_repo_labels: bool,
) -> RepoResult:
result = RepoResult()
repo_labels = client.paginate(repo_path(owner, repo, "/labels"), limit=100)
local_labels = {
str(label.get("name") or ""): label
for label in repo_labels
if str(label.get("name") or "") in org_labels
}
result.matched_local_labels = len(local_labels)
if not local_labels:
return result
local_by_id = {_label_id(label): label for label in local_labels.values() if _label_id(label) is not None}
org_by_name = {name: _label_id(label) for name, label in org_labels.items()}
for issue_type in ("issues", "pulls"):
issues = client.paginate(
repo_path(owner, repo, "/issues"),
query={"state": "all", "type": issue_type},
limit=100,
)
for issue in issues:
issue_number = int(issue.get("number") or issue.get("index"))
issue_label_ids = [_label_id(label) for label in issue.get("labels") or []]
issue_label_ids = [label_id for label_id in issue_label_ids if label_id is not None]
final_label_ids = list(issue_label_ids)
changed = False
migrated_count = 0
local_ids_to_remove: list[int] = []
org_ids_to_add: list[int] = []
for label in issue.get("labels") or []:
local_id = _label_id(label)
if local_id is None or local_id not in local_by_id:
continue
name = str(label.get("name") or "")
org_id = org_by_name.get(name)
if org_id is None:
continue
local_ids_to_remove.append(local_id)
if org_id not in issue_label_ids and org_id not in org_ids_to_add:
org_ids_to_add.append(org_id)
final_label_ids = [label_id for label_id in final_label_ids if label_id != local_id]
if org_id not in final_label_ids:
final_label_ids.append(org_id)
changed = True
migrated_count += 1
if changed:
if apply:
if issue_mode == "delete-add":
for local_id in local_ids_to_remove:
_remove_issue_label(client, owner, repo, issue_number, local_id)
if org_ids_to_add:
client.request_json(
"POST",
repo_path(owner, repo, f"/issues/{issue_number}/labels"),
body={"labels": org_ids_to_add},
)
else:
client.request_json(
"PUT",
repo_path(owner, repo, f"/issues/{issue_number}/labels"),
body={"labels": final_label_ids},
)
result.issue_label_migrations += migrated_count
if delete_repo_labels:
for name, label in sorted(local_labels.items()):
label_id = _label_id(label)
if label_id is None:
result.errors.append(f"{name} has no label id")
continue
if apply:
try:
client.request_json("DELETE", repo_path(owner, repo, f"/labels/{label_id}"))
except GiteaError as exc:
result.errors.append(f"delete repository label {name}: {exc}")
continue
result.repo_label_deletions += 1
return result
def _selected_repositories(client: GiteaClient, owner: str, explicit: list[str], repo_filter: re.Pattern[str]) -> list[str]:
if explicit:
return sorted(set(explicit))
repos = client.paginate(org_path(owner, "/repos"), query={"type": "all"}, limit=100)
names = sorted(str(repo.get("name") or "") for repo in repos if repo.get("name"))
return [name for name in names if repo_filter.search(name)]
def _labels_by_name(labels: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
return {str(label.get("name")): label for label in labels if label.get("name") and _label_id(label) is not None}
def _label_id(label: dict[str, Any]) -> int | None:
value = label.get("id") or label.get("index")
if value is None:
return None
return int(value)
def _remove_issue_label(client: GiteaClient, owner: str, repo: str, issue_number: int, label_id: int) -> None:
try:
client.request_json("DELETE", repo_path(owner, repo, f"/issues/{issue_number}/labels/{label_id}"))
except GiteaError as exc:
if "HTTP 404" in str(exc):
return
raise
if __name__ == "__main__":
raise SystemExit(main())

View File

@@ -15,6 +15,7 @@ from gitea_common import (
infer_target,
load_dotenv,
load_json,
org_path,
repo_path,
repo_root,
require_token,
@@ -30,6 +31,13 @@ def main() -> int:
parser.add_argument("--remote", default="origin", help="git remote to use for target inference")
parser.add_argument("--labels-file", type=pathlib.Path, default=DEFAULT_LABELS_FILE)
parser.add_argument("--env-file", type=pathlib.Path, help="dotenv file to read before using GITEA_* values")
parser.add_argument(
"--scope",
choices=("repository", "organization"),
default="repository",
help="sync repository labels or organization labels",
)
parser.add_argument("--org", help="organization name for --scope organization; defaults to inferred owner")
parser.add_argument("--apply", action="store_true", help="create or update labels in Gitea")
args = parser.parse_args()
@@ -39,8 +47,12 @@ def main() -> int:
target = infer_target(root, args.remote)
labels = _load_labels(args.labels_file)
token = require_token() if args.apply else os.environ.get("GITEA_TOKEN")
org_name = args.org or target.owner
print(f"Target: {target.display}")
if args.scope == "organization":
print(f"Target: {target.base_url.rstrip('/')}/{org_name} organization labels")
else:
print(f"Target: {target.display}")
print(f"Labels file: {args.labels_file}")
if not args.apply and not token:
@@ -50,7 +62,10 @@ def main() -> int:
return 0
client = GiteaClient(target, token)
existing = _labels_by_name(client, target.owner, target.repo)
if args.scope == "organization":
existing = _org_labels_by_name(client, org_name)
else:
existing = _repo_labels_by_name(client, target.owner, target.repo)
creates: list[dict[str, Any]] = []
updates: list[tuple[dict[str, Any], dict[str, Any]]] = []
@@ -71,18 +86,17 @@ def main() -> int:
for label in creates:
print(f"{'create' if args.apply else 'would create'} {label['name']}")
if args.apply:
client.request_json(
"POST",
repo_path(target.owner, target.repo, "/labels"),
body=label,
)
client.request_json("POST", _create_path(args.scope, org_name, target.owner, target.repo), body=label)
for current, patch in updates:
print(f"{'update' if args.apply else 'would update'} {current['name']}: {', '.join(sorted(patch))}")
if args.apply:
label_id = current.get("id") or current.get("index")
if label_id is None:
raise GiteaError(f"{current['name']} has no label id in API response")
client.request_json(
"PATCH",
repo_path(target.owner, target.repo, f"/labels/{current['id']}"),
_update_path(args.scope, org_name, target.owner, target.repo, int(label_id)),
body=patch,
)
@@ -119,11 +133,28 @@ def _load_labels(path: pathlib.Path) -> list[dict[str, Any]]:
return labels
def _labels_by_name(client: GiteaClient, owner: str, repo: str) -> dict[str, dict[str, Any]]:
def _repo_labels_by_name(client: GiteaClient, owner: str, repo: str) -> dict[str, dict[str, Any]]:
labels = client.paginate(repo_path(owner, repo, "/labels"))
return {str(label.get("name")): label for label in labels}
def _org_labels_by_name(client: GiteaClient, owner: str) -> dict[str, dict[str, Any]]:
labels = client.paginate(org_path(owner, "/labels"))
return {str(label.get("name")): label for label in labels}
def _create_path(scope: str, org: str, owner: str, repo: str) -> str:
if scope == "organization":
return org_path(org, "/labels")
return repo_path(owner, repo, "/labels")
def _update_path(scope: str, org: str, owner: str, repo: str, label_id: int) -> str:
if scope == "organization":
return org_path(org, f"/labels/{label_id}")
return repo_path(owner, repo, f"/labels/{label_id}")
def _diff_label(current: dict[str, Any], desired: dict[str, Any]) -> dict[str, Any]:
patch: dict[str, Any] = {}
if _normalize_color(current.get("color")) != desired["color"]:

View File

@@ -73,6 +73,7 @@ def main() -> int:
parser.add_argument("--transport", choices=("git", "api"), default="git", help="sync through the wiki git repository or the Gitea REST API")
parser.add_argument("--wiki-cache-dir", type=pathlib.Path, default=pathlib.Path("/tmp/codex-gitea-wiki-sync"), help="local cache for wiki git checkouts")
parser.add_argument("--commit-message", default="Sync wiki from project files", help="commit message used by git transport")
parser.add_argument("--prune-managed", action="store_true", help="delete managed wiki pages whose source files are no longer discovered")
parser.add_argument("--apply", action="store_true")
args = parser.parse_args()
@@ -116,9 +117,17 @@ def main() -> int:
overwrite_unmanaged=args.overwrite_unmanaged,
commit_message=args.commit_message,
write_index=not bool(args.page),
prune_managed=args.prune_managed and not bool(args.page),
)
else:
sync_repo_wiki(repo, repo_sources, token, overwrite_unmanaged=args.overwrite_unmanaged, write_index=not bool(args.page))
sync_repo_wiki(
repo,
repo_sources,
token,
overwrite_unmanaged=args.overwrite_unmanaged,
write_index=not bool(args.page),
prune_managed=args.prune_managed and not bool(args.page),
)
except GiteaError as exc:
failures += 1
print(f"error syncing wiki for {repo_key}: {exc}", file=sys.stderr)
@@ -267,7 +276,15 @@ def preview_sources(sources: list[WikiSource]) -> None:
print(f" ... {len(repo_sources) - 80} more")
def sync_repo_wiki(repo: RepoInfo, sources: list[WikiSource], token: str, *, overwrite_unmanaged: bool, write_index: bool = True) -> None:
def sync_repo_wiki(
repo: RepoInfo,
sources: list[WikiSource],
token: str,
*,
overwrite_unmanaged: bool,
write_index: bool = True,
prune_managed: bool = False,
) -> None:
target = infer_target(repo.root)
client = GiteaClient(target, token)
existing_pages = list_existing_wiki_pages(client, target.owner, target.repo)
@@ -311,6 +328,8 @@ def sync_repo_wiki(repo: RepoInfo, sources: list[WikiSource], token: str, *, ove
)
else:
print(f"skipped {target.owner}/{target.repo} wiki:Codex-Project-Index during page-limited sync")
if prune_managed and write_index:
prune_managed_wiki_pages_api(client, target.owner, target.repo, existing_page_names, sources)
def sync_repo_wiki_git(
@@ -321,6 +340,7 @@ def sync_repo_wiki_git(
overwrite_unmanaged: bool,
commit_message: str,
write_index: bool = True,
prune_managed: bool = False,
) -> None:
target = infer_target(repo.root)
wiki_root = prepare_wiki_checkout(repo, cache_dir=cache_dir)
@@ -353,6 +373,8 @@ def sync_repo_wiki_git(
)
else:
print(f"skipped {target.owner}/{target.repo} wiki:Codex-Project-Index during page-limited sync")
if prune_managed and write_index:
prune_managed_wiki_pages_git(wiki_root, sources)
if not git_has_changes(wiki_root):
print(f"unchanged {target.owner}/{target.repo} wiki repository")
return
@@ -426,6 +448,38 @@ def write_wiki_page_file(wiki_root: pathlib.Path, title: str, content: str, *, o
return "updated"
def prune_managed_wiki_pages_git(wiki_root: pathlib.Path, sources: list[WikiSource]) -> None:
desired = {f"{wiki_file_stem(source.page_title)}.md" for source in sources}
desired.add(f"{wiki_file_stem('Codex-Project-Index')}.md")
for path in sorted(wiki_root.glob("*.md")):
if path.name in desired:
continue
current = read_text(path)
if MANAGED_MARKER not in current:
continue
path.unlink()
print(f"deleted stale managed wiki page {path.name}")
def prune_managed_wiki_pages_api(
client: GiteaClient,
owner: str,
repo: str,
existing_page_names: dict[str, str],
sources: list[WikiSource],
) -> None:
desired = {source.page_title for source in sources}
desired.add("Codex-Project-Index")
for title, page_name in sorted(existing_page_names.items()):
if title in desired:
continue
current = client.request_json("GET", repo_path(owner, repo, f"/wiki/page/{quote_wiki_page_name(page_name)}"))
if MANAGED_MARKER not in decode_wiki_content(current):
continue
client.request_json("DELETE", repo_path(owner, repo, f"/wiki/page/{quote_wiki_page_name(page_name)}"))
print(f"deleted stale managed wiki page {owner}/{repo}:{title}")
def wiki_file_stem(title: str) -> str:
return re.sub(r"[/\\]+", "-", title).strip() or "Home"

View File

@@ -13,7 +13,7 @@ import subprocess
import sys
from typing import Any
from gitea_common import GiteaClient, GiteaError, infer_target, load_dotenv, repo_path, repo_root, require_token
from gitea_common import GiteaClient, GiteaError, infer_target, label_ids_by_name, load_dotenv, repo_path, repo_root, require_token
MARKER_RE = re.compile(
@@ -347,8 +347,7 @@ def truncate_title(title: str, limit: int = 180) -> str:
def load_label_ids(client: GiteaClient, owner: str, repo: str) -> dict[str, int]:
labels = client.paginate(repo_path(owner, repo, "/labels"))
return {str(label["name"]): int(label["id"]) for label in labels if "name" in label and "id" in label}
return label_ids_by_name(client, owner, repo)
def load_existing_fingerprints(client: GiteaClient, owner: str, repo: str) -> set[str]:

View File

@@ -197,6 +197,47 @@ def repo_path(owner: str, repo: str, suffix: str) -> str:
return f"/repos/{quote_path(owner)}/{quote_path(repo)}{suffix}"
def org_path(owner: str, suffix: str) -> str:
return f"/orgs/{quote_path(owner)}{suffix}"
def labels_by_name(client: GiteaClient, owner: str, repo: str | None = None) -> dict[str, dict[str, Any]]:
"""Return org labels plus optional repository labels, keyed by label name.
Gitea stores organization labels separately from repository labels. Issue
APIs can use label ids from both scopes on supported instances, so issue
creation helpers should resolve both. Repository labels intentionally win
when a repo carries a local label with the same name.
"""
labels: dict[str, dict[str, Any]] = {}
try:
for label in client.paginate(org_path(owner, "/labels")):
name = str(label.get("name") or "")
if name:
labels[name] = label
except GiteaError:
# User-owned repositories or older instances may not expose org labels.
pass
if repo:
for label in client.paginate(repo_path(owner, repo, "/labels")):
name = str(label.get("name") or "")
if name:
labels[name] = label
return labels
def label_ids_by_name(client: GiteaClient, owner: str, repo: str | None = None) -> dict[str, int]:
ids: dict[str, int] = {}
for name, label in labels_by_name(client, owner, repo).items():
label_id = label.get("id") or label.get("index")
if label_id is None:
continue
ids[name] = int(label_id)
return ids
def _git_remote(root: pathlib.Path, remote_name: str) -> str:
result = subprocess.run(
["git", "-C", str(root), "remote", "get-url", remote_name],

View File

@@ -16,6 +16,7 @@ FRONTEND_FORCE_RELOAD="${GOVOPLAN_FRONTEND_FORCE_RELOAD:-1}"
FRONTEND_CLEAR_VITE_CACHE="${GOVOPLAN_FRONTEND_CLEAR_VITE_CACHE:-1}"
FRONTEND_USE_POLLING="${GOVOPLAN_FRONTEND_USE_POLLING:-1}"
FRONTEND_POLLING_INTERVAL="${GOVOPLAN_FRONTEND_POLLING_INTERVAL:-500}"
DEV_DATABASE_BACKEND="${GOVOPLAN_DEV_DATABASE_BACKEND:-postgres}"
LOG_DIR="$ROOT/runtime/dev-launcher"
BACKEND_LOG="$LOG_DIR/backend.log"
@@ -31,6 +32,20 @@ fail() {
exit 1
}
case "$DEV_DATABASE_BACKEND" in
postgres)
export DATABASE_URL="${DATABASE_URL:-postgresql+psycopg://govoplan_dev@127.0.0.1:5432/govoplan_dev}"
export GOVOPLAN_DATABASE_URL_PGTOOLS="${GOVOPLAN_DATABASE_URL_PGTOOLS:-postgresql://govoplan_dev@127.0.0.1:5432/govoplan_dev}"
;;
sqlite)
export DATABASE_URL="${DATABASE_URL:-sqlite:///$ROOT/runtime/multimailer-dev.db}"
;;
*)
fail "Unsupported GOVOPLAN_DEV_DATABASE_BACKEND=$DEV_DATABASE_BACKEND. Use postgres or sqlite."
;;
esac
export DEV_BOOTSTRAP_ENABLED="${DEV_BOOTSTRAP_ENABLED:-true}"
port_is_free() {
"$PYTHON" - "$1" "$2" <<'PY'
import socket
@@ -139,6 +154,7 @@ GovOPlaN is running.
Web UI: $FRONTEND_URL
API: $BACKEND_URL/api/v1
Health: $BACKEND_URL/health
DB: $DATABASE_URL
Logs:
Backend: $BACKEND_LOG

View File

@@ -0,0 +1,238 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="${GOVOPLAN_CORE_ROOT:-/mnt/DATA/git/govoplan-core}"
PROFILE_ROOT="$ROOT/dev/production-like"
PROFILE_ENV="${GOVOPLAN_PRODUCTION_LIKE_ENV:-$PROFILE_ROOT/.env}"
if [ ! -f "$PROFILE_ENV" ]; then
PROFILE_ENV="$PROFILE_ROOT/.env.example"
fi
WEBUI_ROOT="$ROOT/webui"
PYTHON="${PYTHON:-$ROOT/.venv/bin/python}"
NODE_BIN="${NODE_BIN:-/home/zemion/.nvm/versions/node/v22.22.3/bin}"
NPM="${NPM:-$NODE_BIN/npm}"
DOCKER="${DOCKER:-docker}"
BACKEND_HOST="${GOVOPLAN_BACKEND_HOST:-127.0.0.1}"
BACKEND_PORT="${GOVOPLAN_BACKEND_PORT:-8000}"
FRONTEND_HOST="${GOVOPLAN_FRONTEND_HOST:-127.0.0.1}"
FRONTEND_PORT="${GOVOPLAN_FRONTEND_PORT:-5173}"
OPEN_BROWSER="${OPEN_BROWSER:-1}"
STOP_DEPENDENCIES_ON_EXIT="${GOVOPLAN_STOP_PROFILE_DEPENDENCIES_ON_EXIT:-0}"
LOG_DIR="$ROOT/runtime/production-like/logs"
BACKEND_LOG="$LOG_DIR/backend.log"
WORKER_LOG="$LOG_DIR/worker.log"
FRONTEND_LOG="$LOG_DIR/frontend.log"
BACKEND_URL="http://$BACKEND_HOST:$BACKEND_PORT"
FRONTEND_URL="http://$FRONTEND_HOST:$FRONTEND_PORT"
backend_pid=""
worker_pid=""
frontend_pid=""
fail() {
printf 'launch-production-like-dev: %s\n' "$*" >&2
exit 1
}
[ -f "$PROFILE_ENV" ] || fail "profile env file not found: $PROFILE_ENV"
set -a
# shellcheck disable=SC1090
. "$PROFILE_ENV"
set +a
export APP_ENV="${APP_ENV:-staging}"
export ENABLED_MODULES="${ENABLED_MODULES:-tenancy,access,admin,policy,audit,campaigns,files,mail,calendar,docs,ops}"
export DATABASE_URL="${DATABASE_URL:-${GOVOPLAN_PRODUCTION_LIKE_DATABASE_URL:-postgresql+psycopg://govoplan:govoplan-dev@127.0.0.1:55433/govoplan}}"
export GOVOPLAN_DATABASE_URL_PGTOOLS="${GOVOPLAN_DATABASE_URL_PGTOOLS:-${GOVOPLAN_PRODUCTION_LIKE_DATABASE_URL_PGTOOLS:-postgresql://govoplan:govoplan-dev@127.0.0.1:55433/govoplan}}"
export REDIS_URL="${REDIS_URL:-${GOVOPLAN_PRODUCTION_LIKE_REDIS_URL:-redis://127.0.0.1:56379/0}}"
export CELERY_ENABLED="${CELERY_ENABLED:-true}"
export CELERY_QUEUES="${CELERY_QUEUES:-send_email,append_sent,default}"
export FILE_STORAGE_BACKEND="${FILE_STORAGE_BACKEND:-local}"
export FILE_STORAGE_LOCAL_ROOT="${FILE_STORAGE_LOCAL_ROOT:-$ROOT/runtime/production-like/files}"
export DEV_AUTO_MIGRATE_ENABLED="${DEV_AUTO_MIGRATE_ENABLED:-false}"
export DEV_BOOTSTRAP_ENABLED="${DEV_BOOTSTRAP_ENABLED:-true}"
export CORS_ORIGINS="${CORS_ORIGINS:-http://127.0.0.1:$FRONTEND_PORT,http://localhost:$FRONTEND_PORT}"
if [ -z "${MASTER_KEY_B64:-}" ]; then
MASTER_KEY_B64="$("$PYTHON" - <<'PY'
from cryptography.fernet import Fernet
print(Fernet.generate_key().decode())
PY
)"
export MASTER_KEY_B64
fi
compose() {
"$DOCKER" compose --env-file "$PROFILE_ENV" -f "$PROFILE_ROOT/docker-compose.yml" "$@"
}
port_is_free() {
"$PYTHON" - "$1" "$2" <<'PY'
import socket
import sys
host = sys.argv[1]
port = int(sys.argv[2])
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
try:
sock.bind((host, port))
except OSError:
raise SystemExit(1)
PY
}
wait_for_tcp() {
"$PYTHON" - "$1" "$2" <<'PY'
import socket
import sys
import time
host = sys.argv[1]
port = int(sys.argv[2])
deadline = time.monotonic() + 90
last_error = None
while time.monotonic() < deadline:
try:
with socket.create_connection((host, port), timeout=2):
raise SystemExit(0)
except OSError as exc:
last_error = exc
time.sleep(1)
print(f"Timed out waiting for {host}:{port}: {last_error}", file=sys.stderr)
raise SystemExit(1)
PY
}
wait_for_url() {
"$PYTHON" - "$1" <<'PY'
import sys
import time
import urllib.request
url = sys.argv[1]
deadline = time.monotonic() + 90
last_error = None
while time.monotonic() < deadline:
try:
with urllib.request.urlopen(url, timeout=2) as response:
if 200 <= response.status < 500:
raise SystemExit(0)
except Exception as exc: # noqa: BLE001 - printed only on timeout.
last_error = exc
time.sleep(1)
print(f"Timed out waiting for {url}: {last_error}", file=sys.stderr)
raise SystemExit(1)
PY
}
cleanup() {
if [ -n "${frontend_pid:-}" ] && kill -0 "$frontend_pid" 2>/dev/null; then
kill "$frontend_pid" 2>/dev/null || true
fi
if [ -n "${backend_pid:-}" ] && kill -0 "$backend_pid" 2>/dev/null; then
kill "$backend_pid" 2>/dev/null || true
fi
if [ -n "${worker_pid:-}" ] && kill -0 "$worker_pid" 2>/dev/null; then
kill "$worker_pid" 2>/dev/null || true
fi
if [ "$STOP_DEPENDENCIES_ON_EXIT" = "1" ]; then
compose down >/dev/null 2>&1 || true
fi
}
trap cleanup EXIT INT TERM
[ -x "$PYTHON" ] || fail "Python virtualenv not found at $PYTHON. Run: cd $ROOT && ./.venv/bin/python -m pip install -r requirements-dev.txt"
[ -x "$NPM" ] || fail "npm not found at $NPM. Set NODE_BIN or NPM to your Node installation."
[ -f "$WEBUI_ROOT/package.json" ] || fail "WebUI package.json not found at $WEBUI_ROOT/package.json"
[ -d "$WEBUI_ROOT/node_modules" ] || fail "WebUI node_modules missing. Run: cd $WEBUI_ROOT && PATH=$NODE_BIN:\$PATH $NPM install"
mkdir -p "$LOG_DIR" "$FILE_STORAGE_LOCAL_ROOT"
: > "$BACKEND_LOG"
: > "$WORKER_LOG"
: > "$FRONTEND_LOG"
port_is_free "$BACKEND_HOST" "$BACKEND_PORT" || fail "$BACKEND_URL is already in use"
port_is_free "$FRONTEND_HOST" "$FRONTEND_PORT" || fail "$FRONTEND_URL is already in use"
printf 'Starting production-like dependencies through Docker Compose\n'
compose up -d
wait_for_tcp 127.0.0.1 "${GOVOPLAN_PRODUCTION_LIKE_POSTGRES_PORT:-55433}"
wait_for_tcp 127.0.0.1 "${GOVOPLAN_PRODUCTION_LIKE_REDIS_PORT:-56379}"
printf 'Running migrations and development bootstrap against %s\n' "$DATABASE_URL"
(
cd "$ROOT"
"$PYTHON" -m govoplan_core.commands.init_db --database-url "$DATABASE_URL" --with-dev-data
)
printf 'Starting GovOPlaN backend at %s\n' "$BACKEND_URL"
(
cd "$ROOT"
"$PYTHON" -m govoplan_core.devserver --host "$BACKEND_HOST" --port "$BACKEND_PORT"
) >"$BACKEND_LOG" 2>&1 &
backend_pid="$!"
printf 'Waiting for %s/health\n' "$BACKEND_URL"
wait_for_url "$BACKEND_URL/health" || {
tail -n 80 "$BACKEND_LOG" >&2 || true
fail "backend did not become healthy"
}
printf 'Starting Celery worker for queues %s\n' "$CELERY_QUEUES"
(
cd "$ROOT"
"$PYTHON" -m celery -A govoplan_core.celery_app:celery worker \
--queues "$CELERY_QUEUES" \
--hostname "govoplan-production-like@%h" \
--loglevel INFO
) >"$WORKER_LOG" 2>&1 &
worker_pid="$!"
printf 'Starting GovOPlaN WebUI at %s\n' "$FRONTEND_URL"
(
cd "$WEBUI_ROOT"
PATH="$WEBUI_ROOT/node_modules/.bin:$NODE_BIN:$PATH" \
CHOKIDAR_USEPOLLING="${GOVOPLAN_FRONTEND_USE_POLLING:-1}" \
CHOKIDAR_INTERVAL="${GOVOPLAN_FRONTEND_POLLING_INTERVAL:-500}" \
VITE_API_PROXY_TARGET="$BACKEND_URL" \
"$NPM" run dev -- --force
) >"$FRONTEND_LOG" 2>&1 &
frontend_pid="$!"
printf 'Waiting for %s\n' "$FRONTEND_URL"
wait_for_url "$FRONTEND_URL" || {
tail -n 80 "$FRONTEND_LOG" >&2 || true
fail "frontend did not become reachable"
}
if [ "$OPEN_BROWSER" = "1" ] && command -v xdg-open >/dev/null 2>&1; then
xdg-open "$FRONTEND_URL" >/dev/null 2>&1 || true
fi
cat <<EOF
GovOPlaN production-like development profile is running.
Web UI: $FRONTEND_URL
API: $BACKEND_URL/api/v1
Health: $BACKEND_URL/health
Ops: $BACKEND_URL/api/v1/ops/status
DB: $DATABASE_URL
Redis: $REDIS_URL
Files: $FILE_STORAGE_LOCAL_ROOT
Logs:
Backend: $BACKEND_LOG
Worker: $WORKER_LOG
WebUI: $FRONTEND_LOG
Docker dependencies remain running after Ctrl+C unless GOVOPLAN_STOP_PROFILE_DEPENDENCIES_ON_EXIT=1.
Press Ctrl+C to stop API, worker, and WebUI.
EOF
wait

View File

@@ -0,0 +1,512 @@
#!/usr/bin/env python
from __future__ import annotations
import argparse
import json
import os
import shlex
import shutil
import subprocess
import sys
import tempfile
from collections.abc import Callable, Iterable
from contextlib import contextmanager
from pathlib import Path
from types import SimpleNamespace
from typing import Any
from urllib.error import URLError
ROOT = Path(__file__).resolve().parents[1]
SRC = ROOT / "src"
if str(SRC) not in sys.path:
sys.path.insert(0, str(SRC))
os.environ.setdefault("APP_ENV", "test")
os.environ.setdefault("DEV_BOOTSTRAP_ENABLED", "false")
os.environ.setdefault("CELERY_ENABLED", "false")
from govoplan_core.admin.models import SystemSettings
from govoplan_core.core.maintenance import MaintenanceMode, save_maintenance_mode
from govoplan_core.core.module_installer import (
cancel_module_installer_request,
claim_next_module_installer_request,
module_installer_daemon_status,
module_installer_lock_status,
queue_module_installer_request,
read_module_installer_run,
retry_module_installer_request,
rollback_module_install_run,
run_module_install_plan,
supervise_module_install_plan,
update_module_installer_daemon_status,
update_module_installer_request,
)
from govoplan_core.core.module_management import (
ModuleInstallPlan,
ModuleInstallPlanItem,
saved_desired_enabled_modules,
saved_module_install_plan,
save_desired_enabled_modules,
save_module_install_plan,
)
from govoplan_core.core.modules import MigrationRetirementPlan, MigrationSpec, ModuleManifest
from govoplan_core.db.base import Base
from govoplan_core.db.session import configure_database, get_database
from govoplan_core.server.registry import available_module_manifests
Scenario = Callable[[Path], dict[str, object]]
def main() -> int:
parser = argparse.ArgumentParser(description="Run safe module-installer rollback drills in temporary runtimes.")
parser.add_argument("--scenario", action="append", choices=sorted(SCENARIOS), help="Scenario to run. Defaults to all scenarios.")
parser.add_argument("--runtime-root", type=Path, help="Directory for temporary drill runtimes. Defaults to a new temp directory.")
parser.add_argument("--keep-runtime", action="store_true", help="Keep the temporary drill runtime after completion.")
parser.add_argument("--format", choices=("json", "text"), default="text", help="Output format.")
args = parser.parse_args()
runtime_root = args.runtime_root or Path(tempfile.mkdtemp(prefix="govoplan-installer-drill-"))
runtime_root.mkdir(parents=True, exist_ok=True)
selected = args.scenario or sorted(SCENARIOS)
results: list[dict[str, object]] = []
ok = True
try:
for name in selected:
scenario_root = runtime_root / name
scenario_root.mkdir(parents=True, exist_ok=True)
try:
result = SCENARIOS[name](scenario_root)
except Exception as exc:
ok = False
result = {"scenario": name, "ok": False, "error": str(exc), "root": str(scenario_root)}
else:
ok = ok and bool(result.get("ok"))
results.append(result)
finally:
if not args.keep_runtime and args.runtime_root is None:
shutil.rmtree(runtime_root, ignore_errors=True)
payload = {"ok": ok, "runtime_root": str(runtime_root), "scenarios": results}
if args.format == "json":
print(json.dumps(payload, indent=2, sort_keys=True))
else:
for result in results:
status = "ok" if result.get("ok") else "FAILED"
print(f"{status}: {result['scenario']}")
if result.get("run_id"):
print(f" run: {result['run_id']}")
if result.get("record_path"):
print(f" record: {result['record_path']}")
if result.get("error"):
print(f" error: {result['error']}")
print(f"runtime: {runtime_root}")
return 0 if ok else 1
def package_failure(root: Path) -> dict[str, object]:
database_url, database = _sqlite_database(root)
plan = _saved_install_plan(database, "package-failure-example")
with database.session() as session, _patch_subprocess(_package_failure_run):
result = supervise_module_install_plan(
session=session,
plan=plan,
available=available_module_manifests(),
current_enabled=("tenancy", "access"),
desired_enabled=("tenancy", "access"),
database_url=database_url,
runtime_dir=root / "installer",
)
restored_plan = saved_module_install_plan(session)
record = _run_record(root, result.run_id)
return _scenario_result(
"package-failure",
root,
result,
expected_status="rolled-back",
checks={
"supervisor_status": _nested(record, "supervisor", "status") == "rolled-back",
"plan_restored": tuple(item.status for item in restored_plan.items) == ("planned",),
},
)
def migration_failure(root: Path) -> dict[str, object]:
database_url, database = _sqlite_database(root)
plan = _saved_install_plan(database, "migration-failure-example")
with database.session() as session, _patch_subprocess(_migration_failure_run):
result = supervise_module_install_plan(
session=session,
plan=plan,
available=available_module_manifests(),
current_enabled=("tenancy", "access"),
desired_enabled=("tenancy", "access"),
database_url=database_url,
runtime_dir=root / "installer",
migrate_database=True,
)
record = _run_record(root, result.run_id)
return _scenario_result(
"migration-failure",
root,
result,
expected_status="rolled-back",
checks={
"sqlite_backup_recorded": isinstance(_nested(record, "snapshot", "database_backup"), dict),
"supervisor_status": _nested(record, "supervisor", "status") == "rolled-back",
},
)
def restart_failure(root: Path) -> dict[str, object]:
database_url, database = _sqlite_database(root)
plan = _saved_install_plan(database, "restart-failure-example")
with database.session() as session, _patch_subprocess(_restart_failure_run):
result = supervise_module_install_plan(
session=session,
plan=plan,
available=available_module_manifests(),
current_enabled=("tenancy", "access"),
desired_enabled=("tenancy", "access"),
database_url=database_url,
runtime_dir=root / "installer",
restart_command="restart-fails",
)
record = _run_record(root, result.run_id)
return _scenario_result(
"restart-failure",
root,
result,
expected_status="rolled-back",
checks={"supervisor_status": _nested(record, "supervisor", "status") == "rolled-back"},
)
def health_timeout(root: Path) -> dict[str, object]:
database_url, database = _sqlite_database(root)
plan = _saved_install_plan(database, "health-timeout-example")
with database.session() as session, _patch_subprocess(_success_run), _patch_urlopen():
result = supervise_module_install_plan(
session=session,
plan=plan,
available=available_module_manifests(),
current_enabled=("tenancy", "access"),
desired_enabled=("tenancy", "access"),
database_url=database_url,
runtime_dir=root / "installer",
restart_command="restart-ok",
health_url="http://127.0.0.1:9/health",
health_timeout_seconds=0.2,
health_interval_seconds=0.05,
)
record = _run_record(root, result.run_id)
return _scenario_result(
"health-timeout",
root,
result,
expected_status="rolled-back",
checks={
"health_failure_recorded": "127.0.0.1:9/health" in str(_nested(record, "supervisor", "failure_reason")),
"supervisor_status": _nested(record, "supervisor", "status") == "rolled-back",
},
)
def destructive_retirement_failure(root: Path) -> dict[str, object]:
database_url, database = _sqlite_database(root)
available = {"retirement-example": _retirement_failure_manifest()}
with database.session() as session:
save_maintenance_mode(session, MaintenanceMode(enabled=True))
plan = save_module_install_plan(session, [{
"module_id": "retirement-example",
"action": "uninstall",
"python_package": "govoplan-retirement-example",
"destroy_data": True,
}])
session.commit()
with _patch_subprocess(_success_run):
result = run_module_install_plan(
session=session,
plan=plan,
available=available,
current_enabled=("tenancy", "access"),
desired_enabled=("tenancy", "access"),
database_url=database_url,
runtime_dir=root / "installer",
)
record = _run_record(root, result.run_id)
return _scenario_result(
"destructive-retirement-failure",
root,
result,
expected_status="rolled-back",
checks={
"rollback_recorded": isinstance(record.get("destructive_retirement_rollback"), dict),
"sqlite_backup_recorded": isinstance(_nested(record, "snapshot", "database_backup"), dict),
},
)
def postgres_hooks(root: Path) -> dict[str, object]:
_, database = _sqlite_database(root)
backup_script = (
"import json, os, pathlib; "
"pathlib.Path(os.environ['GOVOPLAN_DATABASE_BACKUP_PATH']).write_text('backup', encoding='utf-8'); "
"pathlib.Path(os.environ['GOVOPLAN_DATABASE_BACKUP_METADATA']).write_text(json.dumps({'snapshot': 'external'}), encoding='utf-8')"
)
check_script = (
"import os, pathlib; "
"pathlib.Path(os.environ['GOVOPLAN_INSTALLER_RUN_DIR'], 'restore-check.marker').write_text("
"pathlib.Path(os.environ['GOVOPLAN_DATABASE_BACKUP_PATH']).read_text(encoding='utf-8'), encoding='utf-8')"
)
restore_script = (
"import os, pathlib; "
"pathlib.Path(os.environ['GOVOPLAN_INSTALLER_RUN_DIR'], 'restore.marker').write_text("
"os.environ.get('GOVOPLAN_DATABASE_URL', ''), encoding='utf-8')"
)
with database.session() as session:
save_maintenance_mode(session, MaintenanceMode(enabled=True))
plan = save_module_install_plan(session, [{
"module_id": "postgres-hook-example",
"action": "install",
"python_package": "govoplan-postgres-hook-example",
"python_ref": "govoplan-postgres-hook-example==0.1.0",
}])
session.commit()
result = run_module_install_plan(
session=session,
plan=plan,
available=available_module_manifests(),
current_enabled=("tenancy", "access"),
desired_enabled=("tenancy", "access"),
database_url="postgresql://db.example.invalid/govoplan",
runtime_dir=root / "installer",
migrate_database=True,
database_backup_command=f"{sys.executable} -c {shlex.quote(backup_script)}",
database_restore_command=f"{sys.executable} -c {shlex.quote(restore_script)}",
database_restore_check_command=f"{sys.executable} -c {shlex.quote(check_script)}",
dry_run=True,
)
with _patch_subprocess(_non_shell_success_run):
rollback = rollback_module_install_run(
run_id=result.run_id,
runtime_dir=root / "installer",
database_url="postgresql://db.example.invalid/govoplan",
)
run_dir = result.record_path.parent
record = _run_record(root, result.run_id)
ok = (
result.status == "dry-run"
and rollback.status == "rolled-back"
and (run_dir / "restore-check.marker").read_text(encoding="utf-8") == "backup"
and (run_dir / "restore.marker").read_text(encoding="utf-8") == "postgresql://db.example.invalid/govoplan"
and isinstance(_nested(record, "snapshot", "database_backup"), dict)
)
return {
"scenario": "postgres-hooks",
"ok": ok,
"root": str(root),
"run_id": result.run_id,
"record_path": str(result.record_path),
"rollback_status": rollback.status,
}
def queue_and_lock(root: Path) -> dict[str, object]:
runtime_dir = root / "installer"
status = update_module_installer_daemon_status(
runtime_dir=runtime_dir,
patch={"status": "polling", "current_request_id": None},
)
request = queue_module_installer_request(
runtime_dir=runtime_dir,
requested_by="drill",
options={"migrate_database": True, "health_urls": ["http://127.0.0.1:8000/health"]},
)
claimed = claim_next_module_installer_request(runtime_dir=runtime_dir)
assert claimed is not None
failed = update_module_installer_request(
runtime_dir=runtime_dir,
request_id=str(claimed["request_id"]),
patch={"status": "failed", "error": "simulated drill failure"},
)
retry = retry_module_installer_request(runtime_dir=runtime_dir, request_id=str(failed["request_id"]), requested_by="drill")
cancelled = cancel_module_installer_request(runtime_dir=runtime_dir, request_id=str(retry["request_id"]), cancelled_by="drill")
lock_path = runtime_dir / "install.lock"
lock_path.write_text(json.dumps({"pid": 999999, "created_at": "drill"}), encoding="utf-8")
lock = module_installer_lock_status(runtime_dir=runtime_dir)
lock_path.unlink()
return {
"scenario": "queue-and-lock",
"ok": bool(status["running"]) and request["status"] == "queued" and failed["status"] == "failed" and cancelled["status"] == "cancelled" and lock["locked"] is True and not module_installer_lock_status(runtime_dir=runtime_dir)["locked"],
"root": str(root),
"daemon_status_path": str(runtime_dir / "daemon.status.json"),
}
def _sqlite_database(root: Path) -> tuple[str, Any]:
database_url = f"sqlite:///{root / 'drill.db'}"
configure_database(database_url)
database = get_database()
Base.metadata.create_all(bind=database.engine, tables=[SystemSettings.__table__])
return database_url, database
def _saved_install_plan(database: Any, module_id: str) -> ModuleInstallPlan:
with database.session() as session:
save_maintenance_mode(session, MaintenanceMode(enabled=True))
plan = save_module_install_plan(session, [{
"module_id": module_id,
"action": "install",
"python_package": f"govoplan-{module_id}",
"python_ref": f"govoplan-{module_id}==0.1.0",
}])
save_desired_enabled_modules(session, ("tenancy", "access"))
session.commit()
return plan
def _retirement_failure_manifest() -> ModuleManifest:
def destroy(_session: object, _module_id: str) -> None:
raise RuntimeError("simulated destructive retirement failure")
def provider(_session: object | None, _module_id: str) -> MigrationRetirementPlan:
return MigrationRetirementPlan(
supported=True,
summary="Drill retirement provider supports destructive cleanup.",
destroy_data_supported=True,
destroy_data_summary="Drill destructive cleanup will fail during execution.",
destroy_data_executor=destroy,
)
return ModuleManifest(
id="retirement-example",
name="Retirement example",
version="0.1.0",
migration_spec=MigrationSpec(
module_id="retirement-example",
retirement_supported=True,
retirement_provider=provider,
),
)
def _scenario_result(
scenario: str,
root: Path,
result: Any,
*,
expected_status: str,
checks: dict[str, bool],
) -> dict[str, object]:
return {
"scenario": scenario,
"ok": result.status == expected_status and all(checks.values()),
"root": str(root),
"run_id": result.run_id,
"record_path": str(result.record_path),
"status": result.status,
"checks": checks,
}
def _run_record(root: Path, run_id: str) -> dict[str, object]:
return read_module_installer_run(runtime_dir=root / "installer", run_id=run_id)
def _nested(value: dict[str, object], *keys: str) -> object:
current: object = value
for key in keys:
if not isinstance(current, dict):
return None
current = current.get(key)
return current
def _command_text(argv: object) -> str:
if isinstance(argv, list | tuple):
return " ".join(str(item) for item in argv)
return str(argv)
def _completed(return_code: int = 0, stdout: str = "", stderr: str = "") -> SimpleNamespace:
return SimpleNamespace(returncode=return_code, stdout=stdout, stderr=stderr)
def _success_run(argv: object, *_args: object, **_kwargs: object) -> SimpleNamespace:
text = _command_text(argv)
if "pip freeze" in text:
return _completed(stdout="govoplan-core==0.0.0\n")
return _completed()
def _non_shell_success_run(argv: object, *_args: object, **kwargs: object) -> Any:
if kwargs.get("shell"):
return ORIGINAL_SUBPROCESS_RUN(argv, *_args, **kwargs)
return _success_run(argv, *_args, **kwargs)
def _package_failure_run(argv: object, *_args: object, **kwargs: object) -> SimpleNamespace:
text = _command_text(argv)
if "pip install" in text and "==" in text and "-r" not in text:
return _completed(23, stderr="simulated package install failure")
return _success_run(argv, *_args, **kwargs)
def _migration_failure_run(argv: object, *_args: object, **kwargs: object) -> SimpleNamespace:
text = _command_text(argv)
if "govoplan_core.commands.init_db" in text:
return _completed(24, stderr="simulated migration failure")
return _success_run(argv, *_args, **kwargs)
def _restart_failure_run(argv: object, *_args: object, **kwargs: object) -> SimpleNamespace:
if kwargs.get("shell") and str(argv).strip() == "restart-fails":
return _completed(25, stderr="simulated restart failure")
return _success_run(argv, *_args, **kwargs)
ORIGINAL_SUBPROCESS_RUN = subprocess.run
@contextmanager
def _patch_subprocess(fake_run: Callable[..., Any]) -> Iterable[None]:
import govoplan_core.core.module_installer as installer
original = installer.subprocess.run
installer.subprocess.run = fake_run # type: ignore[assignment]
try:
yield
finally:
installer.subprocess.run = original # type: ignore[assignment]
@contextmanager
def _patch_urlopen() -> Iterable[None]:
import govoplan_core.core.module_installer as installer
original = installer.urllib.request.urlopen
def fail(*_args: object, **_kwargs: object) -> None:
raise URLError("simulated health timeout")
installer.urllib.request.urlopen = fail # type: ignore[assignment]
try:
yield
finally:
installer.urllib.request.urlopen = original # type: ignore[assignment]
SCENARIOS: dict[str, Scenario] = {
"destructive-retirement-failure": destructive_retirement_failure,
"health-timeout": health_timeout,
"migration-failure": migration_failure,
"package-failure": package_failure,
"postgres-hooks": postgres_hooks,
"queue-and-lock": queue_and_lock,
"restart-failure": restart_failure,
}
if __name__ == "__main__":
raise SystemExit(main())

View File

@@ -0,0 +1,128 @@
#!/usr/bin/env python
from __future__ import annotations
import argparse
import base64
import os
from pathlib import Path
import subprocess
import sys
ROOT = Path(__file__).resolve().parents[1]
SRC = ROOT / "src"
DEFAULT_MODULE_SETS: tuple[tuple[str, str], ...] = (
("core-only", "tenancy,access,admin"),
("files-only", "tenancy,access,admin,files"),
("mail-only", "tenancy,access,admin,mail"),
("campaign-only", "tenancy,access,admin,campaigns"),
("campaign-with-files", "tenancy,access,admin,campaigns,files"),
("campaign-with-mail", "tenancy,access,admin,campaigns,mail"),
("full-product", "tenancy,access,admin,policy,audit,campaigns,files,mail,calendar,docs,ops"),
)
def main() -> int:
parser = argparse.ArgumentParser(
description="Run GovOPlaN migrations and startup smoke checks against a disposable PostgreSQL database.",
)
parser.add_argument(
"--database-url",
default=os.environ.get("GOVOPLAN_POSTGRES_DATABASE_URL") or os.environ.get("DATABASE_URL"),
help="SQLAlchemy PostgreSQL URL, for example postgresql+psycopg://user:pass@127.0.0.1:55432/govoplan.",
)
parser.add_argument(
"--module-set",
action="append",
default=[],
metavar="NAME=MODULES",
help="Module permutation to check. May be passed multiple times. Defaults to the standard product permutations.",
)
parser.add_argument(
"--reset-schema",
action="store_true",
help="DROP and recreate the public schema before every module set. Use only with a disposable database.",
)
parser.add_argument("--skip-migrate", action="store_true", help="Skip govoplan_core.commands.init_db.")
parser.add_argument("--skip-smoke", action="store_true", help="Skip govoplan_core.devserver --smoke.")
parser.add_argument("--python", default=sys.executable, help="Python executable to use. Defaults to the current interpreter.")
parser.add_argument("--app-env", default="staging", help="APP_ENV value to pass to subprocesses. Defaults to staging.")
args = parser.parse_args()
database_url = (args.database_url or "").strip()
if not database_url:
parser.error("--database-url or GOVOPLAN_POSTGRES_DATABASE_URL is required")
if not database_url.startswith("postgresql"):
parser.error("This check is PostgreSQL-only. Use a postgresql+psycopg:// or postgresql:// DATABASE_URL.")
module_sets = _parse_module_sets(args.module_set)
ok = True
for name, modules in module_sets:
print(f"\n== {name}: {modules} ==")
if args.reset_schema:
_reset_public_schema(database_url)
env = _check_env(database_url=database_url, modules=modules, app_env=args.app_env)
if not args.skip_migrate:
ok = _run([args.python, "-m", "govoplan_core.commands.init_db", "--database-url", database_url], env=env) and ok
if not args.skip_smoke:
ok = _run([args.python, "-m", "govoplan_core.devserver", "--smoke", "--no-reload"], env=env) and ok
return 0 if ok else 1
def _parse_module_sets(raw: list[str]) -> tuple[tuple[str, str], ...]:
if not raw:
return DEFAULT_MODULE_SETS
parsed: list[tuple[str, str]] = []
for item in raw:
name, separator, modules = item.partition("=")
if not separator or not name.strip() or not modules.strip():
raise SystemExit(f"Invalid --module-set value {item!r}. Use NAME=module_a,module_b.")
parsed.append((name.strip(), modules.strip()))
return tuple(parsed)
def _check_env(*, database_url: str, modules: str, app_env: str) -> dict[str, str]:
env = os.environ.copy()
env["APP_ENV"] = app_env
env["DATABASE_URL"] = database_url
env["ENABLED_MODULES"] = modules
env["DEV_AUTO_MIGRATE_ENABLED"] = "false"
env["DEV_BOOTSTRAP_ENABLED"] = "false"
env["CELERY_ENABLED"] = "false"
env.setdefault("MASTER_KEY_B64", base64.urlsafe_b64encode(os.urandom(32)).decode("ascii"))
env.setdefault("FILE_STORAGE_BACKEND", "local")
env.setdefault("FILE_STORAGE_LOCAL_ROOT", str(ROOT / "runtime" / "postgres-check-files"))
env.setdefault("MOCK_MAILBOX_DIR", str(ROOT / "runtime" / "postgres-check-mock-mailbox"))
env["PYTHONPATH"] = os.pathsep.join(part for part in (str(SRC), env.get("PYTHONPATH", "")) if part)
return env
def _run(command: list[str], *, env: dict[str, str]) -> bool:
print("+ " + " ".join(command))
result = subprocess.run(command, cwd=ROOT, env=env, check=False)
if result.returncode != 0:
print(f"Command failed with exit code {result.returncode}: {' '.join(command)}", file=sys.stderr)
return False
return True
def _reset_public_schema(database_url: str) -> None:
try:
from sqlalchemy import create_engine
except ModuleNotFoundError as exc:
raise SystemExit("SQLAlchemy is required for --reset-schema. Install govoplan-core requirements first.") from exc
engine = create_engine(database_url, isolation_level="AUTOCOMMIT")
try:
with engine.connect() as connection:
connection.exec_driver_sql("DROP SCHEMA IF EXISTS public CASCADE")
connection.exec_driver_sql("CREATE SCHEMA public")
connection.exec_driver_sql("GRANT ALL ON SCHEMA public TO public")
finally:
engine.dispose()
print("Reset PostgreSQL schema: public")
if __name__ == "__main__":
raise SystemExit(main())

View File

@@ -24,6 +24,11 @@ Options:
-m, --message <text> Commit message. Defaults to "Release v<x.y.z>".
--tag-message <text> Annotated tag message. Defaults to the commit message.
--skip-release-lock Do not regenerate core webui/package-lock.release.json.
--warn-migration-audit Run the migration release audit without baseline strictness.
--strict-migration-audit
Require current Alembic heads to be recorded in the
latest migration release baseline.
--skip-migration-audit Do not run the release migration audit.
--publish-web-catalog After core/modules are pushed, publish a signed release
catalog into govoplan-web.
--web-root <path> govoplan-web checkout for --publish-web-catalog.
@@ -51,18 +56,22 @@ Repos:
govoplan-access
govoplan-admin
govoplan-tenancy
govoplan-organizations
govoplan-identity
govoplan-policy
govoplan-audit
govoplan-dashboard
govoplan-files
govoplan-mail
govoplan-campaign
govoplan-calendar
govoplan-ops
govoplan-core
Roadmap/scaffold module repositories are tag-only until they have package
metadata: addresses, appointments, cases, connectors, dms, erp,
fit-connect, forms, identity-trust, idm, ledger, notifications, ops,
payments, portal, reporting, scheduling, search, tasks, templates, workflow, xoev,
fit-connect, forms, identity-trust, idm, ledger, notifications, payments,
portal, reporting, scheduling, search, tasks, templates, workflow, xoev,
xrechnung, and xta-osci.
Examples:
@@ -83,12 +92,16 @@ PACKAGE_MODULE_REPOS=(
"$PARENT/govoplan-access"
"$PARENT/govoplan-admin"
"$PARENT/govoplan-tenancy"
"$PARENT/govoplan-organizations"
"$PARENT/govoplan-identity"
"$PARENT/govoplan-policy"
"$PARENT/govoplan-audit"
"$PARENT/govoplan-dashboard"
"$PARENT/govoplan-files"
"$PARENT/govoplan-mail"
"$PARENT/govoplan-campaign"
"$PARENT/govoplan-calendar"
"$PARENT/govoplan-ops"
)
TAG_ONLY_MODULE_REPOS=(
"$PARENT/govoplan-addresses"
@@ -103,7 +116,6 @@ TAG_ONLY_MODULE_REPOS=(
"$PARENT/govoplan-idm"
"$PARENT/govoplan-ledger"
"$PARENT/govoplan-notifications"
"$PARENT/govoplan-ops"
"$PARENT/govoplan-payments"
"$PARENT/govoplan-portal"
"$PARENT/govoplan-reporting"
@@ -132,6 +144,7 @@ TAG_MESSAGE=""
DRY_RUN=0
YES=0
GENERATE_RELEASE_LOCK=1
MIGRATION_AUDIT_MODE="auto"
NPM_BIN="${NPM:-}"
PUBLISH_WEB_CATALOG=0
WEB_ROOT="$PARENT/govoplan-web"
@@ -282,6 +295,12 @@ update_manifest_version() {
govoplan-tenancy)
manifest_path="$repo/src/govoplan_tenancy/backend/manifest.py"
;;
govoplan-organizations)
manifest_path="$repo/src/govoplan_organizations/backend/manifest.py"
;;
govoplan-identity)
manifest_path="$repo/src/govoplan_identity/backend/manifest.py"
;;
govoplan-policy)
manifest_path="$repo/src/govoplan_policy/backend/manifest.py"
;;
@@ -408,6 +427,32 @@ generate_release_lock() {
run "$ROOT/scripts/generate-release-lock.sh" --npm "$NPM_BIN"
}
run_migration_release_audit() {
local audit_script="$ROOT/scripts/release-migration-audit.py"
local command=("$PYTHON" "$audit_script")
case "$MIGRATION_AUDIT_MODE" in
skip)
echo "Migration release audit: skipped"
return
;;
strict)
command+=("--strict")
;;
auto)
command+=("--strict-if-baseline")
;;
warn)
;;
*)
fail "unknown migration audit mode: $MIGRATION_AUDIT_MODE"
;;
esac
[[ -f "$audit_script" ]] || fail "missing migration audit helper: $audit_script"
run "${command[@]}"
}
print_command() {
printf '+'
printf ' %q' "$@"
@@ -514,6 +559,18 @@ while [[ $# -gt 0 ]]; do
GENERATE_RELEASE_LOCK=0
shift
;;
--warn-migration-audit)
MIGRATION_AUDIT_MODE="warn"
shift
;;
--strict-migration-audit)
MIGRATION_AUDIT_MODE="strict"
shift
;;
--skip-migration-audit)
MIGRATION_AUDIT_MODE="skip"
shift
;;
--publish-web-catalog)
PUBLISH_WEB_CATALOG=1
shift
@@ -690,6 +747,7 @@ if [[ "$GENERATE_RELEASE_LOCK" -eq 1 ]]; then
else
echo " release lock: skipped"
fi
echo " migration audit: $MIGRATION_AUDIT_MODE"
if [[ "$PUBLISH_WEB_CATALOG" -eq 1 ]]; then
echo " web catalog: publish $CATALOG_CHANNEL catalog to $WEB_ROOT after core tag is pushed"
else
@@ -707,6 +765,8 @@ for repo in "${REPOS[@]}"; do
echo
done
run_migration_release_audit
confirm_release
for repo in "${PACKAGE_REPOS[@]}"; do

View File

@@ -0,0 +1,422 @@
#!/usr/bin/env python
from __future__ import annotations
import argparse
import ast
from dataclasses import dataclass
from datetime import datetime, timezone
import json
from pathlib import Path
from typing import Any
ROOT = Path(__file__).resolve().parents[1]
DEFAULT_BASELINE_FILE = ROOT / "docs" / "migration-release-baselines.json"
@dataclass(frozen=True, slots=True)
class Migration:
owner: str
path: Path
revision: str
down_revisions: tuple[str, ...]
branch_labels: tuple[str, ...]
def main() -> int:
parser = argparse.ArgumentParser(
description="Audit GovOPlaN Alembic migrations against the release baseline policy.",
)
parser.add_argument(
"--workspace-root",
type=Path,
default=ROOT.parent,
help="Directory containing govoplan-* checkouts. Defaults to the parent of govoplan-core.",
)
parser.add_argument(
"--baseline-file",
type=Path,
default=DEFAULT_BASELINE_FILE,
help="JSON file recording released migration heads.",
)
parser.add_argument(
"--strict",
action="store_true",
help="Fail when current heads are not recorded in the latest release baseline.",
)
parser.add_argument(
"--strict-if-baseline",
action="store_true",
help="Run in strict mode only after at least one release baseline is recorded.",
)
parser.add_argument(
"--record-release",
metavar="VERSION",
help="Record the current reviewed migration heads as a release baseline.",
)
parser.add_argument(
"--replace-release",
action="store_true",
help="Replace an existing release entry when used with --record-release.",
)
parser.add_argument(
"--squash-plan",
action="store_true",
help="Print the reviewed/manual squash checklist for the current graph.",
)
parser.add_argument("--json", action="store_true", help="Emit machine-readable JSON.")
args = parser.parse_args()
migrations = discover_migrations(args.workspace_root)
baseline = load_baseline_file(args.baseline_file)
report = build_report(
migrations,
baseline=baseline,
baseline_file=args.baseline_file,
workspace_root=args.workspace_root,
)
if args.record_release:
if report["graph_errors"]:
print_text_report(report)
return 1
baseline = record_release_baseline(
baseline,
report,
release=args.record_release,
replace=args.replace_release,
)
write_baseline_file(args.baseline_file, baseline)
report = build_report(
migrations,
baseline=baseline,
baseline_file=args.baseline_file,
workspace_root=args.workspace_root,
)
if args.squash_plan:
print_squash_plan(report)
elif args.json:
print(json.dumps(report, indent=2, sort_keys=True))
else:
print_text_report(report)
has_graph_errors = bool(report["graph_errors"])
has_strict_errors = bool(report["strict_errors"])
strict_required = args.strict or (args.strict_if_baseline and report["release_count"] > 0)
if has_graph_errors or (strict_required and has_strict_errors):
return 1
return 0
def discover_migrations(workspace_root: Path) -> list[Migration]:
migrations: list[Migration] = []
for versions_dir in _migration_version_dirs(workspace_root):
owner = owner_for_versions_dir(versions_dir)
for path in sorted(versions_dir.glob("*.py")):
if path.name == "__init__.py":
continue
migration = parse_migration_file(owner, path)
if migration is not None:
migrations.append(migration)
return migrations
def _migration_version_dirs(workspace_root: Path) -> list[Path]:
candidates = [ROOT / "alembic" / "versions"]
for repo in sorted(workspace_root.glob("govoplan-*")):
candidates.extend(sorted(repo.glob("src/govoplan_*/backend/migrations/versions")))
seen: set[Path] = set()
result: list[Path] = []
for candidate in candidates:
resolved = candidate.resolve()
if resolved in seen or not candidate.is_dir():
continue
seen.add(resolved)
result.append(candidate)
return result
def owner_for_versions_dir(versions_dir: Path) -> str:
if (ROOT / "alembic" / "versions").resolve() == versions_dir.resolve():
return "govoplan-core"
for parent in versions_dir.parents:
if parent.name.startswith("govoplan-"):
return parent.name
return versions_dir.parent.name
def parse_migration_file(owner: str, path: Path) -> Migration | None:
tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
values: dict[str, Any] = {}
for statement in tree.body:
if isinstance(statement, ast.Assign):
for target in statement.targets:
if isinstance(target, ast.Name) and target.id in {"revision", "down_revision", "branch_labels"}:
values[target.id] = ast.literal_eval(statement.value)
elif (
isinstance(statement, ast.AnnAssign)
and isinstance(statement.target, ast.Name)
and statement.target.id in {"revision", "down_revision", "branch_labels"}
and statement.value is not None
):
values[statement.target.id] = ast.literal_eval(statement.value)
revision = values.get("revision")
if not isinstance(revision, str):
return None
return Migration(
owner=owner,
path=path,
revision=revision,
down_revisions=_normalize_revision_tuple(values.get("down_revision")),
branch_labels=_normalize_string_tuple(values.get("branch_labels")),
)
def _normalize_revision_tuple(value: Any) -> tuple[str, ...]:
if value is None:
return ()
if isinstance(value, str):
return (value,)
if isinstance(value, (list, tuple, set)):
return tuple(str(item) for item in value if item is not None)
return (str(value),)
def _normalize_string_tuple(value: Any) -> tuple[str, ...]:
if value is None:
return ()
if isinstance(value, str):
return (value,)
if isinstance(value, (list, tuple, set)):
return tuple(str(item) for item in value)
return (str(value),)
def load_baseline_file(path: Path) -> dict[str, Any]:
if not path.exists():
return {"version": 1, "releases": [], "_missing": True}
payload = json.loads(path.read_text(encoding="utf-8"))
if not isinstance(payload, dict):
raise SystemExit(f"{path} must contain a JSON object")
releases = payload.get("releases")
if not isinstance(releases, list):
raise SystemExit(f"{path} must contain a releases list")
return payload
def build_report(
migrations: list[Migration],
*,
baseline: dict[str, Any],
baseline_file: Path,
workspace_root: Path,
) -> dict[str, Any]:
revisions: dict[str, Migration] = {}
graph_errors: list[str] = []
for migration in migrations:
existing = revisions.get(migration.revision)
if existing is not None:
graph_errors.append(
f"duplicate revision {migration.revision}: {existing.path} and {migration.path}"
)
revisions[migration.revision] = migration
referenced = {revision for migration in migrations for revision in migration.down_revisions}
for migration in migrations:
for down_revision in migration.down_revisions:
if down_revision not in revisions:
graph_errors.append(
f"{migration.revision} references missing down_revision {down_revision} in {migration.path}"
)
current_heads = sorted(revision for revision in revisions if revision not in referenced)
current_head_entries = [
{
"owner": revisions[revision].owner,
"revision": revision,
}
for revision in current_heads
]
owner_rows = []
for owner in sorted({migration.owner for migration in migrations}):
owner_migrations = [migration for migration in migrations if migration.owner == owner]
owner_revisions = {migration.revision for migration in owner_migrations}
owner_referenced = {
revision
for migration in owner_migrations
for revision in migration.down_revisions
if revision in owner_revisions
}
owner_heads = sorted(owner_revisions - owner_referenced)
owner_rows.append(
{
"owner": owner,
"migrations": len(owner_migrations),
"heads": owner_heads,
}
)
releases = baseline.get("releases") or []
latest_release = releases[-1] if releases else None
latest_heads = _latest_release_heads(latest_release)
unrecorded_heads = sorted(set(current_heads) - latest_heads) if latest_release else current_heads
strict_errors: list[str] = []
if baseline.get("_missing"):
strict_errors.append(f"baseline file is missing: {baseline_file}")
if not releases:
strict_errors.append("no released migration baseline has been recorded yet")
if unrecorded_heads:
strict_errors.append(
"current migration heads are not recorded in the latest release baseline: "
+ ", ".join(unrecorded_heads)
)
return {
"workspace_root": str(workspace_root),
"baseline_file": str(baseline_file),
"baseline_file_missing": bool(baseline.get("_missing")),
"release_count": len(releases),
"latest_release": latest_release,
"owners": owner_rows,
"current_heads": current_heads,
"current_head_entries": current_head_entries,
"graph_errors": graph_errors,
"strict_errors": strict_errors,
}
def _latest_release_heads(latest_release: Any) -> set[str]:
if not isinstance(latest_release, dict):
return set()
heads = latest_release.get("heads") or []
graph_heads = latest_release.get("graph_heads") or []
result: set[str] = set()
for head_list in (heads, graph_heads):
if not isinstance(head_list, list):
continue
for entry in head_list:
if isinstance(entry, str):
result.add(entry)
elif isinstance(entry, dict) and isinstance(entry.get("revision"), str):
result.add(entry["revision"])
owner_heads = latest_release.get("owner_heads") or []
if isinstance(owner_heads, list):
for entry in owner_heads:
if not isinstance(entry, dict):
continue
revisions = entry.get("revisions") or []
if isinstance(revisions, str):
result.add(revisions)
elif isinstance(revisions, list):
result.update(revision for revision in revisions if isinstance(revision, str))
return result
def record_release_baseline(
baseline: dict[str, Any],
report: dict[str, Any],
*,
release: str,
replace: bool,
) -> dict[str, Any]:
payload = {
key: value
for key, value in baseline.items()
if not key.startswith("_")
}
payload.setdefault("version", 1)
releases = list(payload.get("releases") or [])
existing_index = next(
(index for index, item in enumerate(releases) if isinstance(item, dict) and item.get("release") == release),
None,
)
if existing_index is not None and not replace:
raise SystemExit(f"release {release} already exists in baseline file; pass --replace-release to update it")
entry = {
"release": release,
"recorded_at": datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z"),
"squash_policy": "reviewed-manual",
"heads": report["current_head_entries"],
"owner_heads": [
{
"owner": owner["owner"],
"revisions": owner["heads"],
}
for owner in report["owners"]
],
}
if existing_index is None:
releases.append(entry)
else:
releases[existing_index] = entry
payload["releases"] = releases
return payload
def write_baseline_file(path: Path, baseline: dict[str, Any]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(baseline, indent=2, sort_keys=True) + "\n", encoding="utf-8")
def print_text_report(report: dict[str, Any]) -> None:
print("Migration release audit")
print(f" workspace: {report['workspace_root']}")
print(f" baseline file: {report['baseline_file']}")
print(f" recorded releases: {report['release_count']}")
print()
print("Owners:")
for owner in report["owners"]:
heads = ", ".join(owner["heads"]) if owner["heads"] else "-"
print(f" {owner['owner']}: {owner['migrations']} migration(s), head(s): {heads}")
print()
print("Current heads:")
for entry in report["current_head_entries"]:
print(f" {entry['owner']}: {entry['revision']}")
if not report["current_head_entries"]:
print(" -")
if report["graph_errors"]:
print()
print("Graph errors:")
for error in report["graph_errors"]:
print(f" - {error}")
if report["strict_errors"]:
print()
print("Release baseline warnings:")
for error in report["strict_errors"]:
print(f" - {error}")
def print_squash_plan(report: dict[str, Any]) -> None:
print("Manual migration squash plan")
print()
print("Policy:")
print(" - Do not rewrite revision IDs that have shipped to real installations.")
print(" - Squash only unreleased development migrations.")
print(" - Review generated baseline/upgrade migrations like normal schema code.")
print(" - Run PostgreSQL migration smoke checks after any squash.")
print()
print("Current owner heads:")
for owner in report["owners"]:
heads = ", ".join(owner["heads"]) if owner["heads"] else "-"
print(f" - {owner['owner']}: {heads}")
print()
print("Current Alembic graph heads:")
for entry in report["current_head_entries"]:
print(f" - {entry['owner']}: {entry['revision']}")
if not report["current_head_entries"]:
print(" -")
print()
print("Release steps:")
print(" 1. Decide which migration revisions are unreleased and may be folded.")
print(" 2. Replace those development chains with reviewed baseline/upgrade migrations.")
print(" 3. Run scripts/release-migration-audit.py and PostgreSQL release checks.")
print(" 4. Record the reviewed heads with scripts/release-migration-audit.py --record-release <x.y.z>.")
print(" 5. Cut the release with scripts/push-release-tag.sh; audit is strict after the first baseline.")
if __name__ == "__main__":
raise SystemExit(main())