chore: consolidate platform split checks
This commit is contained in:
37
scripts/check-dependency-audits.sh
Normal file
37
scripts/check-dependency-audits.sh
Normal file
@@ -0,0 +1,37 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
PYTHON="${PYTHON:-$ROOT/.venv/bin/python}"
|
||||
NPM="${NPM:-/home/zemion/.nvm/versions/node/v22.22.3/bin/npm}"
|
||||
NODE_BIN="$(dirname "$NPM")"
|
||||
|
||||
if [[ ! -x "$PYTHON" ]]; then
|
||||
PYTHON=python
|
||||
fi
|
||||
if [[ ! -x "$NPM" ]]; then
|
||||
NPM=npm
|
||||
NODE_BIN="$(dirname "$(command -v "$NPM")")"
|
||||
fi
|
||||
|
||||
cd "$ROOT"
|
||||
CHECK_TESTCLIENT_DEPRECATIONS=auto bash "$ROOT/scripts/check-dependency-hygiene.sh"
|
||||
|
||||
if ! "$PYTHON" -m pip_audit --version >/dev/null 2>&1; then
|
||||
echo "pip-audit is not installed. Install dev requirements first:" >&2
|
||||
echo " $PYTHON -m pip install -r $ROOT/requirements-dev.txt" >&2
|
||||
exit 127
|
||||
fi
|
||||
|
||||
python_status=0
|
||||
npm_status=0
|
||||
|
||||
"$PYTHON" -m pip_audit --progress-spinner off || python_status=$?
|
||||
|
||||
cd "$ROOT/webui"
|
||||
PATH="$ROOT/webui/node_modules/.bin:$NODE_BIN:$PATH" "$NPM" audit --omit=dev || npm_status=$?
|
||||
|
||||
if [[ "$python_status" -ne 0 || "$npm_status" -ne 0 ]]; then
|
||||
echo "Dependency audit failed: python=$python_status npm=$npm_status" >&2
|
||||
exit 1
|
||||
fi
|
||||
158
scripts/check-dependency-hygiene.sh
Normal file
158
scripts/check-dependency-hygiene.sh
Normal file
@@ -0,0 +1,158 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
PYTHON="${PYTHON:-$ROOT/.venv/bin/python}"
|
||||
CHECK_TESTCLIENT_DEPRECATIONS="${CHECK_TESTCLIENT_DEPRECATIONS:-auto}"
|
||||
|
||||
if [[ ! -x "$PYTHON" ]]; then
|
||||
PYTHON=python
|
||||
fi
|
||||
|
||||
cd "$ROOT"
|
||||
|
||||
"$PYTHON" - <<'PY'
|
||||
import importlib.util
|
||||
import sys
|
||||
|
||||
if importlib.util.find_spec("pip") is None:
|
||||
print("Dependency hygiene failed: this Python environment cannot import pip.", file=sys.stderr)
|
||||
print("Recreate or repair the venv, then reinstall requirements:", file=sys.stderr)
|
||||
print(" python -m venv .venv", file=sys.stderr)
|
||||
print(" ./.venv/bin/python -m pip install --upgrade pip", file=sys.stderr)
|
||||
print(" ./.venv/bin/python -m pip install -r requirements-dev.txt", file=sys.stderr)
|
||||
raise SystemExit(127)
|
||||
PY
|
||||
|
||||
"$PYTHON" -m pip check
|
||||
|
||||
"$PYTHON" - <<'PY'
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.metadata
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
prefix = pathlib.Path(sys.prefix)
|
||||
legacy_patterns = (
|
||||
"__editable__.govoplan_module_multimailer-*.pth",
|
||||
"__editable___govoplan_module_multimailer_*_finder.py",
|
||||
"govoplan_module_multimailer-*.dist-info",
|
||||
)
|
||||
problems: list[pathlib.Path] = []
|
||||
|
||||
for site_packages in (prefix / "lib").glob("python*/site-packages"):
|
||||
for pattern in legacy_patterns:
|
||||
problems.extend(site_packages.glob(pattern))
|
||||
|
||||
for dist in importlib.metadata.distributions():
|
||||
if dist.metadata.get("Name", "").lower() == "govoplan-module-multimailer":
|
||||
problems.append(pathlib.Path(str(dist.locate_file(""))))
|
||||
|
||||
if problems:
|
||||
print("Dependency hygiene failed: stale legacy multimailer install metadata found.", file=sys.stderr)
|
||||
for path in sorted(set(problems)):
|
||||
print(f" {path}", file=sys.stderr)
|
||||
print("Remove the stale files or recreate the venv. This package pins obsolete dependencies.", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
|
||||
print("No stale legacy multimailer package metadata found.")
|
||||
PY
|
||||
|
||||
"$PYTHON" - <<'PY'
|
||||
from __future__ import annotations
|
||||
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
root = pathlib.Path.cwd()
|
||||
scan_roots = [
|
||||
root / "src",
|
||||
root / "tests",
|
||||
root.parent / "govoplan-access" / "src",
|
||||
root.parent / "govoplan-admin" / "src",
|
||||
root.parent / "govoplan-audit" / "src",
|
||||
root.parent / "govoplan-calendar" / "src",
|
||||
root.parent / "govoplan-campaign" / "src",
|
||||
root.parent / "govoplan-dashboard" / "src",
|
||||
root.parent / "govoplan-files" / "src",
|
||||
root.parent / "govoplan-identity" / "src",
|
||||
root.parent / "govoplan-mail" / "src",
|
||||
root.parent / "govoplan-ops" / "src",
|
||||
root.parent / "govoplan-organizations" / "src",
|
||||
root.parent / "govoplan-policy" / "src",
|
||||
root.parent / "govoplan-tenancy" / "src",
|
||||
]
|
||||
deprecated_tokens = {
|
||||
"HTTP_422_UNPROCESSABLE_ENTITY": "Use HTTP_422_UNPROCESSABLE_CONTENT; the numeric status remains 422.",
|
||||
}
|
||||
|
||||
hits: list[str] = []
|
||||
for scan_root in scan_roots:
|
||||
if not scan_root.exists():
|
||||
continue
|
||||
for path in scan_root.rglob("*.py"):
|
||||
try:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
except UnicodeDecodeError:
|
||||
continue
|
||||
for token, replacement in deprecated_tokens.items():
|
||||
if token in text:
|
||||
hits.append(f"{path}: deprecated {token}. {replacement}")
|
||||
|
||||
if hits:
|
||||
print("Dependency hygiene failed: deprecated framework constants are still used.", file=sys.stderr)
|
||||
print("\n".join(hits), file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
|
||||
print("Deprecated framework constant scan passed.")
|
||||
PY
|
||||
|
||||
RUN_TESTCLIENT_DEPRECATION_SMOKE=0
|
||||
case "$CHECK_TESTCLIENT_DEPRECATIONS" in
|
||||
0|false|False|no|No)
|
||||
echo "TestClient deprecation smoke skipped by CHECK_TESTCLIENT_DEPRECATIONS=$CHECK_TESTCLIENT_DEPRECATIONS"
|
||||
;;
|
||||
auto)
|
||||
if "$PYTHON" - <<'PY'
|
||||
import importlib.util
|
||||
raise SystemExit(0 if importlib.util.find_spec("fastapi") and importlib.util.find_spec("starlette") and importlib.util.find_spec("httpx2") else 1)
|
||||
PY
|
||||
then
|
||||
RUN_TESTCLIENT_DEPRECATION_SMOKE=1
|
||||
else
|
||||
echo "TestClient deprecation smoke skipped; install dev requirements to enable it."
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
RUN_TESTCLIENT_DEPRECATION_SMOKE=1
|
||||
;;
|
||||
esac
|
||||
|
||||
if [[ "$RUN_TESTCLIENT_DEPRECATION_SMOKE" -eq 1 ]]; then
|
||||
"$PYTHON" - <<'PY'
|
||||
import warnings
|
||||
|
||||
from starlette.exceptions import StarletteDeprecationWarning
|
||||
|
||||
warnings.simplefilter("error", StarletteDeprecationWarning)
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
app = FastAPI()
|
||||
|
||||
@app.get("/dependency-hygiene-ping")
|
||||
def ping() -> dict[str, bool]:
|
||||
return {"ok": True}
|
||||
|
||||
with TestClient(app) as client:
|
||||
response = client.get("/dependency-hygiene-ping")
|
||||
assert response.status_code == 200
|
||||
assert response.json() == {"ok": True}
|
||||
|
||||
print("TestClient deprecation smoke passed.")
|
||||
PY
|
||||
fi
|
||||
|
||||
echo "Dependency hygiene check passed."
|
||||
@@ -5,11 +5,19 @@ ROOT="/mnt/DATA/git/govoplan-core"
|
||||
NODE="/home/zemion/.nvm/versions/node/v22.22.3/bin"
|
||||
NPM="$NODE/npm"
|
||||
WEBUI_BIN="$ROOT/webui/node_modules/.bin"
|
||||
NPM_USERCONFIG="$(mktemp "${TMPDIR:-/tmp}/govoplan-npmrc.XXXXXXXX")"
|
||||
|
||||
trap 'rm -f "$NPM_USERCONFIG"' EXIT
|
||||
|
||||
export PATH="$WEBUI_BIN:$NODE:$PATH"
|
||||
export NPM_CONFIG_USERCONFIG="$NPM_USERCONFIG"
|
||||
export GOVOPLAN_NPM_USERCONFIG="$NPM_USERCONFIG"
|
||||
unset npm_config_tmp NPM_CONFIG_TMP
|
||||
|
||||
cd "$ROOT"
|
||||
|
||||
CHECK_TESTCLIENT_DEPRECATIONS=1 bash scripts/check-dependency-hygiene.sh
|
||||
|
||||
./.venv/bin/python - <<'PY'
|
||||
import ast
|
||||
import pathlib
|
||||
@@ -20,6 +28,8 @@ roots = [
|
||||
pathlib.Path("/mnt/DATA/git/govoplan-access/src"),
|
||||
pathlib.Path("/mnt/DATA/git/govoplan-admin/src"),
|
||||
pathlib.Path("/mnt/DATA/git/govoplan-tenancy/src"),
|
||||
pathlib.Path("/mnt/DATA/git/govoplan-organizations/src"),
|
||||
pathlib.Path("/mnt/DATA/git/govoplan-identity/src"),
|
||||
pathlib.Path("/mnt/DATA/git/govoplan-policy/src"),
|
||||
pathlib.Path("/mnt/DATA/git/govoplan-audit/src"),
|
||||
pathlib.Path("/mnt/DATA/git/govoplan-mail/src"),
|
||||
@@ -47,7 +57,7 @@ if errors:
|
||||
print(f"AST syntax check passed for {count} Python files")
|
||||
PY
|
||||
|
||||
./.venv/bin/python -c 'import govoplan_core.db.bootstrap; import govoplan_core.admin.service; import govoplan_files.backend.router; import govoplan_mail.backend.sending.imap; print("targeted backend imports passed")'
|
||||
./.venv/bin/python -c 'import govoplan_core.db.bootstrap; import govoplan_access.backend.admin.service; import govoplan_files.backend.router; import govoplan_mail.backend.sending.imap; print("targeted backend imports passed")'
|
||||
./.venv/bin/python scripts/check_dependency_boundaries.py
|
||||
./.venv/bin/python -m unittest tests.test_module_system
|
||||
./.venv/bin/python -m unittest discover -s /mnt/DATA/git/govoplan-mail/tests
|
||||
|
||||
23
scripts/check-module-matrix.sh
Normal file
23
scripts/check-module-matrix.sh
Normal file
@@ -0,0 +1,23 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
PYTHON="${PYTHON:-$ROOT/.venv/bin/python}"
|
||||
NPM="${NPM:-/home/zemion/.nvm/versions/node/v22.22.3/bin/npm}"
|
||||
NODE_BIN="$(dirname "$NPM")"
|
||||
|
||||
if [[ ! -x "$PYTHON" ]]; then
|
||||
PYTHON=python
|
||||
fi
|
||||
if [[ ! -x "$NPM" ]]; then
|
||||
NPM=npm
|
||||
NODE_BIN="$(dirname "$(command -v "$NPM")")"
|
||||
fi
|
||||
|
||||
cd "$ROOT"
|
||||
"$PYTHON" -m unittest tests.test_module_system tests.test_access_contracts
|
||||
"$PYTHON" scripts/check_dependency_boundaries.py
|
||||
|
||||
cd "$ROOT/webui"
|
||||
PATH="$ROOT/webui/node_modules/.bin:$NODE_BIN:$PATH" "$NPM" run test:module-capabilities
|
||||
PATH="$ROOT/webui/node_modules/.bin:$NODE_BIN:$PATH" "$NPM" run test:module-permutations
|
||||
@@ -11,8 +11,11 @@ REPOS = {
|
||||
"access": ROOT.parent / "govoplan-access" / "src" / "govoplan_access",
|
||||
"admin": ROOT.parent / "govoplan-admin" / "src" / "govoplan_admin",
|
||||
"tenancy": ROOT.parent / "govoplan-tenancy" / "src" / "govoplan_tenancy",
|
||||
"organizations": ROOT.parent / "govoplan-organizations" / "src" / "govoplan_organizations",
|
||||
"identity": ROOT.parent / "govoplan-identity" / "src" / "govoplan_identity",
|
||||
"policy": ROOT.parent / "govoplan-policy" / "src" / "govoplan_policy",
|
||||
"audit": ROOT.parent / "govoplan-audit" / "src" / "govoplan_audit",
|
||||
"dashboard": ROOT.parent / "govoplan-dashboard" / "src" / "govoplan_dashboard",
|
||||
"files": ROOT.parent / "govoplan-files" / "src" / "govoplan_files",
|
||||
"mail": ROOT.parent / "govoplan-mail" / "src" / "govoplan_mail",
|
||||
"campaign": ROOT.parent / "govoplan-campaign" / "src" / "govoplan_campaign",
|
||||
@@ -22,15 +25,18 @@ PREFIXES = {
|
||||
"access": "govoplan_access",
|
||||
"admin": "govoplan_admin",
|
||||
"tenancy": "govoplan_tenancy",
|
||||
"organizations": "govoplan_organizations",
|
||||
"identity": "govoplan_identity",
|
||||
"policy": "govoplan_policy",
|
||||
"audit": "govoplan_audit",
|
||||
"dashboard": "govoplan_dashboard",
|
||||
"files": "govoplan_files",
|
||||
"mail": "govoplan_mail",
|
||||
"campaign": "govoplan_campaign",
|
||||
}
|
||||
FEATURE_OWNERS = ("files", "mail", "campaign")
|
||||
PLATFORM_OWNERS = ("admin", "tenancy", "policy", "audit")
|
||||
PUBLIC_ACCESS_IMPORTS = ("govoplan_access.backend.auth.dependencies",)
|
||||
PLATFORM_OWNERS = ("admin", "tenancy", "organizations", "identity", "policy", "audit", "dashboard")
|
||||
PUBLIC_ACCESS_IMPORTS = ("govoplan_access.auth",)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
|
||||
@@ -38,6 +38,22 @@ CATALOG_MODULES = (
|
||||
description="Tenant registry, tenant settings, and tenant resolution platform module.",
|
||||
tags=("official", "platform-module"),
|
||||
),
|
||||
CatalogModule(
|
||||
module_id="organizations",
|
||||
repo="govoplan-organizations",
|
||||
python_package="govoplan-organizations",
|
||||
name="Organizations",
|
||||
description="Organization units, functions, and account-held function assignments.",
|
||||
tags=("official", "platform-module"),
|
||||
),
|
||||
CatalogModule(
|
||||
module_id="identity",
|
||||
repo="govoplan-identity",
|
||||
python_package="govoplan-identity",
|
||||
name="Identity",
|
||||
description="Canonical identities and links between identities and platform accounts.",
|
||||
tags=("official", "platform-module"),
|
||||
),
|
||||
CatalogModule(
|
||||
module_id="access",
|
||||
repo="govoplan-access",
|
||||
@@ -72,6 +88,15 @@ CATALOG_MODULES = (
|
||||
description="Audit-log storage and audit administration routes.",
|
||||
tags=("official", "platform-module"),
|
||||
),
|
||||
CatalogModule(
|
||||
module_id="dashboard",
|
||||
repo="govoplan-dashboard",
|
||||
python_package="govoplan-dashboard",
|
||||
name="Dashboard",
|
||||
description="Configurable user home assembled from module-provided dashboard widgets.",
|
||||
tags=("official", "platform-module"),
|
||||
webui_package="@govoplan/dashboard-webui",
|
||||
),
|
||||
CatalogModule(
|
||||
module_id="files",
|
||||
repo="govoplan-files",
|
||||
@@ -108,6 +133,24 @@ CATALOG_MODULES = (
|
||||
tags=("official", "service-module"),
|
||||
webui_package="@govoplan/calendar-webui",
|
||||
),
|
||||
CatalogModule(
|
||||
module_id="docs",
|
||||
repo="govoplan-docs",
|
||||
python_package="govoplan-docs",
|
||||
name="Docs",
|
||||
description="Configured-system documentation and evidence-aware help surfaces.",
|
||||
tags=("official", "platform-module"),
|
||||
webui_package="@govoplan/docs-webui",
|
||||
),
|
||||
CatalogModule(
|
||||
module_id="ops",
|
||||
repo="govoplan-ops",
|
||||
python_package="govoplan-ops",
|
||||
name="Ops",
|
||||
description="Runtime health, deployment profile, worker split, and sizing visibility.",
|
||||
tags=("official", "platform-module"),
|
||||
webui_package="@govoplan/ops-webui",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -12,11 +12,10 @@ import re
|
||||
import sys
|
||||
from typing import Any, Iterable
|
||||
|
||||
from gitea_common import GiteaClient, GiteaError, infer_target, load_dotenv, repo_path, repo_root, require_token
|
||||
from gitea_common import GiteaClient, GiteaError, infer_target, label_ids_by_name, load_dotenv, repo_path, repo_root, require_token
|
||||
|
||||
|
||||
DEFAULT_PRODUCT_BACKLOG = pathlib.Path("/mnt/DATA/Nextcloud/ADD ideas UG/Products/govoplan/backlog.md")
|
||||
DEFAULT_ACCESS_PLAN = pathlib.Path("/mnt/DATA/git/govoplan-core/docs/ACCESS_EXTRACTION_PLAN.md")
|
||||
DEFAULT_WEB_TODO = pathlib.Path("/mnt/DATA/git/govoplan-web/TODO.md")
|
||||
|
||||
REPO_ROOTS = {
|
||||
@@ -62,7 +61,7 @@ def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--env-file", type=pathlib.Path, help="dotenv file to read before using GITEA_* values")
|
||||
parser.add_argument("--product-backlog", type=pathlib.Path, default=DEFAULT_PRODUCT_BACKLOG)
|
||||
parser.add_argument("--access-plan", type=pathlib.Path, default=DEFAULT_ACCESS_PLAN)
|
||||
parser.add_argument("--access-plan", type=pathlib.Path, help="optional legacy access extraction plan to import")
|
||||
parser.add_argument("--web-todo", type=pathlib.Path, default=DEFAULT_WEB_TODO)
|
||||
parser.add_argument("--apply", action="store_true", help="create missing Gitea issues")
|
||||
args = parser.parse_args()
|
||||
@@ -128,7 +127,7 @@ def main() -> int:
|
||||
def build_candidates(args: argparse.Namespace) -> Iterable[Candidate]:
|
||||
if args.product_backlog.exists():
|
||||
yield from parse_product_backlog(args.product_backlog)
|
||||
if args.access_plan.exists():
|
||||
if args.access_plan and args.access_plan.exists():
|
||||
yield from parse_access_plan(args.access_plan)
|
||||
if args.web_todo.exists():
|
||||
yield from parse_simple_todo(args.web_todo)
|
||||
@@ -539,8 +538,7 @@ def load_repo_states(token: str, repo_keys: list[str]) -> dict[str, RepoState]:
|
||||
root = repo_root(REPO_ROOTS[repo_key])
|
||||
target = infer_target(root)
|
||||
client = GiteaClient(target, token)
|
||||
labels_payload = client.paginate(repo_path(target.owner, target.repo, "/labels"))
|
||||
label_ids = {str(label["name"]): int(label["id"]) for label in labels_payload if "name" in label and "id" in label}
|
||||
label_ids = label_ids_by_name(client, target.owner, target.repo)
|
||||
issues = client.paginate(repo_path(target.owner, target.repo, "/issues"), query={"state": "all"}, limit=100)
|
||||
fingerprints: set[str] = set()
|
||||
titles: set[str] = set()
|
||||
|
||||
@@ -14,7 +14,7 @@ import subprocess
|
||||
import sys
|
||||
from typing import Any, Iterable
|
||||
|
||||
from gitea_common import GiteaClient, GiteaError, infer_target, load_dotenv, repo_path, require_token
|
||||
from gitea_common import GiteaClient, GiteaError, infer_target, label_ids_by_name, load_dotenv, repo_path, require_token
|
||||
|
||||
|
||||
GIT_ROOT = pathlib.Path("/mnt/DATA/git")
|
||||
@@ -246,9 +246,7 @@ def is_excluded_repo_file(path: pathlib.Path) -> bool:
|
||||
return True
|
||||
if text.endswith("/docs/GITEA_ISSUES.md"):
|
||||
return True
|
||||
if text.endswith("/docs/GOVOPLAN_MODULE_ROADMAP.md"):
|
||||
return True
|
||||
if text.endswith("/docs/ACCESS_EXTRACTION_PLAN.md"):
|
||||
if text.endswith("/docs/GOVOPLAN_MASTER_ROADMAP.md"):
|
||||
return True
|
||||
if text.endswith("/govoplan-web/TODO.md"):
|
||||
return True
|
||||
@@ -537,8 +535,7 @@ def grouped_counts(candidates: list[Candidate]) -> dict[str, int]:
|
||||
def load_repo_state(repo: RepoInfo, token: str, *, apply: bool) -> RepoState:
|
||||
target = infer_target(repo.root)
|
||||
client = GiteaClient(target, token)
|
||||
labels = client.paginate(repo_path(target.owner, target.repo, "/labels"), limit=50)
|
||||
label_ids = {str(label["name"]): int(label["id"]) for label in labels}
|
||||
label_ids = label_ids_by_name(client, target.owner, target.repo)
|
||||
if apply:
|
||||
for name, color, description in GENERIC_LABELS:
|
||||
if name in label_ids:
|
||||
|
||||
250
scripts/gitea-migrate-org-labels.py
Normal file
250
scripts/gitea-migrate-org-labels.py
Normal file
@@ -0,0 +1,250 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Move issue labels from repository-local labels to organization labels."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
from gitea_common import (
|
||||
GiteaClient,
|
||||
GiteaError,
|
||||
infer_target,
|
||||
load_dotenv,
|
||||
org_path,
|
||||
repo_path,
|
||||
repo_root,
|
||||
require_token,
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--root", type=pathlib.Path, default=pathlib.Path.cwd(), help="repository root used for Gitea URL inference")
|
||||
parser.add_argument("--remote", default="origin", help="git remote to use for target inference")
|
||||
parser.add_argument("--env-file", type=pathlib.Path, default=pathlib.Path("/home/zemion/.config/gitea/gitea.env"))
|
||||
parser.add_argument("--org", help="organization owner; defaults to inferred owner")
|
||||
parser.add_argument("--repo", action="append", default=[], help="repository name to process; may be repeated")
|
||||
parser.add_argument("--repo-regex", default=".*", help="only process repositories whose name matches this regex")
|
||||
parser.add_argument(
|
||||
"--issue-mode",
|
||||
choices=("replace", "delete-add"),
|
||||
default="replace",
|
||||
help="replace labels in one request, or delete local ids before adding org ids",
|
||||
)
|
||||
parser.add_argument("--delete-repo-labels", action="store_true", help="delete matching repository-local labels after issue and PR migration")
|
||||
parser.add_argument("--apply", action="store_true", help="apply changes; omit for dry-run")
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
root = repo_root(args.root)
|
||||
load_dotenv(args.env_file or root / ".env")
|
||||
target = infer_target(root, args.remote)
|
||||
owner = args.org or target.owner
|
||||
client = GiteaClient(target, require_token() if args.apply else os.environ.get("GITEA_TOKEN"))
|
||||
if client.token is None:
|
||||
raise GiteaError("GITEA_TOKEN is required for dry-run and apply because migration inspects live issue labels.")
|
||||
|
||||
repo_filter = re.compile(args.repo_regex)
|
||||
org_labels = _labels_by_name(client.paginate(org_path(owner, "/labels"), limit=100))
|
||||
if not org_labels:
|
||||
raise GiteaError(f"{owner} has no organization labels")
|
||||
|
||||
repos = _selected_repositories(client, owner, args.repo, repo_filter)
|
||||
totals = Totals()
|
||||
print(f"Organization: {owner}")
|
||||
print(f"Repositories: {len(repos)}")
|
||||
print(f"Mode: {'apply' if args.apply else 'dry-run'}")
|
||||
print(f"Delete repository labels: {args.delete_repo_labels}")
|
||||
|
||||
for index, repo in enumerate(repos, start=1):
|
||||
repo_result = migrate_repository(
|
||||
client,
|
||||
owner=owner,
|
||||
repo=repo,
|
||||
org_labels=org_labels,
|
||||
issue_mode=args.issue_mode,
|
||||
apply=args.apply,
|
||||
delete_repo_labels=args.delete_repo_labels,
|
||||
)
|
||||
totals.add(repo_result)
|
||||
if repo_result.has_work:
|
||||
print(
|
||||
f"[{index}/{len(repos)}] {repo}: "
|
||||
f"issue-labels={repo_result.issue_label_migrations}, "
|
||||
f"delete-labels={repo_result.repo_label_deletions}, "
|
||||
f"errors={len(repo_result.errors)}"
|
||||
)
|
||||
for error in repo_result.errors:
|
||||
print(f" error: {error}")
|
||||
else:
|
||||
print(f"[{index}/{len(repos)}] {repo}: no matching local labels")
|
||||
|
||||
print("Summary:")
|
||||
print(f" repositories processed: {totals.repositories}")
|
||||
print(f" repositories with matching local labels: {totals.repositories_with_work}")
|
||||
print(f" issue/PR local labels migrated: {totals.issue_label_migrations}")
|
||||
print(f" repository labels deleted: {totals.repo_label_deletions}")
|
||||
print(f" errors: {totals.errors}")
|
||||
if totals.errors:
|
||||
return 1
|
||||
return 0
|
||||
except GiteaError as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
class RepoResult:
|
||||
def __init__(self) -> None:
|
||||
self.issue_label_migrations = 0
|
||||
self.repo_label_deletions = 0
|
||||
self.errors: list[str] = []
|
||||
self.matched_local_labels = 0
|
||||
|
||||
@property
|
||||
def has_work(self) -> bool:
|
||||
return bool(self.matched_local_labels or self.issue_label_migrations or self.repo_label_deletions or self.errors)
|
||||
|
||||
|
||||
class Totals:
|
||||
def __init__(self) -> None:
|
||||
self.repositories = 0
|
||||
self.repositories_with_work = 0
|
||||
self.issue_label_migrations = 0
|
||||
self.repo_label_deletions = 0
|
||||
self.errors = 0
|
||||
|
||||
def add(self, result: RepoResult) -> None:
|
||||
self.repositories += 1
|
||||
if result.has_work:
|
||||
self.repositories_with_work += 1
|
||||
self.issue_label_migrations += result.issue_label_migrations
|
||||
self.repo_label_deletions += result.repo_label_deletions
|
||||
self.errors += len(result.errors)
|
||||
|
||||
|
||||
def migrate_repository(
|
||||
client: GiteaClient,
|
||||
*,
|
||||
owner: str,
|
||||
repo: str,
|
||||
org_labels: dict[str, dict[str, Any]],
|
||||
issue_mode: str,
|
||||
apply: bool,
|
||||
delete_repo_labels: bool,
|
||||
) -> RepoResult:
|
||||
result = RepoResult()
|
||||
repo_labels = client.paginate(repo_path(owner, repo, "/labels"), limit=100)
|
||||
local_labels = {
|
||||
str(label.get("name") or ""): label
|
||||
for label in repo_labels
|
||||
if str(label.get("name") or "") in org_labels
|
||||
}
|
||||
result.matched_local_labels = len(local_labels)
|
||||
if not local_labels:
|
||||
return result
|
||||
|
||||
local_by_id = {_label_id(label): label for label in local_labels.values() if _label_id(label) is not None}
|
||||
org_by_name = {name: _label_id(label) for name, label in org_labels.items()}
|
||||
|
||||
for issue_type in ("issues", "pulls"):
|
||||
issues = client.paginate(
|
||||
repo_path(owner, repo, "/issues"),
|
||||
query={"state": "all", "type": issue_type},
|
||||
limit=100,
|
||||
)
|
||||
for issue in issues:
|
||||
issue_number = int(issue.get("number") or issue.get("index"))
|
||||
issue_label_ids = [_label_id(label) for label in issue.get("labels") or []]
|
||||
issue_label_ids = [label_id for label_id in issue_label_ids if label_id is not None]
|
||||
final_label_ids = list(issue_label_ids)
|
||||
changed = False
|
||||
migrated_count = 0
|
||||
local_ids_to_remove: list[int] = []
|
||||
org_ids_to_add: list[int] = []
|
||||
for label in issue.get("labels") or []:
|
||||
local_id = _label_id(label)
|
||||
if local_id is None or local_id not in local_by_id:
|
||||
continue
|
||||
name = str(label.get("name") or "")
|
||||
org_id = org_by_name.get(name)
|
||||
if org_id is None:
|
||||
continue
|
||||
local_ids_to_remove.append(local_id)
|
||||
if org_id not in issue_label_ids and org_id not in org_ids_to_add:
|
||||
org_ids_to_add.append(org_id)
|
||||
final_label_ids = [label_id for label_id in final_label_ids if label_id != local_id]
|
||||
if org_id not in final_label_ids:
|
||||
final_label_ids.append(org_id)
|
||||
changed = True
|
||||
migrated_count += 1
|
||||
if changed:
|
||||
if apply:
|
||||
if issue_mode == "delete-add":
|
||||
for local_id in local_ids_to_remove:
|
||||
_remove_issue_label(client, owner, repo, issue_number, local_id)
|
||||
if org_ids_to_add:
|
||||
client.request_json(
|
||||
"POST",
|
||||
repo_path(owner, repo, f"/issues/{issue_number}/labels"),
|
||||
body={"labels": org_ids_to_add},
|
||||
)
|
||||
else:
|
||||
client.request_json(
|
||||
"PUT",
|
||||
repo_path(owner, repo, f"/issues/{issue_number}/labels"),
|
||||
body={"labels": final_label_ids},
|
||||
)
|
||||
result.issue_label_migrations += migrated_count
|
||||
|
||||
if delete_repo_labels:
|
||||
for name, label in sorted(local_labels.items()):
|
||||
label_id = _label_id(label)
|
||||
if label_id is None:
|
||||
result.errors.append(f"{name} has no label id")
|
||||
continue
|
||||
if apply:
|
||||
try:
|
||||
client.request_json("DELETE", repo_path(owner, repo, f"/labels/{label_id}"))
|
||||
except GiteaError as exc:
|
||||
result.errors.append(f"delete repository label {name}: {exc}")
|
||||
continue
|
||||
result.repo_label_deletions += 1
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def _selected_repositories(client: GiteaClient, owner: str, explicit: list[str], repo_filter: re.Pattern[str]) -> list[str]:
|
||||
if explicit:
|
||||
return sorted(set(explicit))
|
||||
repos = client.paginate(org_path(owner, "/repos"), query={"type": "all"}, limit=100)
|
||||
names = sorted(str(repo.get("name") or "") for repo in repos if repo.get("name"))
|
||||
return [name for name in names if repo_filter.search(name)]
|
||||
|
||||
|
||||
def _labels_by_name(labels: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
|
||||
return {str(label.get("name")): label for label in labels if label.get("name") and _label_id(label) is not None}
|
||||
|
||||
|
||||
def _label_id(label: dict[str, Any]) -> int | None:
|
||||
value = label.get("id") or label.get("index")
|
||||
if value is None:
|
||||
return None
|
||||
return int(value)
|
||||
|
||||
|
||||
def _remove_issue_label(client: GiteaClient, owner: str, repo: str, issue_number: int, label_id: int) -> None:
|
||||
try:
|
||||
client.request_json("DELETE", repo_path(owner, repo, f"/issues/{issue_number}/labels/{label_id}"))
|
||||
except GiteaError as exc:
|
||||
if "HTTP 404" in str(exc):
|
||||
return
|
||||
raise
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -15,6 +15,7 @@ from gitea_common import (
|
||||
infer_target,
|
||||
load_dotenv,
|
||||
load_json,
|
||||
org_path,
|
||||
repo_path,
|
||||
repo_root,
|
||||
require_token,
|
||||
@@ -30,6 +31,13 @@ def main() -> int:
|
||||
parser.add_argument("--remote", default="origin", help="git remote to use for target inference")
|
||||
parser.add_argument("--labels-file", type=pathlib.Path, default=DEFAULT_LABELS_FILE)
|
||||
parser.add_argument("--env-file", type=pathlib.Path, help="dotenv file to read before using GITEA_* values")
|
||||
parser.add_argument(
|
||||
"--scope",
|
||||
choices=("repository", "organization"),
|
||||
default="repository",
|
||||
help="sync repository labels or organization labels",
|
||||
)
|
||||
parser.add_argument("--org", help="organization name for --scope organization; defaults to inferred owner")
|
||||
parser.add_argument("--apply", action="store_true", help="create or update labels in Gitea")
|
||||
args = parser.parse_args()
|
||||
|
||||
@@ -39,8 +47,12 @@ def main() -> int:
|
||||
target = infer_target(root, args.remote)
|
||||
labels = _load_labels(args.labels_file)
|
||||
token = require_token() if args.apply else os.environ.get("GITEA_TOKEN")
|
||||
org_name = args.org or target.owner
|
||||
|
||||
print(f"Target: {target.display}")
|
||||
if args.scope == "organization":
|
||||
print(f"Target: {target.base_url.rstrip('/')}/{org_name} organization labels")
|
||||
else:
|
||||
print(f"Target: {target.display}")
|
||||
print(f"Labels file: {args.labels_file}")
|
||||
|
||||
if not args.apply and not token:
|
||||
@@ -50,7 +62,10 @@ def main() -> int:
|
||||
return 0
|
||||
|
||||
client = GiteaClient(target, token)
|
||||
existing = _labels_by_name(client, target.owner, target.repo)
|
||||
if args.scope == "organization":
|
||||
existing = _org_labels_by_name(client, org_name)
|
||||
else:
|
||||
existing = _repo_labels_by_name(client, target.owner, target.repo)
|
||||
creates: list[dict[str, Any]] = []
|
||||
updates: list[tuple[dict[str, Any], dict[str, Any]]] = []
|
||||
|
||||
@@ -71,18 +86,17 @@ def main() -> int:
|
||||
for label in creates:
|
||||
print(f"{'create' if args.apply else 'would create'} {label['name']}")
|
||||
if args.apply:
|
||||
client.request_json(
|
||||
"POST",
|
||||
repo_path(target.owner, target.repo, "/labels"),
|
||||
body=label,
|
||||
)
|
||||
client.request_json("POST", _create_path(args.scope, org_name, target.owner, target.repo), body=label)
|
||||
|
||||
for current, patch in updates:
|
||||
print(f"{'update' if args.apply else 'would update'} {current['name']}: {', '.join(sorted(patch))}")
|
||||
if args.apply:
|
||||
label_id = current.get("id") or current.get("index")
|
||||
if label_id is None:
|
||||
raise GiteaError(f"{current['name']} has no label id in API response")
|
||||
client.request_json(
|
||||
"PATCH",
|
||||
repo_path(target.owner, target.repo, f"/labels/{current['id']}"),
|
||||
_update_path(args.scope, org_name, target.owner, target.repo, int(label_id)),
|
||||
body=patch,
|
||||
)
|
||||
|
||||
@@ -119,11 +133,28 @@ def _load_labels(path: pathlib.Path) -> list[dict[str, Any]]:
|
||||
return labels
|
||||
|
||||
|
||||
def _labels_by_name(client: GiteaClient, owner: str, repo: str) -> dict[str, dict[str, Any]]:
|
||||
def _repo_labels_by_name(client: GiteaClient, owner: str, repo: str) -> dict[str, dict[str, Any]]:
|
||||
labels = client.paginate(repo_path(owner, repo, "/labels"))
|
||||
return {str(label.get("name")): label for label in labels}
|
||||
|
||||
|
||||
def _org_labels_by_name(client: GiteaClient, owner: str) -> dict[str, dict[str, Any]]:
|
||||
labels = client.paginate(org_path(owner, "/labels"))
|
||||
return {str(label.get("name")): label for label in labels}
|
||||
|
||||
|
||||
def _create_path(scope: str, org: str, owner: str, repo: str) -> str:
|
||||
if scope == "organization":
|
||||
return org_path(org, "/labels")
|
||||
return repo_path(owner, repo, "/labels")
|
||||
|
||||
|
||||
def _update_path(scope: str, org: str, owner: str, repo: str, label_id: int) -> str:
|
||||
if scope == "organization":
|
||||
return org_path(org, f"/labels/{label_id}")
|
||||
return repo_path(owner, repo, f"/labels/{label_id}")
|
||||
|
||||
|
||||
def _diff_label(current: dict[str, Any], desired: dict[str, Any]) -> dict[str, Any]:
|
||||
patch: dict[str, Any] = {}
|
||||
if _normalize_color(current.get("color")) != desired["color"]:
|
||||
|
||||
@@ -73,6 +73,7 @@ def main() -> int:
|
||||
parser.add_argument("--transport", choices=("git", "api"), default="git", help="sync through the wiki git repository or the Gitea REST API")
|
||||
parser.add_argument("--wiki-cache-dir", type=pathlib.Path, default=pathlib.Path("/tmp/codex-gitea-wiki-sync"), help="local cache for wiki git checkouts")
|
||||
parser.add_argument("--commit-message", default="Sync wiki from project files", help="commit message used by git transport")
|
||||
parser.add_argument("--prune-managed", action="store_true", help="delete managed wiki pages whose source files are no longer discovered")
|
||||
parser.add_argument("--apply", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
@@ -116,9 +117,17 @@ def main() -> int:
|
||||
overwrite_unmanaged=args.overwrite_unmanaged,
|
||||
commit_message=args.commit_message,
|
||||
write_index=not bool(args.page),
|
||||
prune_managed=args.prune_managed and not bool(args.page),
|
||||
)
|
||||
else:
|
||||
sync_repo_wiki(repo, repo_sources, token, overwrite_unmanaged=args.overwrite_unmanaged, write_index=not bool(args.page))
|
||||
sync_repo_wiki(
|
||||
repo,
|
||||
repo_sources,
|
||||
token,
|
||||
overwrite_unmanaged=args.overwrite_unmanaged,
|
||||
write_index=not bool(args.page),
|
||||
prune_managed=args.prune_managed and not bool(args.page),
|
||||
)
|
||||
except GiteaError as exc:
|
||||
failures += 1
|
||||
print(f"error syncing wiki for {repo_key}: {exc}", file=sys.stderr)
|
||||
@@ -267,7 +276,15 @@ def preview_sources(sources: list[WikiSource]) -> None:
|
||||
print(f" ... {len(repo_sources) - 80} more")
|
||||
|
||||
|
||||
def sync_repo_wiki(repo: RepoInfo, sources: list[WikiSource], token: str, *, overwrite_unmanaged: bool, write_index: bool = True) -> None:
|
||||
def sync_repo_wiki(
|
||||
repo: RepoInfo,
|
||||
sources: list[WikiSource],
|
||||
token: str,
|
||||
*,
|
||||
overwrite_unmanaged: bool,
|
||||
write_index: bool = True,
|
||||
prune_managed: bool = False,
|
||||
) -> None:
|
||||
target = infer_target(repo.root)
|
||||
client = GiteaClient(target, token)
|
||||
existing_pages = list_existing_wiki_pages(client, target.owner, target.repo)
|
||||
@@ -311,6 +328,8 @@ def sync_repo_wiki(repo: RepoInfo, sources: list[WikiSource], token: str, *, ove
|
||||
)
|
||||
else:
|
||||
print(f"skipped {target.owner}/{target.repo} wiki:Codex-Project-Index during page-limited sync")
|
||||
if prune_managed and write_index:
|
||||
prune_managed_wiki_pages_api(client, target.owner, target.repo, existing_page_names, sources)
|
||||
|
||||
|
||||
def sync_repo_wiki_git(
|
||||
@@ -321,6 +340,7 @@ def sync_repo_wiki_git(
|
||||
overwrite_unmanaged: bool,
|
||||
commit_message: str,
|
||||
write_index: bool = True,
|
||||
prune_managed: bool = False,
|
||||
) -> None:
|
||||
target = infer_target(repo.root)
|
||||
wiki_root = prepare_wiki_checkout(repo, cache_dir=cache_dir)
|
||||
@@ -353,6 +373,8 @@ def sync_repo_wiki_git(
|
||||
)
|
||||
else:
|
||||
print(f"skipped {target.owner}/{target.repo} wiki:Codex-Project-Index during page-limited sync")
|
||||
if prune_managed and write_index:
|
||||
prune_managed_wiki_pages_git(wiki_root, sources)
|
||||
if not git_has_changes(wiki_root):
|
||||
print(f"unchanged {target.owner}/{target.repo} wiki repository")
|
||||
return
|
||||
@@ -426,6 +448,38 @@ def write_wiki_page_file(wiki_root: pathlib.Path, title: str, content: str, *, o
|
||||
return "updated"
|
||||
|
||||
|
||||
def prune_managed_wiki_pages_git(wiki_root: pathlib.Path, sources: list[WikiSource]) -> None:
|
||||
desired = {f"{wiki_file_stem(source.page_title)}.md" for source in sources}
|
||||
desired.add(f"{wiki_file_stem('Codex-Project-Index')}.md")
|
||||
for path in sorted(wiki_root.glob("*.md")):
|
||||
if path.name in desired:
|
||||
continue
|
||||
current = read_text(path)
|
||||
if MANAGED_MARKER not in current:
|
||||
continue
|
||||
path.unlink()
|
||||
print(f"deleted stale managed wiki page {path.name}")
|
||||
|
||||
|
||||
def prune_managed_wiki_pages_api(
|
||||
client: GiteaClient,
|
||||
owner: str,
|
||||
repo: str,
|
||||
existing_page_names: dict[str, str],
|
||||
sources: list[WikiSource],
|
||||
) -> None:
|
||||
desired = {source.page_title for source in sources}
|
||||
desired.add("Codex-Project-Index")
|
||||
for title, page_name in sorted(existing_page_names.items()):
|
||||
if title in desired:
|
||||
continue
|
||||
current = client.request_json("GET", repo_path(owner, repo, f"/wiki/page/{quote_wiki_page_name(page_name)}"))
|
||||
if MANAGED_MARKER not in decode_wiki_content(current):
|
||||
continue
|
||||
client.request_json("DELETE", repo_path(owner, repo, f"/wiki/page/{quote_wiki_page_name(page_name)}"))
|
||||
print(f"deleted stale managed wiki page {owner}/{repo}:{title}")
|
||||
|
||||
|
||||
def wiki_file_stem(title: str) -> str:
|
||||
return re.sub(r"[/\\]+", "-", title).strip() or "Home"
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@ import subprocess
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
from gitea_common import GiteaClient, GiteaError, infer_target, load_dotenv, repo_path, repo_root, require_token
|
||||
from gitea_common import GiteaClient, GiteaError, infer_target, label_ids_by_name, load_dotenv, repo_path, repo_root, require_token
|
||||
|
||||
|
||||
MARKER_RE = re.compile(
|
||||
@@ -347,8 +347,7 @@ def truncate_title(title: str, limit: int = 180) -> str:
|
||||
|
||||
|
||||
def load_label_ids(client: GiteaClient, owner: str, repo: str) -> dict[str, int]:
|
||||
labels = client.paginate(repo_path(owner, repo, "/labels"))
|
||||
return {str(label["name"]): int(label["id"]) for label in labels if "name" in label and "id" in label}
|
||||
return label_ids_by_name(client, owner, repo)
|
||||
|
||||
|
||||
def load_existing_fingerprints(client: GiteaClient, owner: str, repo: str) -> set[str]:
|
||||
|
||||
@@ -197,6 +197,47 @@ def repo_path(owner: str, repo: str, suffix: str) -> str:
|
||||
return f"/repos/{quote_path(owner)}/{quote_path(repo)}{suffix}"
|
||||
|
||||
|
||||
def org_path(owner: str, suffix: str) -> str:
|
||||
return f"/orgs/{quote_path(owner)}{suffix}"
|
||||
|
||||
|
||||
def labels_by_name(client: GiteaClient, owner: str, repo: str | None = None) -> dict[str, dict[str, Any]]:
|
||||
"""Return org labels plus optional repository labels, keyed by label name.
|
||||
|
||||
Gitea stores organization labels separately from repository labels. Issue
|
||||
APIs can use label ids from both scopes on supported instances, so issue
|
||||
creation helpers should resolve both. Repository labels intentionally win
|
||||
when a repo carries a local label with the same name.
|
||||
"""
|
||||
|
||||
labels: dict[str, dict[str, Any]] = {}
|
||||
try:
|
||||
for label in client.paginate(org_path(owner, "/labels")):
|
||||
name = str(label.get("name") or "")
|
||||
if name:
|
||||
labels[name] = label
|
||||
except GiteaError:
|
||||
# User-owned repositories or older instances may not expose org labels.
|
||||
pass
|
||||
|
||||
if repo:
|
||||
for label in client.paginate(repo_path(owner, repo, "/labels")):
|
||||
name = str(label.get("name") or "")
|
||||
if name:
|
||||
labels[name] = label
|
||||
return labels
|
||||
|
||||
|
||||
def label_ids_by_name(client: GiteaClient, owner: str, repo: str | None = None) -> dict[str, int]:
|
||||
ids: dict[str, int] = {}
|
||||
for name, label in labels_by_name(client, owner, repo).items():
|
||||
label_id = label.get("id") or label.get("index")
|
||||
if label_id is None:
|
||||
continue
|
||||
ids[name] = int(label_id)
|
||||
return ids
|
||||
|
||||
|
||||
def _git_remote(root: pathlib.Path, remote_name: str) -> str:
|
||||
result = subprocess.run(
|
||||
["git", "-C", str(root), "remote", "get-url", remote_name],
|
||||
|
||||
@@ -16,6 +16,7 @@ FRONTEND_FORCE_RELOAD="${GOVOPLAN_FRONTEND_FORCE_RELOAD:-1}"
|
||||
FRONTEND_CLEAR_VITE_CACHE="${GOVOPLAN_FRONTEND_CLEAR_VITE_CACHE:-1}"
|
||||
FRONTEND_USE_POLLING="${GOVOPLAN_FRONTEND_USE_POLLING:-1}"
|
||||
FRONTEND_POLLING_INTERVAL="${GOVOPLAN_FRONTEND_POLLING_INTERVAL:-500}"
|
||||
DEV_DATABASE_BACKEND="${GOVOPLAN_DEV_DATABASE_BACKEND:-postgres}"
|
||||
|
||||
LOG_DIR="$ROOT/runtime/dev-launcher"
|
||||
BACKEND_LOG="$LOG_DIR/backend.log"
|
||||
@@ -31,6 +32,20 @@ fail() {
|
||||
exit 1
|
||||
}
|
||||
|
||||
case "$DEV_DATABASE_BACKEND" in
|
||||
postgres)
|
||||
export DATABASE_URL="${DATABASE_URL:-postgresql+psycopg://govoplan_dev@127.0.0.1:5432/govoplan_dev}"
|
||||
export GOVOPLAN_DATABASE_URL_PGTOOLS="${GOVOPLAN_DATABASE_URL_PGTOOLS:-postgresql://govoplan_dev@127.0.0.1:5432/govoplan_dev}"
|
||||
;;
|
||||
sqlite)
|
||||
export DATABASE_URL="${DATABASE_URL:-sqlite:///$ROOT/runtime/multimailer-dev.db}"
|
||||
;;
|
||||
*)
|
||||
fail "Unsupported GOVOPLAN_DEV_DATABASE_BACKEND=$DEV_DATABASE_BACKEND. Use postgres or sqlite."
|
||||
;;
|
||||
esac
|
||||
export DEV_BOOTSTRAP_ENABLED="${DEV_BOOTSTRAP_ENABLED:-true}"
|
||||
|
||||
port_is_free() {
|
||||
"$PYTHON" - "$1" "$2" <<'PY'
|
||||
import socket
|
||||
@@ -139,6 +154,7 @@ GovOPlaN is running.
|
||||
Web UI: $FRONTEND_URL
|
||||
API: $BACKEND_URL/api/v1
|
||||
Health: $BACKEND_URL/health
|
||||
DB: $DATABASE_URL
|
||||
|
||||
Logs:
|
||||
Backend: $BACKEND_LOG
|
||||
|
||||
238
scripts/launch-production-like-dev.sh
Normal file
238
scripts/launch-production-like-dev.sh
Normal file
@@ -0,0 +1,238 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="${GOVOPLAN_CORE_ROOT:-/mnt/DATA/git/govoplan-core}"
|
||||
PROFILE_ROOT="$ROOT/dev/production-like"
|
||||
PROFILE_ENV="${GOVOPLAN_PRODUCTION_LIKE_ENV:-$PROFILE_ROOT/.env}"
|
||||
if [ ! -f "$PROFILE_ENV" ]; then
|
||||
PROFILE_ENV="$PROFILE_ROOT/.env.example"
|
||||
fi
|
||||
|
||||
WEBUI_ROOT="$ROOT/webui"
|
||||
PYTHON="${PYTHON:-$ROOT/.venv/bin/python}"
|
||||
NODE_BIN="${NODE_BIN:-/home/zemion/.nvm/versions/node/v22.22.3/bin}"
|
||||
NPM="${NPM:-$NODE_BIN/npm}"
|
||||
DOCKER="${DOCKER:-docker}"
|
||||
|
||||
BACKEND_HOST="${GOVOPLAN_BACKEND_HOST:-127.0.0.1}"
|
||||
BACKEND_PORT="${GOVOPLAN_BACKEND_PORT:-8000}"
|
||||
FRONTEND_HOST="${GOVOPLAN_FRONTEND_HOST:-127.0.0.1}"
|
||||
FRONTEND_PORT="${GOVOPLAN_FRONTEND_PORT:-5173}"
|
||||
OPEN_BROWSER="${OPEN_BROWSER:-1}"
|
||||
STOP_DEPENDENCIES_ON_EXIT="${GOVOPLAN_STOP_PROFILE_DEPENDENCIES_ON_EXIT:-0}"
|
||||
|
||||
LOG_DIR="$ROOT/runtime/production-like/logs"
|
||||
BACKEND_LOG="$LOG_DIR/backend.log"
|
||||
WORKER_LOG="$LOG_DIR/worker.log"
|
||||
FRONTEND_LOG="$LOG_DIR/frontend.log"
|
||||
BACKEND_URL="http://$BACKEND_HOST:$BACKEND_PORT"
|
||||
FRONTEND_URL="http://$FRONTEND_HOST:$FRONTEND_PORT"
|
||||
|
||||
backend_pid=""
|
||||
worker_pid=""
|
||||
frontend_pid=""
|
||||
|
||||
fail() {
|
||||
printf 'launch-production-like-dev: %s\n' "$*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
[ -f "$PROFILE_ENV" ] || fail "profile env file not found: $PROFILE_ENV"
|
||||
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
. "$PROFILE_ENV"
|
||||
set +a
|
||||
|
||||
export APP_ENV="${APP_ENV:-staging}"
|
||||
export ENABLED_MODULES="${ENABLED_MODULES:-tenancy,access,admin,policy,audit,campaigns,files,mail,calendar,docs,ops}"
|
||||
export DATABASE_URL="${DATABASE_URL:-${GOVOPLAN_PRODUCTION_LIKE_DATABASE_URL:-postgresql+psycopg://govoplan:govoplan-dev@127.0.0.1:55433/govoplan}}"
|
||||
export GOVOPLAN_DATABASE_URL_PGTOOLS="${GOVOPLAN_DATABASE_URL_PGTOOLS:-${GOVOPLAN_PRODUCTION_LIKE_DATABASE_URL_PGTOOLS:-postgresql://govoplan:govoplan-dev@127.0.0.1:55433/govoplan}}"
|
||||
export REDIS_URL="${REDIS_URL:-${GOVOPLAN_PRODUCTION_LIKE_REDIS_URL:-redis://127.0.0.1:56379/0}}"
|
||||
export CELERY_ENABLED="${CELERY_ENABLED:-true}"
|
||||
export CELERY_QUEUES="${CELERY_QUEUES:-send_email,append_sent,default}"
|
||||
export FILE_STORAGE_BACKEND="${FILE_STORAGE_BACKEND:-local}"
|
||||
export FILE_STORAGE_LOCAL_ROOT="${FILE_STORAGE_LOCAL_ROOT:-$ROOT/runtime/production-like/files}"
|
||||
export DEV_AUTO_MIGRATE_ENABLED="${DEV_AUTO_MIGRATE_ENABLED:-false}"
|
||||
export DEV_BOOTSTRAP_ENABLED="${DEV_BOOTSTRAP_ENABLED:-true}"
|
||||
export CORS_ORIGINS="${CORS_ORIGINS:-http://127.0.0.1:$FRONTEND_PORT,http://localhost:$FRONTEND_PORT}"
|
||||
|
||||
if [ -z "${MASTER_KEY_B64:-}" ]; then
|
||||
MASTER_KEY_B64="$("$PYTHON" - <<'PY'
|
||||
from cryptography.fernet import Fernet
|
||||
print(Fernet.generate_key().decode())
|
||||
PY
|
||||
)"
|
||||
export MASTER_KEY_B64
|
||||
fi
|
||||
|
||||
compose() {
|
||||
"$DOCKER" compose --env-file "$PROFILE_ENV" -f "$PROFILE_ROOT/docker-compose.yml" "$@"
|
||||
}
|
||||
|
||||
port_is_free() {
|
||||
"$PYTHON" - "$1" "$2" <<'PY'
|
||||
import socket
|
||||
import sys
|
||||
|
||||
host = sys.argv[1]
|
||||
port = int(sys.argv[2])
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
||||
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||
try:
|
||||
sock.bind((host, port))
|
||||
except OSError:
|
||||
raise SystemExit(1)
|
||||
PY
|
||||
}
|
||||
|
||||
wait_for_tcp() {
|
||||
"$PYTHON" - "$1" "$2" <<'PY'
|
||||
import socket
|
||||
import sys
|
||||
import time
|
||||
|
||||
host = sys.argv[1]
|
||||
port = int(sys.argv[2])
|
||||
deadline = time.monotonic() + 90
|
||||
last_error = None
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
with socket.create_connection((host, port), timeout=2):
|
||||
raise SystemExit(0)
|
||||
except OSError as exc:
|
||||
last_error = exc
|
||||
time.sleep(1)
|
||||
print(f"Timed out waiting for {host}:{port}: {last_error}", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
PY
|
||||
}
|
||||
|
||||
wait_for_url() {
|
||||
"$PYTHON" - "$1" <<'PY'
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
|
||||
url = sys.argv[1]
|
||||
deadline = time.monotonic() + 90
|
||||
last_error = None
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=2) as response:
|
||||
if 200 <= response.status < 500:
|
||||
raise SystemExit(0)
|
||||
except Exception as exc: # noqa: BLE001 - printed only on timeout.
|
||||
last_error = exc
|
||||
time.sleep(1)
|
||||
print(f"Timed out waiting for {url}: {last_error}", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
PY
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
if [ -n "${frontend_pid:-}" ] && kill -0 "$frontend_pid" 2>/dev/null; then
|
||||
kill "$frontend_pid" 2>/dev/null || true
|
||||
fi
|
||||
if [ -n "${backend_pid:-}" ] && kill -0 "$backend_pid" 2>/dev/null; then
|
||||
kill "$backend_pid" 2>/dev/null || true
|
||||
fi
|
||||
if [ -n "${worker_pid:-}" ] && kill -0 "$worker_pid" 2>/dev/null; then
|
||||
kill "$worker_pid" 2>/dev/null || true
|
||||
fi
|
||||
if [ "$STOP_DEPENDENCIES_ON_EXIT" = "1" ]; then
|
||||
compose down >/dev/null 2>&1 || true
|
||||
fi
|
||||
}
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
[ -x "$PYTHON" ] || fail "Python virtualenv not found at $PYTHON. Run: cd $ROOT && ./.venv/bin/python -m pip install -r requirements-dev.txt"
|
||||
[ -x "$NPM" ] || fail "npm not found at $NPM. Set NODE_BIN or NPM to your Node installation."
|
||||
[ -f "$WEBUI_ROOT/package.json" ] || fail "WebUI package.json not found at $WEBUI_ROOT/package.json"
|
||||
[ -d "$WEBUI_ROOT/node_modules" ] || fail "WebUI node_modules missing. Run: cd $WEBUI_ROOT && PATH=$NODE_BIN:\$PATH $NPM install"
|
||||
|
||||
mkdir -p "$LOG_DIR" "$FILE_STORAGE_LOCAL_ROOT"
|
||||
: > "$BACKEND_LOG"
|
||||
: > "$WORKER_LOG"
|
||||
: > "$FRONTEND_LOG"
|
||||
|
||||
port_is_free "$BACKEND_HOST" "$BACKEND_PORT" || fail "$BACKEND_URL is already in use"
|
||||
port_is_free "$FRONTEND_HOST" "$FRONTEND_PORT" || fail "$FRONTEND_URL is already in use"
|
||||
|
||||
printf 'Starting production-like dependencies through Docker Compose\n'
|
||||
compose up -d
|
||||
|
||||
wait_for_tcp 127.0.0.1 "${GOVOPLAN_PRODUCTION_LIKE_POSTGRES_PORT:-55433}"
|
||||
wait_for_tcp 127.0.0.1 "${GOVOPLAN_PRODUCTION_LIKE_REDIS_PORT:-56379}"
|
||||
|
||||
printf 'Running migrations and development bootstrap against %s\n' "$DATABASE_URL"
|
||||
(
|
||||
cd "$ROOT"
|
||||
"$PYTHON" -m govoplan_core.commands.init_db --database-url "$DATABASE_URL" --with-dev-data
|
||||
)
|
||||
|
||||
printf 'Starting GovOPlaN backend at %s\n' "$BACKEND_URL"
|
||||
(
|
||||
cd "$ROOT"
|
||||
"$PYTHON" -m govoplan_core.devserver --host "$BACKEND_HOST" --port "$BACKEND_PORT"
|
||||
) >"$BACKEND_LOG" 2>&1 &
|
||||
backend_pid="$!"
|
||||
|
||||
printf 'Waiting for %s/health\n' "$BACKEND_URL"
|
||||
wait_for_url "$BACKEND_URL/health" || {
|
||||
tail -n 80 "$BACKEND_LOG" >&2 || true
|
||||
fail "backend did not become healthy"
|
||||
}
|
||||
|
||||
printf 'Starting Celery worker for queues %s\n' "$CELERY_QUEUES"
|
||||
(
|
||||
cd "$ROOT"
|
||||
"$PYTHON" -m celery -A govoplan_core.celery_app:celery worker \
|
||||
--queues "$CELERY_QUEUES" \
|
||||
--hostname "govoplan-production-like@%h" \
|
||||
--loglevel INFO
|
||||
) >"$WORKER_LOG" 2>&1 &
|
||||
worker_pid="$!"
|
||||
|
||||
printf 'Starting GovOPlaN WebUI at %s\n' "$FRONTEND_URL"
|
||||
(
|
||||
cd "$WEBUI_ROOT"
|
||||
PATH="$WEBUI_ROOT/node_modules/.bin:$NODE_BIN:$PATH" \
|
||||
CHOKIDAR_USEPOLLING="${GOVOPLAN_FRONTEND_USE_POLLING:-1}" \
|
||||
CHOKIDAR_INTERVAL="${GOVOPLAN_FRONTEND_POLLING_INTERVAL:-500}" \
|
||||
VITE_API_PROXY_TARGET="$BACKEND_URL" \
|
||||
"$NPM" run dev -- --force
|
||||
) >"$FRONTEND_LOG" 2>&1 &
|
||||
frontend_pid="$!"
|
||||
|
||||
printf 'Waiting for %s\n' "$FRONTEND_URL"
|
||||
wait_for_url "$FRONTEND_URL" || {
|
||||
tail -n 80 "$FRONTEND_LOG" >&2 || true
|
||||
fail "frontend did not become reachable"
|
||||
}
|
||||
|
||||
if [ "$OPEN_BROWSER" = "1" ] && command -v xdg-open >/dev/null 2>&1; then
|
||||
xdg-open "$FRONTEND_URL" >/dev/null 2>&1 || true
|
||||
fi
|
||||
|
||||
cat <<EOF
|
||||
|
||||
GovOPlaN production-like development profile is running.
|
||||
Web UI: $FRONTEND_URL
|
||||
API: $BACKEND_URL/api/v1
|
||||
Health: $BACKEND_URL/health
|
||||
Ops: $BACKEND_URL/api/v1/ops/status
|
||||
DB: $DATABASE_URL
|
||||
Redis: $REDIS_URL
|
||||
Files: $FILE_STORAGE_LOCAL_ROOT
|
||||
|
||||
Logs:
|
||||
Backend: $BACKEND_LOG
|
||||
Worker: $WORKER_LOG
|
||||
WebUI: $FRONTEND_LOG
|
||||
|
||||
Docker dependencies remain running after Ctrl+C unless GOVOPLAN_STOP_PROFILE_DEPENDENCIES_ON_EXIT=1.
|
||||
Press Ctrl+C to stop API, worker, and WebUI.
|
||||
EOF
|
||||
|
||||
wait
|
||||
512
scripts/module-installer-rollback-drill.py
Normal file
512
scripts/module-installer-rollback-drill.py
Normal file
@@ -0,0 +1,512 @@
|
||||
#!/usr/bin/env python
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import shlex
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from collections.abc import Callable, Iterable
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
from urllib.error import URLError
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SRC = ROOT / "src"
|
||||
if str(SRC) not in sys.path:
|
||||
sys.path.insert(0, str(SRC))
|
||||
|
||||
os.environ.setdefault("APP_ENV", "test")
|
||||
os.environ.setdefault("DEV_BOOTSTRAP_ENABLED", "false")
|
||||
os.environ.setdefault("CELERY_ENABLED", "false")
|
||||
|
||||
from govoplan_core.admin.models import SystemSettings
|
||||
from govoplan_core.core.maintenance import MaintenanceMode, save_maintenance_mode
|
||||
from govoplan_core.core.module_installer import (
|
||||
cancel_module_installer_request,
|
||||
claim_next_module_installer_request,
|
||||
module_installer_daemon_status,
|
||||
module_installer_lock_status,
|
||||
queue_module_installer_request,
|
||||
read_module_installer_run,
|
||||
retry_module_installer_request,
|
||||
rollback_module_install_run,
|
||||
run_module_install_plan,
|
||||
supervise_module_install_plan,
|
||||
update_module_installer_daemon_status,
|
||||
update_module_installer_request,
|
||||
)
|
||||
from govoplan_core.core.module_management import (
|
||||
ModuleInstallPlan,
|
||||
ModuleInstallPlanItem,
|
||||
saved_desired_enabled_modules,
|
||||
saved_module_install_plan,
|
||||
save_desired_enabled_modules,
|
||||
save_module_install_plan,
|
||||
)
|
||||
from govoplan_core.core.modules import MigrationRetirementPlan, MigrationSpec, ModuleManifest
|
||||
from govoplan_core.db.base import Base
|
||||
from govoplan_core.db.session import configure_database, get_database
|
||||
from govoplan_core.server.registry import available_module_manifests
|
||||
|
||||
|
||||
Scenario = Callable[[Path], dict[str, object]]
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Run safe module-installer rollback drills in temporary runtimes.")
|
||||
parser.add_argument("--scenario", action="append", choices=sorted(SCENARIOS), help="Scenario to run. Defaults to all scenarios.")
|
||||
parser.add_argument("--runtime-root", type=Path, help="Directory for temporary drill runtimes. Defaults to a new temp directory.")
|
||||
parser.add_argument("--keep-runtime", action="store_true", help="Keep the temporary drill runtime after completion.")
|
||||
parser.add_argument("--format", choices=("json", "text"), default="text", help="Output format.")
|
||||
args = parser.parse_args()
|
||||
|
||||
runtime_root = args.runtime_root or Path(tempfile.mkdtemp(prefix="govoplan-installer-drill-"))
|
||||
runtime_root.mkdir(parents=True, exist_ok=True)
|
||||
selected = args.scenario or sorted(SCENARIOS)
|
||||
results: list[dict[str, object]] = []
|
||||
ok = True
|
||||
try:
|
||||
for name in selected:
|
||||
scenario_root = runtime_root / name
|
||||
scenario_root.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
result = SCENARIOS[name](scenario_root)
|
||||
except Exception as exc:
|
||||
ok = False
|
||||
result = {"scenario": name, "ok": False, "error": str(exc), "root": str(scenario_root)}
|
||||
else:
|
||||
ok = ok and bool(result.get("ok"))
|
||||
results.append(result)
|
||||
finally:
|
||||
if not args.keep_runtime and args.runtime_root is None:
|
||||
shutil.rmtree(runtime_root, ignore_errors=True)
|
||||
|
||||
payload = {"ok": ok, "runtime_root": str(runtime_root), "scenarios": results}
|
||||
if args.format == "json":
|
||||
print(json.dumps(payload, indent=2, sort_keys=True))
|
||||
else:
|
||||
for result in results:
|
||||
status = "ok" if result.get("ok") else "FAILED"
|
||||
print(f"{status}: {result['scenario']}")
|
||||
if result.get("run_id"):
|
||||
print(f" run: {result['run_id']}")
|
||||
if result.get("record_path"):
|
||||
print(f" record: {result['record_path']}")
|
||||
if result.get("error"):
|
||||
print(f" error: {result['error']}")
|
||||
print(f"runtime: {runtime_root}")
|
||||
return 0 if ok else 1
|
||||
|
||||
|
||||
def package_failure(root: Path) -> dict[str, object]:
|
||||
database_url, database = _sqlite_database(root)
|
||||
plan = _saved_install_plan(database, "package-failure-example")
|
||||
with database.session() as session, _patch_subprocess(_package_failure_run):
|
||||
result = supervise_module_install_plan(
|
||||
session=session,
|
||||
plan=plan,
|
||||
available=available_module_manifests(),
|
||||
current_enabled=("tenancy", "access"),
|
||||
desired_enabled=("tenancy", "access"),
|
||||
database_url=database_url,
|
||||
runtime_dir=root / "installer",
|
||||
)
|
||||
restored_plan = saved_module_install_plan(session)
|
||||
record = _run_record(root, result.run_id)
|
||||
return _scenario_result(
|
||||
"package-failure",
|
||||
root,
|
||||
result,
|
||||
expected_status="rolled-back",
|
||||
checks={
|
||||
"supervisor_status": _nested(record, "supervisor", "status") == "rolled-back",
|
||||
"plan_restored": tuple(item.status for item in restored_plan.items) == ("planned",),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def migration_failure(root: Path) -> dict[str, object]:
|
||||
database_url, database = _sqlite_database(root)
|
||||
plan = _saved_install_plan(database, "migration-failure-example")
|
||||
with database.session() as session, _patch_subprocess(_migration_failure_run):
|
||||
result = supervise_module_install_plan(
|
||||
session=session,
|
||||
plan=plan,
|
||||
available=available_module_manifests(),
|
||||
current_enabled=("tenancy", "access"),
|
||||
desired_enabled=("tenancy", "access"),
|
||||
database_url=database_url,
|
||||
runtime_dir=root / "installer",
|
||||
migrate_database=True,
|
||||
)
|
||||
record = _run_record(root, result.run_id)
|
||||
return _scenario_result(
|
||||
"migration-failure",
|
||||
root,
|
||||
result,
|
||||
expected_status="rolled-back",
|
||||
checks={
|
||||
"sqlite_backup_recorded": isinstance(_nested(record, "snapshot", "database_backup"), dict),
|
||||
"supervisor_status": _nested(record, "supervisor", "status") == "rolled-back",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def restart_failure(root: Path) -> dict[str, object]:
|
||||
database_url, database = _sqlite_database(root)
|
||||
plan = _saved_install_plan(database, "restart-failure-example")
|
||||
with database.session() as session, _patch_subprocess(_restart_failure_run):
|
||||
result = supervise_module_install_plan(
|
||||
session=session,
|
||||
plan=plan,
|
||||
available=available_module_manifests(),
|
||||
current_enabled=("tenancy", "access"),
|
||||
desired_enabled=("tenancy", "access"),
|
||||
database_url=database_url,
|
||||
runtime_dir=root / "installer",
|
||||
restart_command="restart-fails",
|
||||
)
|
||||
record = _run_record(root, result.run_id)
|
||||
return _scenario_result(
|
||||
"restart-failure",
|
||||
root,
|
||||
result,
|
||||
expected_status="rolled-back",
|
||||
checks={"supervisor_status": _nested(record, "supervisor", "status") == "rolled-back"},
|
||||
)
|
||||
|
||||
|
||||
def health_timeout(root: Path) -> dict[str, object]:
|
||||
database_url, database = _sqlite_database(root)
|
||||
plan = _saved_install_plan(database, "health-timeout-example")
|
||||
with database.session() as session, _patch_subprocess(_success_run), _patch_urlopen():
|
||||
result = supervise_module_install_plan(
|
||||
session=session,
|
||||
plan=plan,
|
||||
available=available_module_manifests(),
|
||||
current_enabled=("tenancy", "access"),
|
||||
desired_enabled=("tenancy", "access"),
|
||||
database_url=database_url,
|
||||
runtime_dir=root / "installer",
|
||||
restart_command="restart-ok",
|
||||
health_url="http://127.0.0.1:9/health",
|
||||
health_timeout_seconds=0.2,
|
||||
health_interval_seconds=0.05,
|
||||
)
|
||||
record = _run_record(root, result.run_id)
|
||||
return _scenario_result(
|
||||
"health-timeout",
|
||||
root,
|
||||
result,
|
||||
expected_status="rolled-back",
|
||||
checks={
|
||||
"health_failure_recorded": "127.0.0.1:9/health" in str(_nested(record, "supervisor", "failure_reason")),
|
||||
"supervisor_status": _nested(record, "supervisor", "status") == "rolled-back",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def destructive_retirement_failure(root: Path) -> dict[str, object]:
|
||||
database_url, database = _sqlite_database(root)
|
||||
available = {"retirement-example": _retirement_failure_manifest()}
|
||||
with database.session() as session:
|
||||
save_maintenance_mode(session, MaintenanceMode(enabled=True))
|
||||
plan = save_module_install_plan(session, [{
|
||||
"module_id": "retirement-example",
|
||||
"action": "uninstall",
|
||||
"python_package": "govoplan-retirement-example",
|
||||
"destroy_data": True,
|
||||
}])
|
||||
session.commit()
|
||||
with _patch_subprocess(_success_run):
|
||||
result = run_module_install_plan(
|
||||
session=session,
|
||||
plan=plan,
|
||||
available=available,
|
||||
current_enabled=("tenancy", "access"),
|
||||
desired_enabled=("tenancy", "access"),
|
||||
database_url=database_url,
|
||||
runtime_dir=root / "installer",
|
||||
)
|
||||
record = _run_record(root, result.run_id)
|
||||
return _scenario_result(
|
||||
"destructive-retirement-failure",
|
||||
root,
|
||||
result,
|
||||
expected_status="rolled-back",
|
||||
checks={
|
||||
"rollback_recorded": isinstance(record.get("destructive_retirement_rollback"), dict),
|
||||
"sqlite_backup_recorded": isinstance(_nested(record, "snapshot", "database_backup"), dict),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def postgres_hooks(root: Path) -> dict[str, object]:
|
||||
_, database = _sqlite_database(root)
|
||||
backup_script = (
|
||||
"import json, os, pathlib; "
|
||||
"pathlib.Path(os.environ['GOVOPLAN_DATABASE_BACKUP_PATH']).write_text('backup', encoding='utf-8'); "
|
||||
"pathlib.Path(os.environ['GOVOPLAN_DATABASE_BACKUP_METADATA']).write_text(json.dumps({'snapshot': 'external'}), encoding='utf-8')"
|
||||
)
|
||||
check_script = (
|
||||
"import os, pathlib; "
|
||||
"pathlib.Path(os.environ['GOVOPLAN_INSTALLER_RUN_DIR'], 'restore-check.marker').write_text("
|
||||
"pathlib.Path(os.environ['GOVOPLAN_DATABASE_BACKUP_PATH']).read_text(encoding='utf-8'), encoding='utf-8')"
|
||||
)
|
||||
restore_script = (
|
||||
"import os, pathlib; "
|
||||
"pathlib.Path(os.environ['GOVOPLAN_INSTALLER_RUN_DIR'], 'restore.marker').write_text("
|
||||
"os.environ.get('GOVOPLAN_DATABASE_URL', ''), encoding='utf-8')"
|
||||
)
|
||||
with database.session() as session:
|
||||
save_maintenance_mode(session, MaintenanceMode(enabled=True))
|
||||
plan = save_module_install_plan(session, [{
|
||||
"module_id": "postgres-hook-example",
|
||||
"action": "install",
|
||||
"python_package": "govoplan-postgres-hook-example",
|
||||
"python_ref": "govoplan-postgres-hook-example==0.1.0",
|
||||
}])
|
||||
session.commit()
|
||||
result = run_module_install_plan(
|
||||
session=session,
|
||||
plan=plan,
|
||||
available=available_module_manifests(),
|
||||
current_enabled=("tenancy", "access"),
|
||||
desired_enabled=("tenancy", "access"),
|
||||
database_url="postgresql://db.example.invalid/govoplan",
|
||||
runtime_dir=root / "installer",
|
||||
migrate_database=True,
|
||||
database_backup_command=f"{sys.executable} -c {shlex.quote(backup_script)}",
|
||||
database_restore_command=f"{sys.executable} -c {shlex.quote(restore_script)}",
|
||||
database_restore_check_command=f"{sys.executable} -c {shlex.quote(check_script)}",
|
||||
dry_run=True,
|
||||
)
|
||||
with _patch_subprocess(_non_shell_success_run):
|
||||
rollback = rollback_module_install_run(
|
||||
run_id=result.run_id,
|
||||
runtime_dir=root / "installer",
|
||||
database_url="postgresql://db.example.invalid/govoplan",
|
||||
)
|
||||
run_dir = result.record_path.parent
|
||||
record = _run_record(root, result.run_id)
|
||||
ok = (
|
||||
result.status == "dry-run"
|
||||
and rollback.status == "rolled-back"
|
||||
and (run_dir / "restore-check.marker").read_text(encoding="utf-8") == "backup"
|
||||
and (run_dir / "restore.marker").read_text(encoding="utf-8") == "postgresql://db.example.invalid/govoplan"
|
||||
and isinstance(_nested(record, "snapshot", "database_backup"), dict)
|
||||
)
|
||||
return {
|
||||
"scenario": "postgres-hooks",
|
||||
"ok": ok,
|
||||
"root": str(root),
|
||||
"run_id": result.run_id,
|
||||
"record_path": str(result.record_path),
|
||||
"rollback_status": rollback.status,
|
||||
}
|
||||
|
||||
|
||||
def queue_and_lock(root: Path) -> dict[str, object]:
|
||||
runtime_dir = root / "installer"
|
||||
status = update_module_installer_daemon_status(
|
||||
runtime_dir=runtime_dir,
|
||||
patch={"status": "polling", "current_request_id": None},
|
||||
)
|
||||
request = queue_module_installer_request(
|
||||
runtime_dir=runtime_dir,
|
||||
requested_by="drill",
|
||||
options={"migrate_database": True, "health_urls": ["http://127.0.0.1:8000/health"]},
|
||||
)
|
||||
claimed = claim_next_module_installer_request(runtime_dir=runtime_dir)
|
||||
assert claimed is not None
|
||||
failed = update_module_installer_request(
|
||||
runtime_dir=runtime_dir,
|
||||
request_id=str(claimed["request_id"]),
|
||||
patch={"status": "failed", "error": "simulated drill failure"},
|
||||
)
|
||||
retry = retry_module_installer_request(runtime_dir=runtime_dir, request_id=str(failed["request_id"]), requested_by="drill")
|
||||
cancelled = cancel_module_installer_request(runtime_dir=runtime_dir, request_id=str(retry["request_id"]), cancelled_by="drill")
|
||||
lock_path = runtime_dir / "install.lock"
|
||||
lock_path.write_text(json.dumps({"pid": 999999, "created_at": "drill"}), encoding="utf-8")
|
||||
lock = module_installer_lock_status(runtime_dir=runtime_dir)
|
||||
lock_path.unlink()
|
||||
return {
|
||||
"scenario": "queue-and-lock",
|
||||
"ok": bool(status["running"]) and request["status"] == "queued" and failed["status"] == "failed" and cancelled["status"] == "cancelled" and lock["locked"] is True and not module_installer_lock_status(runtime_dir=runtime_dir)["locked"],
|
||||
"root": str(root),
|
||||
"daemon_status_path": str(runtime_dir / "daemon.status.json"),
|
||||
}
|
||||
|
||||
|
||||
def _sqlite_database(root: Path) -> tuple[str, Any]:
|
||||
database_url = f"sqlite:///{root / 'drill.db'}"
|
||||
configure_database(database_url)
|
||||
database = get_database()
|
||||
Base.metadata.create_all(bind=database.engine, tables=[SystemSettings.__table__])
|
||||
return database_url, database
|
||||
|
||||
|
||||
def _saved_install_plan(database: Any, module_id: str) -> ModuleInstallPlan:
|
||||
with database.session() as session:
|
||||
save_maintenance_mode(session, MaintenanceMode(enabled=True))
|
||||
plan = save_module_install_plan(session, [{
|
||||
"module_id": module_id,
|
||||
"action": "install",
|
||||
"python_package": f"govoplan-{module_id}",
|
||||
"python_ref": f"govoplan-{module_id}==0.1.0",
|
||||
}])
|
||||
save_desired_enabled_modules(session, ("tenancy", "access"))
|
||||
session.commit()
|
||||
return plan
|
||||
|
||||
|
||||
def _retirement_failure_manifest() -> ModuleManifest:
|
||||
def destroy(_session: object, _module_id: str) -> None:
|
||||
raise RuntimeError("simulated destructive retirement failure")
|
||||
|
||||
def provider(_session: object | None, _module_id: str) -> MigrationRetirementPlan:
|
||||
return MigrationRetirementPlan(
|
||||
supported=True,
|
||||
summary="Drill retirement provider supports destructive cleanup.",
|
||||
destroy_data_supported=True,
|
||||
destroy_data_summary="Drill destructive cleanup will fail during execution.",
|
||||
destroy_data_executor=destroy,
|
||||
)
|
||||
|
||||
return ModuleManifest(
|
||||
id="retirement-example",
|
||||
name="Retirement example",
|
||||
version="0.1.0",
|
||||
migration_spec=MigrationSpec(
|
||||
module_id="retirement-example",
|
||||
retirement_supported=True,
|
||||
retirement_provider=provider,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _scenario_result(
|
||||
scenario: str,
|
||||
root: Path,
|
||||
result: Any,
|
||||
*,
|
||||
expected_status: str,
|
||||
checks: dict[str, bool],
|
||||
) -> dict[str, object]:
|
||||
return {
|
||||
"scenario": scenario,
|
||||
"ok": result.status == expected_status and all(checks.values()),
|
||||
"root": str(root),
|
||||
"run_id": result.run_id,
|
||||
"record_path": str(result.record_path),
|
||||
"status": result.status,
|
||||
"checks": checks,
|
||||
}
|
||||
|
||||
|
||||
def _run_record(root: Path, run_id: str) -> dict[str, object]:
|
||||
return read_module_installer_run(runtime_dir=root / "installer", run_id=run_id)
|
||||
|
||||
|
||||
def _nested(value: dict[str, object], *keys: str) -> object:
|
||||
current: object = value
|
||||
for key in keys:
|
||||
if not isinstance(current, dict):
|
||||
return None
|
||||
current = current.get(key)
|
||||
return current
|
||||
|
||||
|
||||
def _command_text(argv: object) -> str:
|
||||
if isinstance(argv, list | tuple):
|
||||
return " ".join(str(item) for item in argv)
|
||||
return str(argv)
|
||||
|
||||
|
||||
def _completed(return_code: int = 0, stdout: str = "", stderr: str = "") -> SimpleNamespace:
|
||||
return SimpleNamespace(returncode=return_code, stdout=stdout, stderr=stderr)
|
||||
|
||||
|
||||
def _success_run(argv: object, *_args: object, **_kwargs: object) -> SimpleNamespace:
|
||||
text = _command_text(argv)
|
||||
if "pip freeze" in text:
|
||||
return _completed(stdout="govoplan-core==0.0.0\n")
|
||||
return _completed()
|
||||
|
||||
|
||||
def _non_shell_success_run(argv: object, *_args: object, **kwargs: object) -> Any:
|
||||
if kwargs.get("shell"):
|
||||
return ORIGINAL_SUBPROCESS_RUN(argv, *_args, **kwargs)
|
||||
return _success_run(argv, *_args, **kwargs)
|
||||
|
||||
|
||||
def _package_failure_run(argv: object, *_args: object, **kwargs: object) -> SimpleNamespace:
|
||||
text = _command_text(argv)
|
||||
if "pip install" in text and "==" in text and "-r" not in text:
|
||||
return _completed(23, stderr="simulated package install failure")
|
||||
return _success_run(argv, *_args, **kwargs)
|
||||
|
||||
|
||||
def _migration_failure_run(argv: object, *_args: object, **kwargs: object) -> SimpleNamespace:
|
||||
text = _command_text(argv)
|
||||
if "govoplan_core.commands.init_db" in text:
|
||||
return _completed(24, stderr="simulated migration failure")
|
||||
return _success_run(argv, *_args, **kwargs)
|
||||
|
||||
|
||||
def _restart_failure_run(argv: object, *_args: object, **kwargs: object) -> SimpleNamespace:
|
||||
if kwargs.get("shell") and str(argv).strip() == "restart-fails":
|
||||
return _completed(25, stderr="simulated restart failure")
|
||||
return _success_run(argv, *_args, **kwargs)
|
||||
|
||||
|
||||
ORIGINAL_SUBPROCESS_RUN = subprocess.run
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _patch_subprocess(fake_run: Callable[..., Any]) -> Iterable[None]:
|
||||
import govoplan_core.core.module_installer as installer
|
||||
|
||||
original = installer.subprocess.run
|
||||
installer.subprocess.run = fake_run # type: ignore[assignment]
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
installer.subprocess.run = original # type: ignore[assignment]
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _patch_urlopen() -> Iterable[None]:
|
||||
import govoplan_core.core.module_installer as installer
|
||||
|
||||
original = installer.urllib.request.urlopen
|
||||
|
||||
def fail(*_args: object, **_kwargs: object) -> None:
|
||||
raise URLError("simulated health timeout")
|
||||
|
||||
installer.urllib.request.urlopen = fail # type: ignore[assignment]
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
installer.urllib.request.urlopen = original # type: ignore[assignment]
|
||||
|
||||
|
||||
SCENARIOS: dict[str, Scenario] = {
|
||||
"destructive-retirement-failure": destructive_retirement_failure,
|
||||
"health-timeout": health_timeout,
|
||||
"migration-failure": migration_failure,
|
||||
"package-failure": package_failure,
|
||||
"postgres-hooks": postgres_hooks,
|
||||
"queue-and-lock": queue_and_lock,
|
||||
"restart-failure": restart_failure,
|
||||
}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
128
scripts/postgres-integration-check.py
Normal file
128
scripts/postgres-integration-check.py
Normal file
@@ -0,0 +1,128 @@
|
||||
#!/usr/bin/env python
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SRC = ROOT / "src"
|
||||
|
||||
DEFAULT_MODULE_SETS: tuple[tuple[str, str], ...] = (
|
||||
("core-only", "tenancy,access,admin"),
|
||||
("files-only", "tenancy,access,admin,files"),
|
||||
("mail-only", "tenancy,access,admin,mail"),
|
||||
("campaign-only", "tenancy,access,admin,campaigns"),
|
||||
("campaign-with-files", "tenancy,access,admin,campaigns,files"),
|
||||
("campaign-with-mail", "tenancy,access,admin,campaigns,mail"),
|
||||
("full-product", "tenancy,access,admin,policy,audit,campaigns,files,mail,calendar,docs,ops"),
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Run GovOPlaN migrations and startup smoke checks against a disposable PostgreSQL database.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--database-url",
|
||||
default=os.environ.get("GOVOPLAN_POSTGRES_DATABASE_URL") or os.environ.get("DATABASE_URL"),
|
||||
help="SQLAlchemy PostgreSQL URL, for example postgresql+psycopg://user:pass@127.0.0.1:55432/govoplan.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--module-set",
|
||||
action="append",
|
||||
default=[],
|
||||
metavar="NAME=MODULES",
|
||||
help="Module permutation to check. May be passed multiple times. Defaults to the standard product permutations.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--reset-schema",
|
||||
action="store_true",
|
||||
help="DROP and recreate the public schema before every module set. Use only with a disposable database.",
|
||||
)
|
||||
parser.add_argument("--skip-migrate", action="store_true", help="Skip govoplan_core.commands.init_db.")
|
||||
parser.add_argument("--skip-smoke", action="store_true", help="Skip govoplan_core.devserver --smoke.")
|
||||
parser.add_argument("--python", default=sys.executable, help="Python executable to use. Defaults to the current interpreter.")
|
||||
parser.add_argument("--app-env", default="staging", help="APP_ENV value to pass to subprocesses. Defaults to staging.")
|
||||
args = parser.parse_args()
|
||||
|
||||
database_url = (args.database_url or "").strip()
|
||||
if not database_url:
|
||||
parser.error("--database-url or GOVOPLAN_POSTGRES_DATABASE_URL is required")
|
||||
if not database_url.startswith("postgresql"):
|
||||
parser.error("This check is PostgreSQL-only. Use a postgresql+psycopg:// or postgresql:// DATABASE_URL.")
|
||||
|
||||
module_sets = _parse_module_sets(args.module_set)
|
||||
ok = True
|
||||
for name, modules in module_sets:
|
||||
print(f"\n== {name}: {modules} ==")
|
||||
if args.reset_schema:
|
||||
_reset_public_schema(database_url)
|
||||
env = _check_env(database_url=database_url, modules=modules, app_env=args.app_env)
|
||||
if not args.skip_migrate:
|
||||
ok = _run([args.python, "-m", "govoplan_core.commands.init_db", "--database-url", database_url], env=env) and ok
|
||||
if not args.skip_smoke:
|
||||
ok = _run([args.python, "-m", "govoplan_core.devserver", "--smoke", "--no-reload"], env=env) and ok
|
||||
return 0 if ok else 1
|
||||
|
||||
|
||||
def _parse_module_sets(raw: list[str]) -> tuple[tuple[str, str], ...]:
|
||||
if not raw:
|
||||
return DEFAULT_MODULE_SETS
|
||||
parsed: list[tuple[str, str]] = []
|
||||
for item in raw:
|
||||
name, separator, modules = item.partition("=")
|
||||
if not separator or not name.strip() or not modules.strip():
|
||||
raise SystemExit(f"Invalid --module-set value {item!r}. Use NAME=module_a,module_b.")
|
||||
parsed.append((name.strip(), modules.strip()))
|
||||
return tuple(parsed)
|
||||
|
||||
|
||||
def _check_env(*, database_url: str, modules: str, app_env: str) -> dict[str, str]:
|
||||
env = os.environ.copy()
|
||||
env["APP_ENV"] = app_env
|
||||
env["DATABASE_URL"] = database_url
|
||||
env["ENABLED_MODULES"] = modules
|
||||
env["DEV_AUTO_MIGRATE_ENABLED"] = "false"
|
||||
env["DEV_BOOTSTRAP_ENABLED"] = "false"
|
||||
env["CELERY_ENABLED"] = "false"
|
||||
env.setdefault("MASTER_KEY_B64", base64.urlsafe_b64encode(os.urandom(32)).decode("ascii"))
|
||||
env.setdefault("FILE_STORAGE_BACKEND", "local")
|
||||
env.setdefault("FILE_STORAGE_LOCAL_ROOT", str(ROOT / "runtime" / "postgres-check-files"))
|
||||
env.setdefault("MOCK_MAILBOX_DIR", str(ROOT / "runtime" / "postgres-check-mock-mailbox"))
|
||||
env["PYTHONPATH"] = os.pathsep.join(part for part in (str(SRC), env.get("PYTHONPATH", "")) if part)
|
||||
return env
|
||||
|
||||
|
||||
def _run(command: list[str], *, env: dict[str, str]) -> bool:
|
||||
print("+ " + " ".join(command))
|
||||
result = subprocess.run(command, cwd=ROOT, env=env, check=False)
|
||||
if result.returncode != 0:
|
||||
print(f"Command failed with exit code {result.returncode}: {' '.join(command)}", file=sys.stderr)
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _reset_public_schema(database_url: str) -> None:
|
||||
try:
|
||||
from sqlalchemy import create_engine
|
||||
except ModuleNotFoundError as exc:
|
||||
raise SystemExit("SQLAlchemy is required for --reset-schema. Install govoplan-core requirements first.") from exc
|
||||
|
||||
engine = create_engine(database_url, isolation_level="AUTOCOMMIT")
|
||||
try:
|
||||
with engine.connect() as connection:
|
||||
connection.exec_driver_sql("DROP SCHEMA IF EXISTS public CASCADE")
|
||||
connection.exec_driver_sql("CREATE SCHEMA public")
|
||||
connection.exec_driver_sql("GRANT ALL ON SCHEMA public TO public")
|
||||
finally:
|
||||
engine.dispose()
|
||||
print("Reset PostgreSQL schema: public")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -24,6 +24,11 @@ Options:
|
||||
-m, --message <text> Commit message. Defaults to "Release v<x.y.z>".
|
||||
--tag-message <text> Annotated tag message. Defaults to the commit message.
|
||||
--skip-release-lock Do not regenerate core webui/package-lock.release.json.
|
||||
--warn-migration-audit Run the migration release audit without baseline strictness.
|
||||
--strict-migration-audit
|
||||
Require current Alembic heads to be recorded in the
|
||||
latest migration release baseline.
|
||||
--skip-migration-audit Do not run the release migration audit.
|
||||
--publish-web-catalog After core/modules are pushed, publish a signed release
|
||||
catalog into govoplan-web.
|
||||
--web-root <path> govoplan-web checkout for --publish-web-catalog.
|
||||
@@ -51,18 +56,22 @@ Repos:
|
||||
govoplan-access
|
||||
govoplan-admin
|
||||
govoplan-tenancy
|
||||
govoplan-organizations
|
||||
govoplan-identity
|
||||
govoplan-policy
|
||||
govoplan-audit
|
||||
govoplan-dashboard
|
||||
govoplan-files
|
||||
govoplan-mail
|
||||
govoplan-campaign
|
||||
govoplan-calendar
|
||||
govoplan-ops
|
||||
govoplan-core
|
||||
|
||||
Roadmap/scaffold module repositories are tag-only until they have package
|
||||
metadata: addresses, appointments, cases, connectors, dms, erp,
|
||||
fit-connect, forms, identity-trust, idm, ledger, notifications, ops,
|
||||
payments, portal, reporting, scheduling, search, tasks, templates, workflow, xoev,
|
||||
fit-connect, forms, identity-trust, idm, ledger, notifications, payments,
|
||||
portal, reporting, scheduling, search, tasks, templates, workflow, xoev,
|
||||
xrechnung, and xta-osci.
|
||||
|
||||
Examples:
|
||||
@@ -83,12 +92,16 @@ PACKAGE_MODULE_REPOS=(
|
||||
"$PARENT/govoplan-access"
|
||||
"$PARENT/govoplan-admin"
|
||||
"$PARENT/govoplan-tenancy"
|
||||
"$PARENT/govoplan-organizations"
|
||||
"$PARENT/govoplan-identity"
|
||||
"$PARENT/govoplan-policy"
|
||||
"$PARENT/govoplan-audit"
|
||||
"$PARENT/govoplan-dashboard"
|
||||
"$PARENT/govoplan-files"
|
||||
"$PARENT/govoplan-mail"
|
||||
"$PARENT/govoplan-campaign"
|
||||
"$PARENT/govoplan-calendar"
|
||||
"$PARENT/govoplan-ops"
|
||||
)
|
||||
TAG_ONLY_MODULE_REPOS=(
|
||||
"$PARENT/govoplan-addresses"
|
||||
@@ -103,7 +116,6 @@ TAG_ONLY_MODULE_REPOS=(
|
||||
"$PARENT/govoplan-idm"
|
||||
"$PARENT/govoplan-ledger"
|
||||
"$PARENT/govoplan-notifications"
|
||||
"$PARENT/govoplan-ops"
|
||||
"$PARENT/govoplan-payments"
|
||||
"$PARENT/govoplan-portal"
|
||||
"$PARENT/govoplan-reporting"
|
||||
@@ -132,6 +144,7 @@ TAG_MESSAGE=""
|
||||
DRY_RUN=0
|
||||
YES=0
|
||||
GENERATE_RELEASE_LOCK=1
|
||||
MIGRATION_AUDIT_MODE="auto"
|
||||
NPM_BIN="${NPM:-}"
|
||||
PUBLISH_WEB_CATALOG=0
|
||||
WEB_ROOT="$PARENT/govoplan-web"
|
||||
@@ -282,6 +295,12 @@ update_manifest_version() {
|
||||
govoplan-tenancy)
|
||||
manifest_path="$repo/src/govoplan_tenancy/backend/manifest.py"
|
||||
;;
|
||||
govoplan-organizations)
|
||||
manifest_path="$repo/src/govoplan_organizations/backend/manifest.py"
|
||||
;;
|
||||
govoplan-identity)
|
||||
manifest_path="$repo/src/govoplan_identity/backend/manifest.py"
|
||||
;;
|
||||
govoplan-policy)
|
||||
manifest_path="$repo/src/govoplan_policy/backend/manifest.py"
|
||||
;;
|
||||
@@ -408,6 +427,32 @@ generate_release_lock() {
|
||||
run "$ROOT/scripts/generate-release-lock.sh" --npm "$NPM_BIN"
|
||||
}
|
||||
|
||||
run_migration_release_audit() {
|
||||
local audit_script="$ROOT/scripts/release-migration-audit.py"
|
||||
local command=("$PYTHON" "$audit_script")
|
||||
|
||||
case "$MIGRATION_AUDIT_MODE" in
|
||||
skip)
|
||||
echo "Migration release audit: skipped"
|
||||
return
|
||||
;;
|
||||
strict)
|
||||
command+=("--strict")
|
||||
;;
|
||||
auto)
|
||||
command+=("--strict-if-baseline")
|
||||
;;
|
||||
warn)
|
||||
;;
|
||||
*)
|
||||
fail "unknown migration audit mode: $MIGRATION_AUDIT_MODE"
|
||||
;;
|
||||
esac
|
||||
|
||||
[[ -f "$audit_script" ]] || fail "missing migration audit helper: $audit_script"
|
||||
run "${command[@]}"
|
||||
}
|
||||
|
||||
print_command() {
|
||||
printf '+'
|
||||
printf ' %q' "$@"
|
||||
@@ -514,6 +559,18 @@ while [[ $# -gt 0 ]]; do
|
||||
GENERATE_RELEASE_LOCK=0
|
||||
shift
|
||||
;;
|
||||
--warn-migration-audit)
|
||||
MIGRATION_AUDIT_MODE="warn"
|
||||
shift
|
||||
;;
|
||||
--strict-migration-audit)
|
||||
MIGRATION_AUDIT_MODE="strict"
|
||||
shift
|
||||
;;
|
||||
--skip-migration-audit)
|
||||
MIGRATION_AUDIT_MODE="skip"
|
||||
shift
|
||||
;;
|
||||
--publish-web-catalog)
|
||||
PUBLISH_WEB_CATALOG=1
|
||||
shift
|
||||
@@ -690,6 +747,7 @@ if [[ "$GENERATE_RELEASE_LOCK" -eq 1 ]]; then
|
||||
else
|
||||
echo " release lock: skipped"
|
||||
fi
|
||||
echo " migration audit: $MIGRATION_AUDIT_MODE"
|
||||
if [[ "$PUBLISH_WEB_CATALOG" -eq 1 ]]; then
|
||||
echo " web catalog: publish $CATALOG_CHANNEL catalog to $WEB_ROOT after core tag is pushed"
|
||||
else
|
||||
@@ -707,6 +765,8 @@ for repo in "${REPOS[@]}"; do
|
||||
echo
|
||||
done
|
||||
|
||||
run_migration_release_audit
|
||||
|
||||
confirm_release
|
||||
|
||||
for repo in "${PACKAGE_REPOS[@]}"; do
|
||||
|
||||
422
scripts/release-migration-audit.py
Normal file
422
scripts/release-migration-audit.py
Normal file
@@ -0,0 +1,422 @@
|
||||
#!/usr/bin/env python
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import ast
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DEFAULT_BASELINE_FILE = ROOT / "docs" / "migration-release-baselines.json"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Migration:
|
||||
owner: str
|
||||
path: Path
|
||||
revision: str
|
||||
down_revisions: tuple[str, ...]
|
||||
branch_labels: tuple[str, ...]
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Audit GovOPlaN Alembic migrations against the release baseline policy.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--workspace-root",
|
||||
type=Path,
|
||||
default=ROOT.parent,
|
||||
help="Directory containing govoplan-* checkouts. Defaults to the parent of govoplan-core.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--baseline-file",
|
||||
type=Path,
|
||||
default=DEFAULT_BASELINE_FILE,
|
||||
help="JSON file recording released migration heads.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--strict",
|
||||
action="store_true",
|
||||
help="Fail when current heads are not recorded in the latest release baseline.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--strict-if-baseline",
|
||||
action="store_true",
|
||||
help="Run in strict mode only after at least one release baseline is recorded.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--record-release",
|
||||
metavar="VERSION",
|
||||
help="Record the current reviewed migration heads as a release baseline.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--replace-release",
|
||||
action="store_true",
|
||||
help="Replace an existing release entry when used with --record-release.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--squash-plan",
|
||||
action="store_true",
|
||||
help="Print the reviewed/manual squash checklist for the current graph.",
|
||||
)
|
||||
parser.add_argument("--json", action="store_true", help="Emit machine-readable JSON.")
|
||||
args = parser.parse_args()
|
||||
|
||||
migrations = discover_migrations(args.workspace_root)
|
||||
baseline = load_baseline_file(args.baseline_file)
|
||||
report = build_report(
|
||||
migrations,
|
||||
baseline=baseline,
|
||||
baseline_file=args.baseline_file,
|
||||
workspace_root=args.workspace_root,
|
||||
)
|
||||
|
||||
if args.record_release:
|
||||
if report["graph_errors"]:
|
||||
print_text_report(report)
|
||||
return 1
|
||||
baseline = record_release_baseline(
|
||||
baseline,
|
||||
report,
|
||||
release=args.record_release,
|
||||
replace=args.replace_release,
|
||||
)
|
||||
write_baseline_file(args.baseline_file, baseline)
|
||||
report = build_report(
|
||||
migrations,
|
||||
baseline=baseline,
|
||||
baseline_file=args.baseline_file,
|
||||
workspace_root=args.workspace_root,
|
||||
)
|
||||
|
||||
if args.squash_plan:
|
||||
print_squash_plan(report)
|
||||
elif args.json:
|
||||
print(json.dumps(report, indent=2, sort_keys=True))
|
||||
else:
|
||||
print_text_report(report)
|
||||
|
||||
has_graph_errors = bool(report["graph_errors"])
|
||||
has_strict_errors = bool(report["strict_errors"])
|
||||
strict_required = args.strict or (args.strict_if_baseline and report["release_count"] > 0)
|
||||
if has_graph_errors or (strict_required and has_strict_errors):
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
def discover_migrations(workspace_root: Path) -> list[Migration]:
|
||||
migrations: list[Migration] = []
|
||||
for versions_dir in _migration_version_dirs(workspace_root):
|
||||
owner = owner_for_versions_dir(versions_dir)
|
||||
for path in sorted(versions_dir.glob("*.py")):
|
||||
if path.name == "__init__.py":
|
||||
continue
|
||||
migration = parse_migration_file(owner, path)
|
||||
if migration is not None:
|
||||
migrations.append(migration)
|
||||
return migrations
|
||||
|
||||
|
||||
def _migration_version_dirs(workspace_root: Path) -> list[Path]:
|
||||
candidates = [ROOT / "alembic" / "versions"]
|
||||
for repo in sorted(workspace_root.glob("govoplan-*")):
|
||||
candidates.extend(sorted(repo.glob("src/govoplan_*/backend/migrations/versions")))
|
||||
seen: set[Path] = set()
|
||||
result: list[Path] = []
|
||||
for candidate in candidates:
|
||||
resolved = candidate.resolve()
|
||||
if resolved in seen or not candidate.is_dir():
|
||||
continue
|
||||
seen.add(resolved)
|
||||
result.append(candidate)
|
||||
return result
|
||||
|
||||
|
||||
def owner_for_versions_dir(versions_dir: Path) -> str:
|
||||
if (ROOT / "alembic" / "versions").resolve() == versions_dir.resolve():
|
||||
return "govoplan-core"
|
||||
for parent in versions_dir.parents:
|
||||
if parent.name.startswith("govoplan-"):
|
||||
return parent.name
|
||||
return versions_dir.parent.name
|
||||
|
||||
|
||||
def parse_migration_file(owner: str, path: Path) -> Migration | None:
|
||||
tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
|
||||
values: dict[str, Any] = {}
|
||||
for statement in tree.body:
|
||||
if isinstance(statement, ast.Assign):
|
||||
for target in statement.targets:
|
||||
if isinstance(target, ast.Name) and target.id in {"revision", "down_revision", "branch_labels"}:
|
||||
values[target.id] = ast.literal_eval(statement.value)
|
||||
elif (
|
||||
isinstance(statement, ast.AnnAssign)
|
||||
and isinstance(statement.target, ast.Name)
|
||||
and statement.target.id in {"revision", "down_revision", "branch_labels"}
|
||||
and statement.value is not None
|
||||
):
|
||||
values[statement.target.id] = ast.literal_eval(statement.value)
|
||||
revision = values.get("revision")
|
||||
if not isinstance(revision, str):
|
||||
return None
|
||||
return Migration(
|
||||
owner=owner,
|
||||
path=path,
|
||||
revision=revision,
|
||||
down_revisions=_normalize_revision_tuple(values.get("down_revision")),
|
||||
branch_labels=_normalize_string_tuple(values.get("branch_labels")),
|
||||
)
|
||||
|
||||
|
||||
def _normalize_revision_tuple(value: Any) -> tuple[str, ...]:
|
||||
if value is None:
|
||||
return ()
|
||||
if isinstance(value, str):
|
||||
return (value,)
|
||||
if isinstance(value, (list, tuple, set)):
|
||||
return tuple(str(item) for item in value if item is not None)
|
||||
return (str(value),)
|
||||
|
||||
|
||||
def _normalize_string_tuple(value: Any) -> tuple[str, ...]:
|
||||
if value is None:
|
||||
return ()
|
||||
if isinstance(value, str):
|
||||
return (value,)
|
||||
if isinstance(value, (list, tuple, set)):
|
||||
return tuple(str(item) for item in value)
|
||||
return (str(value),)
|
||||
|
||||
|
||||
def load_baseline_file(path: Path) -> dict[str, Any]:
|
||||
if not path.exists():
|
||||
return {"version": 1, "releases": [], "_missing": True}
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
if not isinstance(payload, dict):
|
||||
raise SystemExit(f"{path} must contain a JSON object")
|
||||
releases = payload.get("releases")
|
||||
if not isinstance(releases, list):
|
||||
raise SystemExit(f"{path} must contain a releases list")
|
||||
return payload
|
||||
|
||||
|
||||
def build_report(
|
||||
migrations: list[Migration],
|
||||
*,
|
||||
baseline: dict[str, Any],
|
||||
baseline_file: Path,
|
||||
workspace_root: Path,
|
||||
) -> dict[str, Any]:
|
||||
revisions: dict[str, Migration] = {}
|
||||
graph_errors: list[str] = []
|
||||
for migration in migrations:
|
||||
existing = revisions.get(migration.revision)
|
||||
if existing is not None:
|
||||
graph_errors.append(
|
||||
f"duplicate revision {migration.revision}: {existing.path} and {migration.path}"
|
||||
)
|
||||
revisions[migration.revision] = migration
|
||||
|
||||
referenced = {revision for migration in migrations for revision in migration.down_revisions}
|
||||
for migration in migrations:
|
||||
for down_revision in migration.down_revisions:
|
||||
if down_revision not in revisions:
|
||||
graph_errors.append(
|
||||
f"{migration.revision} references missing down_revision {down_revision} in {migration.path}"
|
||||
)
|
||||
|
||||
current_heads = sorted(revision for revision in revisions if revision not in referenced)
|
||||
current_head_entries = [
|
||||
{
|
||||
"owner": revisions[revision].owner,
|
||||
"revision": revision,
|
||||
}
|
||||
for revision in current_heads
|
||||
]
|
||||
owner_rows = []
|
||||
for owner in sorted({migration.owner for migration in migrations}):
|
||||
owner_migrations = [migration for migration in migrations if migration.owner == owner]
|
||||
owner_revisions = {migration.revision for migration in owner_migrations}
|
||||
owner_referenced = {
|
||||
revision
|
||||
for migration in owner_migrations
|
||||
for revision in migration.down_revisions
|
||||
if revision in owner_revisions
|
||||
}
|
||||
owner_heads = sorted(owner_revisions - owner_referenced)
|
||||
owner_rows.append(
|
||||
{
|
||||
"owner": owner,
|
||||
"migrations": len(owner_migrations),
|
||||
"heads": owner_heads,
|
||||
}
|
||||
)
|
||||
|
||||
releases = baseline.get("releases") or []
|
||||
latest_release = releases[-1] if releases else None
|
||||
latest_heads = _latest_release_heads(latest_release)
|
||||
unrecorded_heads = sorted(set(current_heads) - latest_heads) if latest_release else current_heads
|
||||
|
||||
strict_errors: list[str] = []
|
||||
if baseline.get("_missing"):
|
||||
strict_errors.append(f"baseline file is missing: {baseline_file}")
|
||||
if not releases:
|
||||
strict_errors.append("no released migration baseline has been recorded yet")
|
||||
if unrecorded_heads:
|
||||
strict_errors.append(
|
||||
"current migration heads are not recorded in the latest release baseline: "
|
||||
+ ", ".join(unrecorded_heads)
|
||||
)
|
||||
return {
|
||||
"workspace_root": str(workspace_root),
|
||||
"baseline_file": str(baseline_file),
|
||||
"baseline_file_missing": bool(baseline.get("_missing")),
|
||||
"release_count": len(releases),
|
||||
"latest_release": latest_release,
|
||||
"owners": owner_rows,
|
||||
"current_heads": current_heads,
|
||||
"current_head_entries": current_head_entries,
|
||||
"graph_errors": graph_errors,
|
||||
"strict_errors": strict_errors,
|
||||
}
|
||||
|
||||
|
||||
def _latest_release_heads(latest_release: Any) -> set[str]:
|
||||
if not isinstance(latest_release, dict):
|
||||
return set()
|
||||
heads = latest_release.get("heads") or []
|
||||
graph_heads = latest_release.get("graph_heads") or []
|
||||
result: set[str] = set()
|
||||
for head_list in (heads, graph_heads):
|
||||
if not isinstance(head_list, list):
|
||||
continue
|
||||
for entry in head_list:
|
||||
if isinstance(entry, str):
|
||||
result.add(entry)
|
||||
elif isinstance(entry, dict) and isinstance(entry.get("revision"), str):
|
||||
result.add(entry["revision"])
|
||||
owner_heads = latest_release.get("owner_heads") or []
|
||||
if isinstance(owner_heads, list):
|
||||
for entry in owner_heads:
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
revisions = entry.get("revisions") or []
|
||||
if isinstance(revisions, str):
|
||||
result.add(revisions)
|
||||
elif isinstance(revisions, list):
|
||||
result.update(revision for revision in revisions if isinstance(revision, str))
|
||||
return result
|
||||
|
||||
|
||||
def record_release_baseline(
|
||||
baseline: dict[str, Any],
|
||||
report: dict[str, Any],
|
||||
*,
|
||||
release: str,
|
||||
replace: bool,
|
||||
) -> dict[str, Any]:
|
||||
payload = {
|
||||
key: value
|
||||
for key, value in baseline.items()
|
||||
if not key.startswith("_")
|
||||
}
|
||||
payload.setdefault("version", 1)
|
||||
releases = list(payload.get("releases") or [])
|
||||
existing_index = next(
|
||||
(index for index, item in enumerate(releases) if isinstance(item, dict) and item.get("release") == release),
|
||||
None,
|
||||
)
|
||||
if existing_index is not None and not replace:
|
||||
raise SystemExit(f"release {release} already exists in baseline file; pass --replace-release to update it")
|
||||
|
||||
entry = {
|
||||
"release": release,
|
||||
"recorded_at": datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z"),
|
||||
"squash_policy": "reviewed-manual",
|
||||
"heads": report["current_head_entries"],
|
||||
"owner_heads": [
|
||||
{
|
||||
"owner": owner["owner"],
|
||||
"revisions": owner["heads"],
|
||||
}
|
||||
for owner in report["owners"]
|
||||
],
|
||||
}
|
||||
if existing_index is None:
|
||||
releases.append(entry)
|
||||
else:
|
||||
releases[existing_index] = entry
|
||||
payload["releases"] = releases
|
||||
return payload
|
||||
|
||||
|
||||
def write_baseline_file(path: Path, baseline: dict[str, Any]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(baseline, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def print_text_report(report: dict[str, Any]) -> None:
|
||||
print("Migration release audit")
|
||||
print(f" workspace: {report['workspace_root']}")
|
||||
print(f" baseline file: {report['baseline_file']}")
|
||||
print(f" recorded releases: {report['release_count']}")
|
||||
print()
|
||||
print("Owners:")
|
||||
for owner in report["owners"]:
|
||||
heads = ", ".join(owner["heads"]) if owner["heads"] else "-"
|
||||
print(f" {owner['owner']}: {owner['migrations']} migration(s), head(s): {heads}")
|
||||
print()
|
||||
print("Current heads:")
|
||||
for entry in report["current_head_entries"]:
|
||||
print(f" {entry['owner']}: {entry['revision']}")
|
||||
if not report["current_head_entries"]:
|
||||
print(" -")
|
||||
|
||||
if report["graph_errors"]:
|
||||
print()
|
||||
print("Graph errors:")
|
||||
for error in report["graph_errors"]:
|
||||
print(f" - {error}")
|
||||
|
||||
if report["strict_errors"]:
|
||||
print()
|
||||
print("Release baseline warnings:")
|
||||
for error in report["strict_errors"]:
|
||||
print(f" - {error}")
|
||||
|
||||
|
||||
def print_squash_plan(report: dict[str, Any]) -> None:
|
||||
print("Manual migration squash plan")
|
||||
print()
|
||||
print("Policy:")
|
||||
print(" - Do not rewrite revision IDs that have shipped to real installations.")
|
||||
print(" - Squash only unreleased development migrations.")
|
||||
print(" - Review generated baseline/upgrade migrations like normal schema code.")
|
||||
print(" - Run PostgreSQL migration smoke checks after any squash.")
|
||||
print()
|
||||
print("Current owner heads:")
|
||||
for owner in report["owners"]:
|
||||
heads = ", ".join(owner["heads"]) if owner["heads"] else "-"
|
||||
print(f" - {owner['owner']}: {heads}")
|
||||
print()
|
||||
print("Current Alembic graph heads:")
|
||||
for entry in report["current_head_entries"]:
|
||||
print(f" - {entry['owner']}: {entry['revision']}")
|
||||
if not report["current_head_entries"]:
|
||||
print(" -")
|
||||
print()
|
||||
print("Release steps:")
|
||||
print(" 1. Decide which migration revisions are unreleased and may be folded.")
|
||||
print(" 2. Replace those development chains with reviewed baseline/upgrade migrations.")
|
||||
print(" 3. Run scripts/release-migration-audit.py and PostgreSQL release checks.")
|
||||
print(" 4. Record the reviewed heads with scripts/release-migration-audit.py --record-release <x.y.z>.")
|
||||
print(" 5. Cut the release with scripts/push-release-tag.sh; audit is strict after the first baseline.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user