Release v0.1.6
This commit is contained in:
@@ -17,6 +17,11 @@ import sys
|
||||
|
||||
roots = [
|
||||
pathlib.Path("/mnt/DATA/git/govoplan-core/src"),
|
||||
pathlib.Path("/mnt/DATA/git/govoplan-access/src"),
|
||||
pathlib.Path("/mnt/DATA/git/govoplan-admin/src"),
|
||||
pathlib.Path("/mnt/DATA/git/govoplan-tenancy/src"),
|
||||
pathlib.Path("/mnt/DATA/git/govoplan-policy/src"),
|
||||
pathlib.Path("/mnt/DATA/git/govoplan-audit/src"),
|
||||
pathlib.Path("/mnt/DATA/git/govoplan-mail/src"),
|
||||
pathlib.Path("/mnt/DATA/git/govoplan-files/src"),
|
||||
pathlib.Path("/mnt/DATA/git/govoplan-campaign/src"),
|
||||
@@ -43,6 +48,7 @@ print(f"AST syntax check passed for {count} Python files")
|
||||
PY
|
||||
|
||||
./.venv/bin/python -c 'import govoplan_core.db.bootstrap; import govoplan_core.admin.service; import govoplan_files.backend.router; import govoplan_mail.backend.sending.imap; print("targeted backend imports passed")'
|
||||
./.venv/bin/python scripts/check_dependency_boundaries.py
|
||||
./.venv/bin/python -m unittest tests.test_module_system
|
||||
./.venv/bin/python -m unittest discover -s /mnt/DATA/git/govoplan-mail/tests
|
||||
./.venv/bin/python -m unittest tests.test_api_smoke.ApiSmokeTests.test_mailbox_message_listing_reports_total_count
|
||||
|
||||
137
scripts/check_dependency_boundaries.py
Normal file
137
scripts/check_dependency_boundaries.py
Normal file
@@ -0,0 +1,137 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
REPOS = {
|
||||
"core": ROOT / "src" / "govoplan_core",
|
||||
"access": ROOT.parent / "govoplan-access" / "src" / "govoplan_access",
|
||||
"admin": ROOT.parent / "govoplan-admin" / "src" / "govoplan_admin",
|
||||
"tenancy": ROOT.parent / "govoplan-tenancy" / "src" / "govoplan_tenancy",
|
||||
"policy": ROOT.parent / "govoplan-policy" / "src" / "govoplan_policy",
|
||||
"audit": ROOT.parent / "govoplan-audit" / "src" / "govoplan_audit",
|
||||
"files": ROOT.parent / "govoplan-files" / "src" / "govoplan_files",
|
||||
"mail": ROOT.parent / "govoplan-mail" / "src" / "govoplan_mail",
|
||||
"campaign": ROOT.parent / "govoplan-campaign" / "src" / "govoplan_campaign",
|
||||
}
|
||||
PREFIXES = {
|
||||
"core": "govoplan_core",
|
||||
"access": "govoplan_access",
|
||||
"admin": "govoplan_admin",
|
||||
"tenancy": "govoplan_tenancy",
|
||||
"policy": "govoplan_policy",
|
||||
"audit": "govoplan_audit",
|
||||
"files": "govoplan_files",
|
||||
"mail": "govoplan_mail",
|
||||
"campaign": "govoplan_campaign",
|
||||
}
|
||||
FEATURE_OWNERS = ("files", "mail", "campaign")
|
||||
PLATFORM_OWNERS = ("admin", "tenancy", "policy", "audit")
|
||||
PUBLIC_ACCESS_IMPORTS = ("govoplan_access.backend.auth.dependencies",)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AllowlistedImport:
|
||||
owner: str
|
||||
relative_path: str
|
||||
imported_owner: str
|
||||
reason: str
|
||||
|
||||
|
||||
# Transitional exceptions. This should remain empty; add entries only with a
|
||||
# dated removal plan and a capability/API contract issue.
|
||||
ALLOWLIST: tuple[AllowlistedImport, ...] = ()
|
||||
ALLOWLIST_KEYS = {(item.owner, item.relative_path, item.imported_owner) for item in ALLOWLIST}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Violation:
|
||||
owner: str
|
||||
path: Path
|
||||
lineno: int
|
||||
imported: str
|
||||
imported_owner: str
|
||||
|
||||
|
||||
def _import_owner(module_name: str) -> str | None:
|
||||
for owner, prefix in PREFIXES.items():
|
||||
if module_name == prefix or module_name.startswith(f"{prefix}."):
|
||||
return owner
|
||||
return None
|
||||
|
||||
|
||||
def _imports(path: Path) -> list[tuple[int, str]]:
|
||||
tree = ast.parse(path.read_text(), filename=str(path))
|
||||
result: list[tuple[int, str]] = []
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.Import):
|
||||
for alias in node.names:
|
||||
result.append((node.lineno, alias.name))
|
||||
elif isinstance(node, ast.ImportFrom) and node.module:
|
||||
result.append((node.lineno, node.module))
|
||||
return result
|
||||
|
||||
|
||||
def _relative(owner: str, path: Path) -> str:
|
||||
return path.relative_to(REPOS[owner]).as_posix()
|
||||
|
||||
|
||||
def _allowed(owner: str, path: Path, imported_owner: str) -> bool:
|
||||
return (owner, _relative(owner, path), imported_owner) in ALLOWLIST_KEYS
|
||||
|
||||
|
||||
def _is_public_access_import(imported: str) -> bool:
|
||||
return any(imported == item or imported.startswith(f"{item}.") for item in PUBLIC_ACCESS_IMPORTS)
|
||||
|
||||
|
||||
def _violations() -> list[Violation]:
|
||||
violations: list[Violation] = []
|
||||
for owner, root in REPOS.items():
|
||||
if not root.exists():
|
||||
continue
|
||||
for path in root.rglob("*.py"):
|
||||
if "__pycache__" in path.parts:
|
||||
continue
|
||||
for lineno, imported in _imports(path):
|
||||
imported_owner = _import_owner(imported)
|
||||
if imported_owner is None or imported_owner == owner:
|
||||
continue
|
||||
if imported_owner == "access" and _is_public_access_import(imported):
|
||||
continue
|
||||
if owner == "core" and imported_owner in FEATURE_OWNERS:
|
||||
if not _allowed(owner, path, imported_owner):
|
||||
violations.append(Violation(owner, path, lineno, imported, imported_owner))
|
||||
elif owner == "access" and imported_owner in FEATURE_OWNERS:
|
||||
violations.append(Violation(owner, path, lineno, imported, imported_owner))
|
||||
elif owner in PLATFORM_OWNERS and imported_owner == "access":
|
||||
if not _allowed(owner, path, imported_owner):
|
||||
violations.append(Violation(owner, path, lineno, imported, imported_owner))
|
||||
elif owner in FEATURE_OWNERS and imported_owner in FEATURE_OWNERS:
|
||||
if not _allowed(owner, path, imported_owner):
|
||||
violations.append(Violation(owner, path, lineno, imported, imported_owner))
|
||||
elif owner in FEATURE_OWNERS and imported_owner == "access":
|
||||
violations.append(Violation(owner, path, lineno, imported, imported_owner))
|
||||
return violations
|
||||
|
||||
|
||||
def main() -> int:
|
||||
violations = _violations()
|
||||
if not violations:
|
||||
print(f"Dependency boundary check passed ({len(ALLOWLIST)} transitional exceptions tracked)")
|
||||
return 0
|
||||
|
||||
print("Dependency boundary violations:")
|
||||
for item in violations:
|
||||
print(
|
||||
f"- {item.path}:{item.lineno}: {item.owner} imports {item.imported} "
|
||||
f"({item.imported_owner}); use kernel capability/API/event contracts instead"
|
||||
)
|
||||
print("\nIf this is unavoidable transitional debt, add a reasoned ALLOWLIST entry.")
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
83
scripts/generate-catalog-keypair.py
Normal file
83
scripts/generate-catalog-keypair.py
Normal file
@@ -0,0 +1,83 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate an Ed25519 keypair for signing GovOPlaN release catalogs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
from datetime import UTC, datetime
|
||||
import json
|
||||
from pathlib import Path
|
||||
import stat
|
||||
|
||||
from cryptography.hazmat.primitives import serialization
|
||||
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--key-id", required=True, help="Public key id recorded in catalog signatures.")
|
||||
parser.add_argument("--private-key", type=Path, required=True, help="Private PEM output path. Keep outside git.")
|
||||
parser.add_argument("--public-key", type=Path, help="Optional base64 public key output path.")
|
||||
parser.add_argument("--keyring", type=Path, help="Optional public keyring JSON output path.")
|
||||
parser.add_argument("--status", default="active", choices=("active", "next", "retired", "revoked", "disabled"))
|
||||
parser.add_argument("--force", action="store_true", help="Overwrite existing key files.")
|
||||
args = parser.parse_args()
|
||||
|
||||
private_path = args.private_key.expanduser()
|
||||
public_path = args.public_key.expanduser() if args.public_key else None
|
||||
keyring_path = args.keyring.expanduser() if args.keyring else None
|
||||
if private_path.exists() and not args.force:
|
||||
raise SystemExit(f"Private key already exists: {private_path}")
|
||||
if public_path is not None and public_path.exists() and not args.force:
|
||||
raise SystemExit(f"Public key already exists: {public_path}")
|
||||
|
||||
private_key = Ed25519PrivateKey.generate()
|
||||
private_bytes = private_key.private_bytes(
|
||||
encoding=serialization.Encoding.PEM,
|
||||
format=serialization.PrivateFormat.PKCS8,
|
||||
encryption_algorithm=serialization.NoEncryption(),
|
||||
)
|
||||
public_bytes = private_key.public_key().public_bytes(
|
||||
encoding=serialization.Encoding.Raw,
|
||||
format=serialization.PublicFormat.Raw,
|
||||
)
|
||||
public_base64 = base64.b64encode(public_bytes).decode("ascii")
|
||||
|
||||
private_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
private_path.write_bytes(private_bytes)
|
||||
private_path.chmod(stat.S_IRUSR | stat.S_IWUSR)
|
||||
|
||||
if public_path is not None:
|
||||
public_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
public_path.write_text(public_base64 + "\n", encoding="utf-8")
|
||||
|
||||
if keyring_path is not None:
|
||||
keyring_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
keyring = {
|
||||
"keyring_version": "1",
|
||||
"purpose": "govoplan module package catalog signatures",
|
||||
"generated_at": datetime.now(tz=UTC).isoformat().replace("+00:00", "Z"),
|
||||
"keys": [
|
||||
{
|
||||
"key_id": args.key_id,
|
||||
"status": args.status,
|
||||
"public_key": public_base64,
|
||||
"not_before": datetime.now(tz=UTC).date().isoformat() + "T00:00:00Z",
|
||||
}
|
||||
],
|
||||
}
|
||||
keyring_path.write_text(json.dumps(keyring, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
|
||||
print(f"private_key={private_path}")
|
||||
if public_path is not None:
|
||||
print(f"public_key={public_path}")
|
||||
if keyring_path is not None:
|
||||
print(f"keyring={keyring_path}")
|
||||
print(f"key_id={args.key_id}")
|
||||
print(f"public_key_base64={public_base64}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
284
scripts/generate-release-catalog.py
Normal file
284
scripts/generate-release-catalog.py
Normal file
@@ -0,0 +1,284 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate and sign a GovOPlaN module package release catalog."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime, timedelta
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from cryptography.hazmat.primitives import serialization
|
||||
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
|
||||
|
||||
|
||||
GITEA_BASE = "git+ssh://git@git.add-ideas.de/add-ideas"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CatalogModule:
|
||||
module_id: str
|
||||
repo: str
|
||||
python_package: str
|
||||
name: str
|
||||
description: str
|
||||
tags: tuple[str, ...]
|
||||
webui_package: str | None = None
|
||||
|
||||
|
||||
CATALOG_MODULES = (
|
||||
CatalogModule(
|
||||
module_id="tenancy",
|
||||
repo="govoplan-tenancy",
|
||||
python_package="govoplan-tenancy",
|
||||
name="Tenancy",
|
||||
description="Tenant registry, tenant settings, and tenant resolution platform module.",
|
||||
tags=("official", "platform-module"),
|
||||
),
|
||||
CatalogModule(
|
||||
module_id="access",
|
||||
repo="govoplan-access",
|
||||
python_package="govoplan-access",
|
||||
name="Access",
|
||||
description="Authentication, accounts, users, groups, roles, API keys, and access capabilities.",
|
||||
tags=("official", "platform-module"),
|
||||
webui_package="@govoplan/access-webui",
|
||||
),
|
||||
CatalogModule(
|
||||
module_id="admin",
|
||||
repo="govoplan-admin",
|
||||
python_package="govoplan-admin",
|
||||
name="Admin",
|
||||
description="System settings, governance templates, module management, and admin shell contributions.",
|
||||
tags=("official", "platform-module"),
|
||||
webui_package="@govoplan/admin-webui",
|
||||
),
|
||||
CatalogModule(
|
||||
module_id="policy",
|
||||
repo="govoplan-policy",
|
||||
python_package="govoplan-policy",
|
||||
name="Policy",
|
||||
description="Policy and governance capability module.",
|
||||
tags=("official", "platform-module"),
|
||||
),
|
||||
CatalogModule(
|
||||
module_id="audit",
|
||||
repo="govoplan-audit",
|
||||
python_package="govoplan-audit",
|
||||
name="Audit",
|
||||
description="Audit-log storage and audit administration routes.",
|
||||
tags=("official", "platform-module"),
|
||||
),
|
||||
CatalogModule(
|
||||
module_id="files",
|
||||
repo="govoplan-files",
|
||||
python_package="govoplan-files",
|
||||
name="Files",
|
||||
description="Managed file spaces and campaign attachment integration.",
|
||||
tags=("official", "service-module"),
|
||||
webui_package="@govoplan/files-webui",
|
||||
),
|
||||
CatalogModule(
|
||||
module_id="mail",
|
||||
repo="govoplan-mail",
|
||||
python_package="govoplan-mail",
|
||||
name="Mail",
|
||||
description="SMTP/IMAP profile management, credential policy, and read-only mailbox access.",
|
||||
tags=("official", "service-module"),
|
||||
webui_package="@govoplan/mail-webui",
|
||||
),
|
||||
CatalogModule(
|
||||
module_id="campaigns",
|
||||
repo="govoplan-campaign",
|
||||
python_package="govoplan-campaign",
|
||||
name="Campaigns",
|
||||
description="Campaign authoring, validation, queueing, delivery control, and reports.",
|
||||
tags=("official", "business-module"),
|
||||
webui_package="@govoplan/campaign-webui",
|
||||
),
|
||||
CatalogModule(
|
||||
module_id="calendar",
|
||||
repo="govoplan-calendar",
|
||||
python_package="govoplan-calendar",
|
||||
name="Calendar",
|
||||
description="Calendar collections, events, CalDAV sources, and calendar WebUI routes.",
|
||||
tags=("official", "service-module"),
|
||||
webui_package="@govoplan/calendar-webui",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--version", required=True, help="GovOPlaN release version, without leading v.")
|
||||
parser.add_argument("--channel", default="stable")
|
||||
parser.add_argument("--sequence", type=int, help="Monotonic channel sequence. Defaults to UTC timestamp.")
|
||||
parser.add_argument("--expires-days", type=int, default=90)
|
||||
parser.add_argument("--catalog-output", type=Path, required=True)
|
||||
parser.add_argument("--keyring-output", type=Path)
|
||||
parser.add_argument(
|
||||
"--catalog-signing-key",
|
||||
action="append",
|
||||
default=[],
|
||||
metavar="KEY_ID=PRIVATE_KEY",
|
||||
help="Ed25519 private key used to sign the catalog; may be repeated for rotation.",
|
||||
)
|
||||
parser.add_argument("--public-base-url", default="https://govoplan.add-ideas.de")
|
||||
parser.add_argument("--repository-base", default=GITEA_BASE)
|
||||
args = parser.parse_args()
|
||||
|
||||
version = args.version.removeprefix("v")
|
||||
tag = f"v{version}"
|
||||
generated_at = datetime.now(tz=UTC)
|
||||
sequence = args.sequence if args.sequence is not None else int(generated_at.strftime("%Y%m%d%H%M"))
|
||||
expires_at = generated_at + timedelta(days=args.expires_days)
|
||||
signing_keys = [_parse_signing_key(value) for value in args.catalog_signing_key]
|
||||
|
||||
catalog = _catalog_payload(
|
||||
version=version,
|
||||
tag=tag,
|
||||
channel=args.channel,
|
||||
sequence=sequence,
|
||||
generated_at=generated_at,
|
||||
expires_at=expires_at,
|
||||
repository_base=args.repository_base.rstrip("/"),
|
||||
public_base_url=args.public_base_url.rstrip("/"),
|
||||
)
|
||||
if signing_keys:
|
||||
catalog["signatures"] = [_signature(catalog, key_id=key_id, private_key=private_key) for key_id, private_key in signing_keys]
|
||||
|
||||
output = args.catalog_output.expanduser()
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
output.write_text(json.dumps(catalog, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
|
||||
if args.keyring_output is not None:
|
||||
keyring_output = args.keyring_output.expanduser()
|
||||
keyring_output.parent.mkdir(parents=True, exist_ok=True)
|
||||
keyring_output.write_text(
|
||||
json.dumps(_keyring(signing_keys=signing_keys, generated_at=generated_at), indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
print(f"catalog={output}")
|
||||
if args.keyring_output is not None:
|
||||
print(f"keyring={args.keyring_output.expanduser()}")
|
||||
print(f"channel={args.channel}")
|
||||
print(f"sequence={sequence}")
|
||||
print(f"version={version}")
|
||||
return 0
|
||||
|
||||
|
||||
def _catalog_payload(
|
||||
*,
|
||||
version: str,
|
||||
tag: str,
|
||||
channel: str,
|
||||
sequence: int,
|
||||
generated_at: datetime,
|
||||
expires_at: datetime,
|
||||
repository_base: str,
|
||||
public_base_url: str,
|
||||
) -> dict[str, Any]:
|
||||
modules: list[dict[str, Any]] = []
|
||||
for module in CATALOG_MODULES:
|
||||
entry: dict[str, Any] = {
|
||||
"module_id": module.module_id,
|
||||
"name": module.name,
|
||||
"description": module.description,
|
||||
"version": version,
|
||||
"action": "install",
|
||||
"python_package": module.python_package,
|
||||
"python_ref": f"{module.python_package} @ {repository_base}/{module.repo}.git@{tag}",
|
||||
"license_features": [f"module.{module.module_id}"],
|
||||
"tags": list(module.tags),
|
||||
}
|
||||
if module.webui_package:
|
||||
entry["webui_package"] = module.webui_package
|
||||
entry["webui_ref"] = f"{repository_base}/{module.repo}.git#{tag}"
|
||||
modules.append(entry)
|
||||
|
||||
return {
|
||||
"catalog_version": "1",
|
||||
"channel": channel,
|
||||
"sequence": sequence,
|
||||
"generated_at": _json_datetime(generated_at),
|
||||
"expires_at": _json_datetime(expires_at),
|
||||
"release": {
|
||||
"version": version,
|
||||
"tag": tag,
|
||||
"catalog_url": f"{public_base_url}/catalogs/v1/channels/{channel}.json",
|
||||
"keyring_url": f"{public_base_url}/catalogs/v1/keyring.json",
|
||||
},
|
||||
"core_release": {
|
||||
"name": "GovOPlaN Core",
|
||||
"version": version,
|
||||
"python_package": "govoplan-core",
|
||||
"python_ref": f"govoplan-core[server] @ {repository_base}/govoplan-core.git@{tag}",
|
||||
"webui_package": "@govoplan/core-webui",
|
||||
"webui_ref": f"{repository_base}/govoplan-core.git#{tag}",
|
||||
},
|
||||
"modules": modules,
|
||||
}
|
||||
|
||||
|
||||
def _parse_signing_key(value: str) -> tuple[str, Ed25519PrivateKey]:
|
||||
key_id, separator, path_text = value.partition("=")
|
||||
if not separator or not key_id.strip() or not path_text.strip():
|
||||
raise SystemExit("--catalog-signing-key must use KEY_ID=/path/to/private.pem")
|
||||
path = Path(path_text).expanduser()
|
||||
private_key = serialization.load_pem_private_key(path.read_bytes(), password=None)
|
||||
if not isinstance(private_key, Ed25519PrivateKey):
|
||||
raise SystemExit(f"Catalog signing key must be an Ed25519 private key: {path}")
|
||||
return key_id.strip(), private_key
|
||||
|
||||
|
||||
def _signature(payload: dict[str, Any], *, key_id: str, private_key: Ed25519PrivateKey) -> dict[str, str]:
|
||||
signature_payload = dict(payload)
|
||||
signature_payload.pop("signature", None)
|
||||
signature_payload.pop("signatures", None)
|
||||
signature = private_key.sign(_canonical_bytes(signature_payload))
|
||||
return {
|
||||
"algorithm": "ed25519",
|
||||
"key_id": key_id,
|
||||
"value": base64.b64encode(signature).decode("ascii"),
|
||||
}
|
||||
|
||||
|
||||
def _keyring(*, signing_keys: list[tuple[str, Ed25519PrivateKey]], generated_at: datetime) -> dict[str, Any]:
|
||||
return {
|
||||
"keyring_version": "1",
|
||||
"purpose": "govoplan module package catalog signatures",
|
||||
"generated_at": _json_datetime(generated_at),
|
||||
"keys": [
|
||||
{
|
||||
"key_id": key_id,
|
||||
"status": "active",
|
||||
"public_key": _public_key_base64(private_key),
|
||||
"not_before": generated_at.date().isoformat() + "T00:00:00Z",
|
||||
}
|
||||
for key_id, private_key in signing_keys
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _public_key_base64(private_key: Ed25519PrivateKey) -> str:
|
||||
public_bytes = private_key.public_key().public_bytes(
|
||||
encoding=serialization.Encoding.Raw,
|
||||
format=serialization.PublicFormat.Raw,
|
||||
)
|
||||
return base64.b64encode(public_bytes).decode("ascii")
|
||||
|
||||
|
||||
def _canonical_bytes(payload: object) -> bytes:
|
||||
return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
|
||||
|
||||
|
||||
def _json_datetime(value: datetime) -> str:
|
||||
return value.astimezone(UTC).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
582
scripts/gitea-backlog-import.py
Normal file
582
scripts/gitea-backlog-import.py
Normal file
@@ -0,0 +1,582 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Import markdown backlog/planning files into Gitea issues."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import dataclasses
|
||||
import hashlib
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import sys
|
||||
from typing import Any, Iterable
|
||||
|
||||
from gitea_common import GiteaClient, GiteaError, infer_target, load_dotenv, repo_path, repo_root, require_token
|
||||
|
||||
|
||||
DEFAULT_PRODUCT_BACKLOG = pathlib.Path("/mnt/DATA/Nextcloud/ADD ideas UG/Products/govoplan/backlog.md")
|
||||
DEFAULT_ACCESS_PLAN = pathlib.Path("/mnt/DATA/git/govoplan-core/docs/ACCESS_EXTRACTION_PLAN.md")
|
||||
DEFAULT_WEB_TODO = pathlib.Path("/mnt/DATA/git/govoplan-web/TODO.md")
|
||||
|
||||
REPO_ROOTS = {
|
||||
"core": pathlib.Path("/mnt/DATA/git/govoplan-core"),
|
||||
"access": pathlib.Path("/mnt/DATA/git/govoplan-access"),
|
||||
"mail": pathlib.Path("/mnt/DATA/git/govoplan-mail"),
|
||||
"files": pathlib.Path("/mnt/DATA/git/govoplan-files"),
|
||||
"campaign": pathlib.Path("/mnt/DATA/git/govoplan-campaign"),
|
||||
"web": pathlib.Path("/mnt/DATA/git/govoplan-web"),
|
||||
}
|
||||
|
||||
MODULE_LABELS = {
|
||||
"core": "module/core",
|
||||
"access": "module/access",
|
||||
"mail": "module/mail",
|
||||
"files": "module/files",
|
||||
"campaign": "module/campaign",
|
||||
"web": "module/web",
|
||||
}
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class Candidate:
|
||||
repo_key: str
|
||||
title: str
|
||||
body: str
|
||||
labels: tuple[str, ...]
|
||||
fingerprint: str
|
||||
source: str
|
||||
line: int
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class RepoState:
|
||||
target: Any
|
||||
client: GiteaClient
|
||||
label_ids: dict[str, int]
|
||||
fingerprints: set[str]
|
||||
titles: set[str]
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--env-file", type=pathlib.Path, help="dotenv file to read before using GITEA_* values")
|
||||
parser.add_argument("--product-backlog", type=pathlib.Path, default=DEFAULT_PRODUCT_BACKLOG)
|
||||
parser.add_argument("--access-plan", type=pathlib.Path, default=DEFAULT_ACCESS_PLAN)
|
||||
parser.add_argument("--web-todo", type=pathlib.Path, default=DEFAULT_WEB_TODO)
|
||||
parser.add_argument("--apply", action="store_true", help="create missing Gitea issues")
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
load_dotenv(args.env_file)
|
||||
candidates = list(build_candidates(args))
|
||||
candidates.sort(key=lambda item: (item.repo_key, item.source, item.line, item.title))
|
||||
|
||||
print(f"Prepared {len(candidates)} backlog issue candidate(s).")
|
||||
for repo_key, grouped in group_candidates(candidates).items():
|
||||
print(f"{repo_key}: {len(grouped)} candidate(s)")
|
||||
|
||||
token = require_token() if args.apply else os.environ.get("GITEA_TOKEN")
|
||||
if not token:
|
||||
print("Dry run without GITEA_TOKEN: duplicate comparison skipped.")
|
||||
print_preview(candidates)
|
||||
return 0
|
||||
|
||||
states = load_repo_states(token, sorted({candidate.repo_key for candidate in candidates}))
|
||||
missing = [candidate for candidate in candidates if not already_imported(candidate, states[candidate.repo_key])]
|
||||
skipped = len(candidates) - len(missing)
|
||||
|
||||
print(f"Skipped {skipped} existing issue candidate(s).")
|
||||
print(f"Missing {len(missing)} issue candidate(s).")
|
||||
print_preview(missing)
|
||||
|
||||
if not args.apply:
|
||||
print("Dry run only. Re-run with --apply to create missing issues.")
|
||||
return 0
|
||||
|
||||
created_by_repo: dict[str, list[str]] = {}
|
||||
for candidate in missing:
|
||||
state = states[candidate.repo_key]
|
||||
validate_labels(candidate, state)
|
||||
issue = state.client.request_json(
|
||||
"POST",
|
||||
repo_path(state.target.owner, state.target.repo, "/issues"),
|
||||
body={
|
||||
"title": candidate.title,
|
||||
"body": candidate.body,
|
||||
"labels": [state.label_ids[label] for label in candidate.labels],
|
||||
},
|
||||
)
|
||||
number = issue.get("number") or issue.get("index")
|
||||
created_by_repo.setdefault(candidate.repo_key, []).append(f"#{number} {candidate.title}")
|
||||
state.fingerprints.add(candidate.fingerprint)
|
||||
state.titles.add(normalize_title(candidate.title))
|
||||
|
||||
print("Created issues:")
|
||||
for repo_key, titles in created_by_repo.items():
|
||||
print(f"{repo_key}:")
|
||||
for title in titles:
|
||||
print(f" {title}")
|
||||
if not created_by_repo:
|
||||
print(" none")
|
||||
return 0
|
||||
except GiteaError as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
def build_candidates(args: argparse.Namespace) -> Iterable[Candidate]:
|
||||
if args.product_backlog.exists():
|
||||
yield from parse_product_backlog(args.product_backlog)
|
||||
if args.access_plan.exists():
|
||||
yield from parse_access_plan(args.access_plan)
|
||||
if args.web_todo.exists():
|
||||
yield from parse_simple_todo(args.web_todo)
|
||||
|
||||
|
||||
def parse_product_backlog(path: pathlib.Path) -> Iterable[Candidate]:
|
||||
headings: list[tuple[int, str]] = []
|
||||
status = ""
|
||||
lines = path.read_text(encoding="utf-8").splitlines()
|
||||
for index, line in enumerate(lines):
|
||||
line_number = index + 1
|
||||
heading_match = re.match(r"^(#{2,4})\s+(.+?)\s*$", line)
|
||||
if heading_match:
|
||||
level = len(heading_match.group(1))
|
||||
title = heading_match.group(2).strip()
|
||||
if level == 2 and title == "Completed Work Archive":
|
||||
break
|
||||
headings = [(h_level, h_title) for h_level, h_title in headings if h_level < level]
|
||||
headings.append((level, title))
|
||||
status = ""
|
||||
continue
|
||||
|
||||
status_match = re.match(r"^\*\*Status:\*\*\s*(.+?)\s*$", line)
|
||||
if status_match:
|
||||
status = status_match.group(1).strip()
|
||||
continue
|
||||
|
||||
item_match = re.match(r"^\s*[-*]\s+\[\s\]\s+(.+?)\s*$", line)
|
||||
if not item_match:
|
||||
continue
|
||||
|
||||
item = collect_continued_item(lines, index, item_match.group(1))
|
||||
section = section_title(headings)
|
||||
repo_key, labels = classify_product_item(item, section)
|
||||
prefix = type_prefix(labels)
|
||||
title = format_title(prefix, item)
|
||||
source = str(path)
|
||||
body = body_for_item(
|
||||
fingerprint=make_fingerprint(source, line_number, repo_key, item),
|
||||
source=source,
|
||||
line=line_number,
|
||||
section=section,
|
||||
status=status,
|
||||
item=item,
|
||||
note="Imported from the consolidated GovOPlaN product backlog.",
|
||||
)
|
||||
yield Candidate(
|
||||
repo_key=repo_key,
|
||||
title=title,
|
||||
body=body,
|
||||
labels=tuple(sorted(set(labels))),
|
||||
fingerprint=make_fingerprint(source, line_number, repo_key, item),
|
||||
source=source,
|
||||
line=line_number,
|
||||
)
|
||||
|
||||
|
||||
def parse_access_plan(path: pathlib.Path) -> Iterable[Candidate]:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
lines = text.splitlines()
|
||||
yield from access_stage_candidates(path, lines)
|
||||
yield from access_immediate_candidates(path, lines)
|
||||
yield from access_decision_candidates(path, lines)
|
||||
|
||||
|
||||
def access_stage_candidates(path: pathlib.Path, lines: list[str]) -> Iterable[Candidate]:
|
||||
index = 0
|
||||
while index < len(lines):
|
||||
match = re.match(r"^### Stage ([1-7]):\s*(.+?)\s*$", lines[index])
|
||||
if not match:
|
||||
index += 1
|
||||
continue
|
||||
stage = match.group(1)
|
||||
name = match.group(2).strip()
|
||||
start_line = index + 1
|
||||
index += 1
|
||||
block: list[str] = []
|
||||
while index < len(lines) and not re.match(r"^### Stage \d+:", lines[index]) and not re.match(r"^## ", lines[index]):
|
||||
block.append(lines[index])
|
||||
index += 1
|
||||
item = f"Access extraction Stage {stage}: {name}"
|
||||
fingerprint = make_fingerprint(str(path), start_line, "core", item)
|
||||
body = "\n".join(
|
||||
[
|
||||
f"<!-- codex-backlog-fingerprint:{fingerprint} -->",
|
||||
"",
|
||||
"Imported from the GovOPlaN access extraction plan.",
|
||||
"",
|
||||
f"- Source: `{path}`",
|
||||
f"- Line: `{start_line}`",
|
||||
f"- Section: `Stage {stage}: {name}`",
|
||||
"",
|
||||
"Plan excerpt:",
|
||||
"",
|
||||
"```markdown",
|
||||
f"### Stage {stage}: {name}",
|
||||
*block,
|
||||
"```",
|
||||
]
|
||||
)
|
||||
yield Candidate(
|
||||
repo_key="core",
|
||||
title=format_title("[Task]", item),
|
||||
body=body,
|
||||
labels=labels("core", "type/task", "area/module-system", "module/access"),
|
||||
fingerprint=fingerprint,
|
||||
source=str(path),
|
||||
line=start_line,
|
||||
)
|
||||
|
||||
|
||||
def access_immediate_candidates(path: pathlib.Path, lines: list[str]) -> Iterable[Candidate]:
|
||||
in_section = False
|
||||
for index, line in enumerate(lines):
|
||||
line_number = index + 1
|
||||
if line == "## Immediate Backlog":
|
||||
in_section = True
|
||||
continue
|
||||
if in_section and line.startswith("## "):
|
||||
break
|
||||
if not in_section:
|
||||
continue
|
||||
item_match = re.match(r"^\s*[-*]\s+\[\s\]\s+(.+?)\s*$", line)
|
||||
if not item_match:
|
||||
continue
|
||||
item = collect_continued_item(lines, index, item_match.group(1))
|
||||
title_item = access_title_item(item)
|
||||
fingerprint = make_fingerprint(str(path), line_number, "core", item)
|
||||
body = body_for_item(
|
||||
fingerprint=fingerprint,
|
||||
source=str(path),
|
||||
line=line_number,
|
||||
section="Immediate Backlog",
|
||||
status="open",
|
||||
item=item,
|
||||
note="Imported from the GovOPlaN access extraction plan immediate backlog.",
|
||||
)
|
||||
yield Candidate(
|
||||
repo_key="core",
|
||||
title=format_title("[Task]", title_item),
|
||||
body=body,
|
||||
labels=labels("core", "type/task", "area/auth", "area/module-system", "module/access"),
|
||||
fingerprint=fingerprint,
|
||||
source=str(path),
|
||||
line=line_number,
|
||||
)
|
||||
|
||||
|
||||
def access_decision_candidates(path: pathlib.Path, lines: list[str]) -> Iterable[Candidate]:
|
||||
in_section = False
|
||||
for index, line in enumerate(lines):
|
||||
line_number = index + 1
|
||||
if line == "## Open Decisions":
|
||||
in_section = True
|
||||
continue
|
||||
if in_section and line.startswith("## "):
|
||||
break
|
||||
if not in_section:
|
||||
continue
|
||||
item_match = re.match(r"^\s*[-*]\s+(.+?)\s*$", line)
|
||||
if not item_match:
|
||||
continue
|
||||
item = collect_continued_item(lines, index, item_match.group(1))
|
||||
fingerprint = make_fingerprint(str(path), line_number, "core", item)
|
||||
body = body_for_item(
|
||||
fingerprint=fingerprint,
|
||||
source=str(path),
|
||||
line=line_number,
|
||||
section="Open Decisions",
|
||||
status="decision needed",
|
||||
item=item,
|
||||
note="Imported from the GovOPlaN access extraction plan open decisions.",
|
||||
)
|
||||
yield Candidate(
|
||||
repo_key="core",
|
||||
title=format_title("[Task]", f"Decide: {item}"),
|
||||
body=body,
|
||||
labels=labels("core", "type/task", "status/needs-info", "codex/needs-human", "area/auth", "module/access"),
|
||||
fingerprint=fingerprint,
|
||||
source=str(path),
|
||||
line=line_number,
|
||||
)
|
||||
|
||||
|
||||
def parse_simple_todo(path: pathlib.Path) -> Iterable[Candidate]:
|
||||
lines = path.read_text(encoding="utf-8").splitlines()
|
||||
for index, line in enumerate(lines):
|
||||
line_number = index + 1
|
||||
item_match = re.match(r"^\s*[-*]\s+(.+?)\s*$", line)
|
||||
if not item_match:
|
||||
continue
|
||||
item = collect_continued_item(lines, index, item_match.group(1))
|
||||
fingerprint = make_fingerprint(str(path), line_number, "web", item)
|
||||
yield Candidate(
|
||||
repo_key="web",
|
||||
title=format_title("[Task]", item),
|
||||
body=body_for_item(
|
||||
fingerprint=fingerprint,
|
||||
source=str(path),
|
||||
line=line_number,
|
||||
section="TODO",
|
||||
status="open",
|
||||
item=item,
|
||||
note="Imported from the GovOPlaN website TODO file.",
|
||||
),
|
||||
labels=labels("web", "type/task", "area/marketing"),
|
||||
fingerprint=fingerprint,
|
||||
source=str(path),
|
||||
line=line_number,
|
||||
)
|
||||
|
||||
|
||||
def classify_product_item(item: str, section: str) -> tuple[str, tuple[str, ...]]:
|
||||
text = f"{section} {item}".lower()
|
||||
repo_key = "core"
|
||||
area = "area/devex"
|
||||
issue_type = "type/task"
|
||||
|
||||
if "dsar search/export/delete" in text:
|
||||
return "core", labels("core", "type/feature", "area/governance")
|
||||
if "fully stream zip uploads" in text:
|
||||
return "files", labels("files", "type/task", "area/api")
|
||||
if "installer packaging approach" in text:
|
||||
return "core", labels("core", "type/task", "area/devex")
|
||||
if "profile reselection/revalidation" in text:
|
||||
return "campaign", labels("campaign", "type/task", "area/api", "module/mail")
|
||||
if "module boundaries" in text:
|
||||
return "core", labels("core", issue_type, "area/module-system")
|
||||
if "external storage connectors" in text:
|
||||
repo_key = "files"
|
||||
area = "area/api"
|
||||
elif any(phrase in text for phrase in ("report, evidence", "recipient import", "campaign and attachment", "collaboration and advanced governance", "address book, templates")):
|
||||
repo_key = "campaign"
|
||||
area = "area/webui" if any(word in text for word in ("page", "table", "display", "ui", "wizard", "editor", "flow", "polish")) else "area/api"
|
||||
elif "controlled sending" in text:
|
||||
repo_key = "campaign"
|
||||
area = "area/api"
|
||||
elif "mail profiles" in text:
|
||||
repo_key = "mail"
|
||||
area = "area/api"
|
||||
|
||||
if any(word in text for word in ("smtp", "imap", "mailbox", "mail profile", "deliverability", "bounce", "openpgp", "s/mime", "pop3", "jmap")):
|
||||
repo_key = "mail"
|
||||
area = "area/api"
|
||||
if any(word in text for word in ("seafile", "nextcloud", "webdav", "smb", "connector", "file rename", "managed storage", "live remote file")):
|
||||
repo_key = "files"
|
||||
area = "area/api"
|
||||
if "external storage connectors" not in text and any(
|
||||
word in text
|
||||
for word in (
|
||||
"campaign",
|
||||
"recipient",
|
||||
"attachment",
|
||||
"zip",
|
||||
"evidence",
|
||||
"wizard",
|
||||
"send",
|
||||
"archive",
|
||||
"message preview",
|
||||
"address book",
|
||||
"carddav",
|
||||
"mailing lists",
|
||||
)
|
||||
):
|
||||
repo_key = "campaign"
|
||||
area = "area/webui" if any(word in text for word in ("page", "table", "display", "ui", "wizard", "editor", "flow", "polish")) else "area/api"
|
||||
if repo_key == "core" and any(word in text for word in ("session", "device", "auth", "rbac", "role", "tenant", "governance", "policy", "retention", "dsar", "audit", "encryption", "ldap", "oidc", "saml")):
|
||||
repo_key = "core"
|
||||
area = "area/auth" if any(word in text for word in ("session", "auth", "ldap", "oidc", "saml")) else "area/governance"
|
||||
if repo_key == "core" and any(word in text for word in ("module", "manifest", "module boundaries", "installable", "activatable", "deactivatable", "uninstallable")):
|
||||
repo_key = "core"
|
||||
area = "area/module-system"
|
||||
if repo_key == "core" and any(word in text for word in ("release", "version metadata", "lockfile", "package", "tagging")):
|
||||
repo_key = "core"
|
||||
area = "area/release"
|
||||
if repo_key == "core" and not any(
|
||||
phrase in text
|
||||
for phrase in (
|
||||
"external storage connectors",
|
||||
"controlled sending",
|
||||
"report, evidence",
|
||||
"recipient import",
|
||||
"campaign and attachment",
|
||||
)
|
||||
) and any(word in text for word in ("postgres", "redis", "celery", "worker", "deployment", "installer", "backup", "monitoring", "configuration", "dev profile", ".env")):
|
||||
repo_key = "core"
|
||||
area = "area/devex"
|
||||
if any(word in text for word in ("documentation", "handbook", "docs", "runbook")):
|
||||
area = "area/docs"
|
||||
if "ui lists often start at index 2" in text:
|
||||
issue_type = "type/bug"
|
||||
area = "area/webui"
|
||||
repo_key = "core"
|
||||
if any(word in text for word in ("add ", "implement ", "build ", "provide ", "allow ", "make ", "extend ", "replace ")):
|
||||
issue_type = "type/feature"
|
||||
if any(word in text for word in ("cleanup", "remove legacy", "fragile", "deferred cleanup")):
|
||||
issue_type = "type/debt"
|
||||
|
||||
return repo_key, labels(repo_key, issue_type, area)
|
||||
|
||||
|
||||
def collect_continued_item(lines: list[str], index: int, initial: str) -> str:
|
||||
parts = [initial.strip()]
|
||||
next_index = index + 1
|
||||
while next_index < len(lines):
|
||||
line = lines[next_index]
|
||||
if not line.startswith((" ", "\t")):
|
||||
break
|
||||
stripped = line.strip()
|
||||
if not stripped:
|
||||
break
|
||||
if re.match(r"^[-*]\s+", stripped) or stripped.startswith("#"):
|
||||
break
|
||||
parts.append(stripped)
|
||||
next_index += 1
|
||||
return " ".join(parts)
|
||||
|
||||
|
||||
def labels(repo_key: str, *extra: str) -> tuple[str, ...]:
|
||||
base = {"status/triage", "source/backlog-import", "codex/ready", MODULE_LABELS[repo_key]}
|
||||
base.update(extra)
|
||||
if "status/needs-info" in base:
|
||||
base.discard("status/triage")
|
||||
return tuple(sorted(base))
|
||||
|
||||
|
||||
def type_prefix(label_set: Iterable[str]) -> str:
|
||||
labels_set = set(label_set)
|
||||
if "type/bug" in labels_set:
|
||||
return "[Bug]"
|
||||
if "type/feature" in labels_set:
|
||||
return "[Feature]"
|
||||
if "type/debt" in labels_set:
|
||||
return "[Debt]"
|
||||
if "type/docs" in labels_set:
|
||||
return "[Docs]"
|
||||
return "[Task]"
|
||||
|
||||
|
||||
def format_title(prefix: str, item: str) -> str:
|
||||
clean = re.sub(r"\s+", " ", item).strip().rstrip(".")
|
||||
title = f"{prefix} {clean}"
|
||||
if len(title) <= 180:
|
||||
return title
|
||||
return f"{title[:177].rstrip()}..."
|
||||
|
||||
|
||||
def access_title_item(item: str) -> str:
|
||||
lowered = item.lower()
|
||||
if "compatibility wrappers" in lowered:
|
||||
return "Wire core auth compatibility through access capabilities"
|
||||
return item
|
||||
|
||||
|
||||
def body_for_item(
|
||||
*,
|
||||
fingerprint: str,
|
||||
source: str,
|
||||
line: int,
|
||||
section: str,
|
||||
status: str,
|
||||
item: str,
|
||||
note: str,
|
||||
) -> str:
|
||||
lines = [
|
||||
f"<!-- codex-backlog-fingerprint:{fingerprint} -->",
|
||||
"",
|
||||
note,
|
||||
"",
|
||||
f"- Source: `{source}`",
|
||||
f"- Line: `{line}`",
|
||||
f"- Section: `{section}`",
|
||||
]
|
||||
if status:
|
||||
lines.append(f"- Source status: `{status}`")
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
"Imported backlog item:",
|
||||
"",
|
||||
"```markdown",
|
||||
f"- [ ] {item}",
|
||||
"```",
|
||||
]
|
||||
)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def section_title(headings: list[tuple[int, str]]) -> str:
|
||||
return " > ".join(title for _, title in headings)
|
||||
|
||||
|
||||
def make_fingerprint(source: str, line: int, repo_key: str, item: str) -> str:
|
||||
stable = "\n".join([source, str(line), repo_key, item])
|
||||
return hashlib.sha256(stable.encode("utf-8")).hexdigest()[:24]
|
||||
|
||||
|
||||
def group_candidates(candidates: list[Candidate]) -> dict[str, list[Candidate]]:
|
||||
grouped: dict[str, list[Candidate]] = {}
|
||||
for candidate in candidates:
|
||||
grouped.setdefault(candidate.repo_key, []).append(candidate)
|
||||
return grouped
|
||||
|
||||
|
||||
def load_repo_states(token: str, repo_keys: list[str]) -> dict[str, RepoState]:
|
||||
states: dict[str, RepoState] = {}
|
||||
for repo_key in repo_keys:
|
||||
root = repo_root(REPO_ROOTS[repo_key])
|
||||
target = infer_target(root)
|
||||
client = GiteaClient(target, token)
|
||||
labels_payload = client.paginate(repo_path(target.owner, target.repo, "/labels"))
|
||||
label_ids = {str(label["name"]): int(label["id"]) for label in labels_payload if "name" in label and "id" in label}
|
||||
issues = client.paginate(repo_path(target.owner, target.repo, "/issues"), query={"state": "all"}, limit=100)
|
||||
fingerprints: set[str] = set()
|
||||
titles: set[str] = set()
|
||||
for issue in issues:
|
||||
body = str(issue.get("body") or "")
|
||||
if issue.get("state") == "closed" and "Moved to `" in body:
|
||||
continue
|
||||
titles.add(normalize_title(str(issue.get("title") or "")))
|
||||
fingerprints.update(re.findall(r"codex-backlog-fingerprint:([0-9a-f]{24})", body))
|
||||
states[repo_key] = RepoState(target=target, client=client, label_ids=label_ids, fingerprints=fingerprints, titles=titles)
|
||||
return states
|
||||
|
||||
|
||||
def already_imported(candidate: Candidate, state: RepoState) -> bool:
|
||||
return candidate.fingerprint in state.fingerprints or normalize_title(candidate.title) in state.titles
|
||||
|
||||
|
||||
def normalize_title(title: str) -> str:
|
||||
return re.sub(r"[^a-z0-9]+", " ", title.lower()).strip()
|
||||
|
||||
|
||||
def validate_labels(candidate: Candidate, state: RepoState) -> None:
|
||||
missing = sorted(label for label in candidate.labels if label not in state.label_ids)
|
||||
if missing:
|
||||
joined = ", ".join(missing)
|
||||
raise GiteaError(f"{candidate.repo_key} is missing labels for {candidate.title!r}: {joined}")
|
||||
|
||||
|
||||
def print_preview(candidates: list[Candidate], limit: int = 160) -> None:
|
||||
for repo_key, grouped in group_candidates(candidates).items():
|
||||
print(f"{repo_key}:")
|
||||
for candidate in grouped[:limit]:
|
||||
print(f" {candidate.title}")
|
||||
if len(grouped) > limit:
|
||||
print(f" ... {len(grouped) - limit} more")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
97
scripts/gitea-codex-note.py
Normal file
97
scripts/gitea-codex-note.py
Normal file
@@ -0,0 +1,97 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Post a standardized Codex state update comment to a Gitea issue."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
from gitea_common import GiteaClient, GiteaError, infer_target, load_dotenv, repo_path, repo_root, require_token
|
||||
|
||||
|
||||
STATUS_LABELS = {
|
||||
"started": "status/in-progress",
|
||||
"progress": "status/in-progress",
|
||||
"blocked": "status/blocked",
|
||||
"needs-info": "status/needs-info",
|
||||
"ready": "status/ready",
|
||||
"done": "",
|
||||
"note": "",
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--root", type=pathlib.Path, default=pathlib.Path.cwd(), help="repository root or child path")
|
||||
parser.add_argument("--remote", default="origin", help="git remote to use for target inference")
|
||||
parser.add_argument("--env-file", type=pathlib.Path, help="dotenv file to read before using GITEA_* values")
|
||||
parser.add_argument("--issue", type=int, required=True, help="Gitea issue number")
|
||||
parser.add_argument("--status", choices=sorted(STATUS_LABELS), default="note")
|
||||
parser.add_argument("--summary", action="append", default=[], help="summary bullet; may be repeated")
|
||||
parser.add_argument("--changed", action="append", default=[], help="changed file path; may be repeated")
|
||||
parser.add_argument("--test", action="append", default=[], help="verification command/result; may be repeated")
|
||||
parser.add_argument("--next", action="append", default=[], help="next step or blocker; may be repeated")
|
||||
parser.add_argument("--body-file", type=pathlib.Path, help="additional Markdown body to append")
|
||||
parser.add_argument("--close", action="store_true", help="close the issue after posting the note")
|
||||
parser.add_argument("--apply", action="store_true", help="post the comment and optional state update")
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
root = repo_root(args.root)
|
||||
load_dotenv(args.env_file or root / ".env")
|
||||
target = infer_target(root, args.remote)
|
||||
token = require_token() if args.apply else os.environ.get("GITEA_TOKEN")
|
||||
body = build_body(args)
|
||||
|
||||
print(f"Target: {target.display}")
|
||||
print(f"Issue: #{args.issue}")
|
||||
print(body)
|
||||
|
||||
if not args.apply:
|
||||
print("Dry run only. Re-run with --apply and GITEA_TOKEN to post.")
|
||||
return 0
|
||||
|
||||
client = GiteaClient(target, token)
|
||||
client.request_json(
|
||||
"POST",
|
||||
repo_path(target.owner, target.repo, f"/issues/{args.issue}/comments"),
|
||||
body={"body": body},
|
||||
)
|
||||
if args.close:
|
||||
client.request_json(
|
||||
"PATCH",
|
||||
repo_path(target.owner, target.repo, f"/issues/{args.issue}"),
|
||||
body={"state": "closed"},
|
||||
)
|
||||
print("Posted Codex note.")
|
||||
return 0
|
||||
except GiteaError as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
def build_body(args: argparse.Namespace) -> str:
|
||||
lines = [f"## Codex State: {args.status}", ""]
|
||||
append_section(lines, "Summary", args.summary)
|
||||
append_section(lines, "Changed Files", [f"`{item}`" for item in args.changed])
|
||||
append_section(lines, "Verification", [f"`{item}`" for item in args.test])
|
||||
append_section(lines, "Next / Blocked", args.next)
|
||||
if STATUS_LABELS[args.status]:
|
||||
lines.extend(["", f"Suggested status label: `{STATUS_LABELS[args.status]}`"])
|
||||
if args.body_file:
|
||||
lines.extend(["", args.body_file.read_text(encoding="utf-8").strip()])
|
||||
return "\n".join(lines).rstrip() + "\n"
|
||||
|
||||
|
||||
def append_section(lines: list[str], title: str, items: list[str]) -> None:
|
||||
if not items:
|
||||
return
|
||||
lines.extend([f"### {title}", ""])
|
||||
lines.extend(f"- {item}" for item in items)
|
||||
lines.append("")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
600
scripts/gitea-import-all-backlogs.py
Normal file
600
scripts/gitea-import-all-backlogs.py
Normal file
@@ -0,0 +1,600 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Import backlog-like files from git.add-ideas.de repositories into Gitea."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import dataclasses
|
||||
import hashlib
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from typing import Any, Iterable
|
||||
|
||||
from gitea_common import GiteaClient, GiteaError, infer_target, load_dotenv, repo_path, require_token
|
||||
|
||||
|
||||
GIT_ROOT = pathlib.Path("/mnt/DATA/git")
|
||||
PRODUCTS_ROOT = pathlib.Path("/mnt/DATA/Nextcloud/ADD ideas UG/Products")
|
||||
EXCLUDED_FILE_PATTERNS = (
|
||||
"/.git/",
|
||||
"/.venv/",
|
||||
"/venv/",
|
||||
"/site-packages/",
|
||||
"/node_modules/",
|
||||
"/__pycache__/",
|
||||
"/dist/",
|
||||
"/build/",
|
||||
"/.cache/",
|
||||
"/.module-test-build/",
|
||||
)
|
||||
BACKLOG_NAME_RE = re.compile(r"(backlog|todo|roadmap|plan|issue|milestone|action)", re.IGNORECASE)
|
||||
ACTION_RE = re.compile(
|
||||
r"^(add|avoid|build|convert|create|define|delay|design|document|enable|ensure|expand|extract|fetch|fix|finalize|generate|"
|
||||
r"implement|import|improve|integrate|keep|make|move|optimize|persist|precompute|promote|provide|"
|
||||
r"publish|query|replace|review|run|seed|serve|split|start|store|support|test|track|use|wire)\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
SKIP_SECTION_RE = re.compile(
|
||||
r"(completed|implemented|current state|recent fixes|recently completed|working assumptions|stable decisions|"
|
||||
r"available now|done|archive)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
ACTIVE_SECTION_RE = re.compile(
|
||||
r"(p0|p1|p2|p3|mvp|milestone|phase|todo|backlog|roadmap|tasks|testing|frontend|backend|routing|"
|
||||
r"data outputs|future|critical path|next sprint|optimization|hardening|enhancements?)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
GENERIC_LABELS = [
|
||||
("type/task", "1d76db", "Implementation, maintenance, migration, or operational work."),
|
||||
("type/feature", "0e8a16", "New user-visible behavior or product capability."),
|
||||
("type/debt", "fbca04", "Cleanup, refactoring, risk reduction, or deferred engineering work."),
|
||||
("type/docs", "5319e7", "Documentation, process, or developer workflow work."),
|
||||
("status/triage", "d4c5f9", "Needs review, ownership, priority, or acceptance criteria."),
|
||||
("source/backlog-import", "bfdadc", "Imported from markdown, text, CSV, roadmap, backlog, plan, or TODO files."),
|
||||
("codex/ready", "0e8a16", "Suitable for Codex to pick up with the existing issue context."),
|
||||
("priority/p0", "b60205", "Immediate stop-the-line priority."),
|
||||
("priority/p1", "d93f0b", "High priority for the next focused work window."),
|
||||
("priority/p2", "fbca04", "Normal planned priority."),
|
||||
("priority/p3", "c2e0c6", "Low priority or opportunistic cleanup."),
|
||||
]
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class RepoInfo:
|
||||
root: pathlib.Path
|
||||
remote: str
|
||||
owner: str
|
||||
repo: str
|
||||
|
||||
@property
|
||||
def key(self) -> str:
|
||||
return self.root.name
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class SourceFile:
|
||||
repo_key: str
|
||||
path: pathlib.Path
|
||||
source_kind: str
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class Candidate:
|
||||
repo_key: str
|
||||
title: str
|
||||
body: str
|
||||
labels: tuple[str, ...]
|
||||
priority: str
|
||||
milestone: str
|
||||
fingerprint: str
|
||||
source: str
|
||||
line: int
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--env-file", type=pathlib.Path, default=pathlib.Path("/home/zemion/.config/gitea/gitea.env"))
|
||||
parser.add_argument("--git-root", type=pathlib.Path, default=GIT_ROOT)
|
||||
parser.add_argument("--products-root", type=pathlib.Path, default=PRODUCTS_ROOT)
|
||||
parser.add_argument("--apply", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
load_dotenv(args.env_file)
|
||||
repos = discover_hosted_repos(args.git_root)
|
||||
sources = discover_sources(repos, args.products_root)
|
||||
candidates = list(build_candidates(sources))
|
||||
candidates = dedupe_candidates(candidates)
|
||||
candidates.sort(key=lambda item: (item.repo_key, item.source, item.line, item.title))
|
||||
|
||||
print(f"Hosted repositories: {len(repos)}")
|
||||
print(f"Backlog-like source files: {len(sources)}")
|
||||
print(f"Issue candidates: {len(candidates)}")
|
||||
for repo_key, count in grouped_counts(candidates).items():
|
||||
print(f" {repo_key}: {count}")
|
||||
|
||||
token = require_token() if args.apply else os.environ.get("GITEA_TOKEN")
|
||||
if not token:
|
||||
print_preview(candidates)
|
||||
print("Dry run without GITEA_TOKEN: duplicate comparison skipped.")
|
||||
return 0
|
||||
|
||||
missing_by_repo: dict[str, list[Candidate]] = {}
|
||||
skipped = 0
|
||||
states: dict[str, RepoState] = {}
|
||||
for repo_key in sorted({candidate.repo_key for candidate in candidates}):
|
||||
states[repo_key] = load_repo_state(repos[repo_key], token, apply=args.apply)
|
||||
for candidate in candidates:
|
||||
state = states[candidate.repo_key]
|
||||
if candidate.fingerprint in state.fingerprints or normalize_title(candidate.title) in state.titles:
|
||||
skipped += 1
|
||||
continue
|
||||
missing_by_repo.setdefault(candidate.repo_key, []).append(candidate)
|
||||
|
||||
missing_total = sum(len(items) for items in missing_by_repo.values())
|
||||
print(f"Skipped existing: {skipped}")
|
||||
print(f"Missing candidates: {missing_total}")
|
||||
print_preview([candidate for items in missing_by_repo.values() for candidate in items])
|
||||
|
||||
if not args.apply:
|
||||
print("Dry run only. Re-run with --apply to create missing issues.")
|
||||
return 0
|
||||
|
||||
for repo_key, repo_candidates in missing_by_repo.items():
|
||||
state = states[repo_key]
|
||||
for candidate in repo_candidates:
|
||||
label_ids = [state.label_ids[label] for label in candidate.labels]
|
||||
if candidate.priority not in candidate.labels:
|
||||
label_ids.append(state.label_ids[candidate.priority])
|
||||
milestone_id = ensure_milestone(state, candidate.milestone)
|
||||
created = state.client.request_json(
|
||||
"POST",
|
||||
repo_path(state.target.owner, state.target.repo, "/issues"),
|
||||
body={
|
||||
"title": candidate.title,
|
||||
"body": candidate.body,
|
||||
"labels": label_ids,
|
||||
"milestone": milestone_id,
|
||||
},
|
||||
)
|
||||
number = created.get("number") or created.get("index")
|
||||
print(f"created {state.target.owner}/{state.target.repo}#{number}: {candidate.title}")
|
||||
state.fingerprints.add(candidate.fingerprint)
|
||||
state.titles.add(normalize_title(candidate.title))
|
||||
return 0
|
||||
except GiteaError as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class RepoState:
|
||||
target: Any
|
||||
client: GiteaClient
|
||||
label_ids: dict[str, int]
|
||||
milestone_ids: dict[str, int]
|
||||
fingerprints: set[str]
|
||||
titles: set[str]
|
||||
|
||||
|
||||
def discover_hosted_repos(git_root: pathlib.Path) -> dict[str, RepoInfo]:
|
||||
repos: dict[str, RepoInfo] = {}
|
||||
for gitdir in sorted(git_root.glob("*/.git")):
|
||||
root = gitdir.parent
|
||||
result = subprocess.run(
|
||||
["git", "-C", str(root), "remote", "get-url", "origin"],
|
||||
check=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
text=True,
|
||||
)
|
||||
remote = result.stdout.strip()
|
||||
if "git.add-ideas.de" not in remote:
|
||||
continue
|
||||
target = infer_target(root)
|
||||
repos[root.name] = RepoInfo(root=root, remote=remote, owner=target.owner, repo=target.repo)
|
||||
return repos
|
||||
|
||||
|
||||
def discover_sources(repos: dict[str, RepoInfo], products_root: pathlib.Path) -> list[SourceFile]:
|
||||
sources: list[SourceFile] = []
|
||||
for repo in repos.values():
|
||||
for path in repo.root.rglob("*"):
|
||||
if not path.is_file() or not BACKLOG_NAME_RE.search(path.name):
|
||||
continue
|
||||
path_text = path.as_posix()
|
||||
if any(pattern in path_text for pattern in EXCLUDED_FILE_PATTERNS):
|
||||
continue
|
||||
if is_excluded_repo_file(path):
|
||||
continue
|
||||
if is_text_backlog(path):
|
||||
sources.append(SourceFile(repo_key=repo.key, path=path, source_kind="repo"))
|
||||
|
||||
product_map = {
|
||||
"co2api": "emission-api-lib",
|
||||
"govoplan": "govoplan-core",
|
||||
"meubility": "meubility-workbench",
|
||||
"mousehold": "mousehold",
|
||||
"prvnncr": "prvnncr-server",
|
||||
}
|
||||
if products_root.exists():
|
||||
for product_dir in sorted(path for path in products_root.iterdir() if path.is_dir()):
|
||||
repo_key = product_map.get(product_dir.name)
|
||||
if not repo_key or repo_key not in repos:
|
||||
continue
|
||||
for path in product_dir.rglob("*"):
|
||||
if path.is_file() and BACKLOG_NAME_RE.search(path.name) and is_text_backlog(path):
|
||||
sources.append(SourceFile(repo_key=repo_key, path=path, source_kind="product"))
|
||||
|
||||
unique: dict[tuple[str, pathlib.Path], SourceFile] = {}
|
||||
for source in sources:
|
||||
unique[(source.repo_key, source.path.resolve())] = source
|
||||
return sorted(unique.values(), key=lambda item: (item.repo_key, str(item.path)))
|
||||
|
||||
|
||||
def is_excluded_repo_file(path: pathlib.Path) -> bool:
|
||||
text = path.as_posix()
|
||||
name = path.name.lower()
|
||||
if "/scripts/gitea-" in text or text.endswith("/scripts/gitea_common.py"):
|
||||
return True
|
||||
if "testing_plan" in name or "test_plan" in name:
|
||||
return True
|
||||
if text.endswith("/docs/GITEA_ISSUES.md"):
|
||||
return True
|
||||
if text.endswith("/docs/GOVOPLAN_MODULE_ROADMAP.md"):
|
||||
return True
|
||||
if text.endswith("/docs/ACCESS_EXTRACTION_PLAN.md"):
|
||||
return True
|
||||
if text.endswith("/govoplan-web/TODO.md"):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def is_text_backlog(path: pathlib.Path) -> bool:
|
||||
if path.suffix.lower() not in {".md", ".txt", ".csv"}:
|
||||
return False
|
||||
try:
|
||||
chunk = path.read_bytes()[:4096]
|
||||
except OSError:
|
||||
return False
|
||||
if b"\x00" in chunk:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def build_candidates(sources: list[SourceFile]) -> Iterable[Candidate]:
|
||||
for source in sources:
|
||||
suffix = source.path.suffix.lower()
|
||||
if suffix == ".csv":
|
||||
yield from parse_csv_source(source)
|
||||
else:
|
||||
yield from parse_text_source(source)
|
||||
|
||||
|
||||
def parse_csv_source(source: SourceFile) -> Iterable[Candidate]:
|
||||
with source.path.open("r", encoding="utf-8-sig", newline="") as handle:
|
||||
reader = csv.DictReader(handle)
|
||||
for index, row in enumerate(reader, start=2):
|
||||
story = (row.get("story") or row.get("title") or row.get("name") or "").strip()
|
||||
if not story:
|
||||
continue
|
||||
identifier = (row.get("id") or "").strip()
|
||||
priority = normalize_priority(row.get("priority") or "")
|
||||
milestone = (row.get("epic") or "Backlog").strip()
|
||||
title = format_title("[Feature]", f"{identifier}: {story}" if identifier else story)
|
||||
fingerprint = make_fingerprint(source.path, index, source.repo_key, story)
|
||||
body = "\n".join(
|
||||
[
|
||||
f"<!-- codex-generic-backlog-fingerprint:{fingerprint} -->",
|
||||
"",
|
||||
"Imported from a CSV backlog file.",
|
||||
"",
|
||||
f"- Source: `{source.path}`",
|
||||
f"- Line: `{index}`",
|
||||
f"- Source kind: `{source.source_kind}`",
|
||||
f"- Milestone: `{milestone}`",
|
||||
"",
|
||||
"CSV row:",
|
||||
"",
|
||||
"```text",
|
||||
", ".join(f"{key}={value}" for key, value in row.items()),
|
||||
"```",
|
||||
]
|
||||
)
|
||||
yield Candidate(
|
||||
repo_key=source.repo_key,
|
||||
title=title,
|
||||
body=body,
|
||||
labels=("type/feature", "status/triage", "source/backlog-import", "codex/ready"),
|
||||
priority=priority,
|
||||
milestone=milestone,
|
||||
fingerprint=fingerprint,
|
||||
source=str(source.path),
|
||||
line=index,
|
||||
)
|
||||
|
||||
|
||||
def parse_text_source(source: SourceFile) -> Iterable[Candidate]:
|
||||
try:
|
||||
lines = source.path.read_text(encoding="utf-8").splitlines()
|
||||
except UnicodeDecodeError:
|
||||
lines = source.path.read_text(encoding="latin-1").splitlines()
|
||||
|
||||
headings: list[tuple[int, str]] = []
|
||||
in_tasks = False
|
||||
for index, line in enumerate(lines):
|
||||
line_number = index + 1
|
||||
heading = parse_heading(line)
|
||||
if heading:
|
||||
level, title = heading
|
||||
headings = [(h_level, h_title) for h_level, h_title in headings if h_level < level]
|
||||
headings.append((level, title))
|
||||
in_tasks = False
|
||||
continue
|
||||
|
||||
if re.match(r"^\s*(tasks|action items|immediate todos?|recommended next sprint)\s*:?\s*$", line, re.IGNORECASE):
|
||||
in_tasks = True
|
||||
continue
|
||||
|
||||
item = parse_open_item(line)
|
||||
if item is None and is_action_context(source.path, headings, in_tasks):
|
||||
item = parse_plain_action_item(line)
|
||||
if item is None:
|
||||
continue
|
||||
if not item or should_skip_item(item, headings):
|
||||
continue
|
||||
|
||||
section = section_title(headings)
|
||||
priority = priority_from_context(section, item)
|
||||
milestone = milestone_from_context(source.path, section)
|
||||
prefix = "[Feature]" if ACTION_RE.match(item) else "[Task]"
|
||||
title = format_title(prefix, item)
|
||||
fingerprint = make_fingerprint(source.path, line_number, source.repo_key, item)
|
||||
body = "\n".join(
|
||||
[
|
||||
f"<!-- codex-generic-backlog-fingerprint:{fingerprint} -->",
|
||||
"",
|
||||
"Imported from a backlog-like text file.",
|
||||
"",
|
||||
f"- Source: `{source.path}`",
|
||||
f"- Line: `{line_number}`",
|
||||
f"- Source kind: `{source.source_kind}`",
|
||||
f"- Section: `{section or 'none'}`",
|
||||
"",
|
||||
"Imported item:",
|
||||
"",
|
||||
"```text",
|
||||
item,
|
||||
"```",
|
||||
]
|
||||
)
|
||||
yield Candidate(
|
||||
repo_key=source.repo_key,
|
||||
title=title,
|
||||
body=body,
|
||||
labels=("type/feature" if prefix == "[Feature]" else "type/task", "status/triage", "source/backlog-import", "codex/ready"),
|
||||
priority=priority,
|
||||
milestone=milestone,
|
||||
fingerprint=fingerprint,
|
||||
source=str(source.path),
|
||||
line=line_number,
|
||||
)
|
||||
|
||||
|
||||
def parse_heading(line: str) -> tuple[int, str] | None:
|
||||
markdown = re.match(r"^(#{1,6})\s+(.+?)\s*$", line)
|
||||
if markdown:
|
||||
return len(markdown.group(1)), clean_text(markdown.group(2))
|
||||
underlined = re.match(r"^(.+?)\s*$", line)
|
||||
if underlined and line.strip() and len(line.strip()) < 120:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def parse_open_item(line: str) -> str | None:
|
||||
patterns = [
|
||||
r"^(?P<indent>\s*)[-*]\s+\[\s\]\s+(?P<text>.+?)\s*$",
|
||||
r"^(?P<indent>\s*)[-*]\s+☐\s+(?P<text>.+?)\s*$",
|
||||
r"^(?P<indent>\s*)\d+[.)]\s+☐\s+(?P<text>.+?)\s*$",
|
||||
r"^(?P<indent>\s*)[-*]\s+\[(?!x\]|X\])(?:[^\]]+)\]\s+(?P<text>.+?)\s*$",
|
||||
]
|
||||
for pattern in patterns:
|
||||
match = re.match(pattern, line)
|
||||
if match and leading_width(match.group("indent")) == 0:
|
||||
return clean_text(match.group("text"))
|
||||
return None
|
||||
|
||||
|
||||
def parse_plain_action_item(line: str) -> str | None:
|
||||
match = re.match(r"^(?P<indent>\s*)[-*]\s+(?P<text>.+?)\s*$", line)
|
||||
if match and leading_width(match.group("indent")) == 0:
|
||||
text = clean_text(match.group("text"))
|
||||
if "✅" in text or text.startswith(("✅", "[x]", "[X]")):
|
||||
return None
|
||||
if ACTION_RE.match(text) or is_short_noun_task(text):
|
||||
return text
|
||||
return None
|
||||
|
||||
|
||||
def is_action_context(path: pathlib.Path, headings: list[tuple[int, str]], in_tasks: bool) -> bool:
|
||||
if in_tasks:
|
||||
return True
|
||||
context = section_title(headings)
|
||||
if not context:
|
||||
return False
|
||||
if SKIP_SECTION_RE.search(context):
|
||||
return False
|
||||
if ACTIVE_SECTION_RE.search(context):
|
||||
return True
|
||||
return path.name.lower() in {"todo.txt", "todo.md"}
|
||||
|
||||
|
||||
def should_skip_item(item: str, headings: list[tuple[int, str]]) -> bool:
|
||||
context = section_title(headings)
|
||||
if "✅" in item or item.startswith(("✅", "[x]", "[X]")):
|
||||
return True
|
||||
if item.endswith((",", ";")):
|
||||
return True
|
||||
if SKIP_SECTION_RE.search(context):
|
||||
return True
|
||||
if len(item) < 4:
|
||||
return True
|
||||
if item.endswith(":") and len(item.split()) <= 6:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def is_short_noun_task(text: str) -> bool:
|
||||
return len(text.split()) <= 8 and not text.endswith(".") and not re.search(r"://", text)
|
||||
|
||||
|
||||
def clean_text(text: str) -> str:
|
||||
text = text.strip()
|
||||
text = re.sub(r"^☐\s*", "", text)
|
||||
text = re.sub(r"^✅\s*", "", text)
|
||||
text = re.sub(r"\s+", " ", text)
|
||||
return text.strip(" -")
|
||||
|
||||
|
||||
def leading_width(indent: str) -> int:
|
||||
return len(indent.replace("\t", " "))
|
||||
|
||||
|
||||
def section_title(headings: list[tuple[int, str]]) -> str:
|
||||
return " > ".join(title for _, title in headings)
|
||||
|
||||
|
||||
def priority_from_context(section: str, item: str) -> str:
|
||||
text = f"{section} {item}".lower()
|
||||
if "p0" in text or "critical path" in text:
|
||||
return "priority/p0"
|
||||
if "p1" in text or "immediate" in text or "next sprint" in text:
|
||||
return "priority/p1"
|
||||
if "p2" in text:
|
||||
return "priority/p2"
|
||||
if "p3" in text or "future" in text or "later" in text:
|
||||
return "priority/p3"
|
||||
return "priority/p2"
|
||||
|
||||
|
||||
def normalize_priority(value: str) -> str:
|
||||
value = value.strip().lower()
|
||||
if value == "p0":
|
||||
return "priority/p0"
|
||||
if value == "p1":
|
||||
return "priority/p1"
|
||||
if value == "p2":
|
||||
return "priority/p2"
|
||||
if value == "p3":
|
||||
return "priority/p3"
|
||||
return "priority/p2"
|
||||
|
||||
|
||||
def milestone_from_context(path: pathlib.Path, section: str) -> str:
|
||||
if section:
|
||||
parts = [part for part in section.split(" > ") if ACTIVE_SECTION_RE.search(part)]
|
||||
if parts:
|
||||
return parts[-1][:120]
|
||||
return path.stem.replace("_", " ").replace("-", " ").title()
|
||||
|
||||
|
||||
def format_title(prefix: str, item: str) -> str:
|
||||
title = f"{prefix} {clean_text(item).rstrip('.')}"
|
||||
if len(title) <= 180:
|
||||
return title
|
||||
return f"{title[:177].rstrip()}..."
|
||||
|
||||
|
||||
def make_fingerprint(path: pathlib.Path, line: int, repo_key: str, item: str) -> str:
|
||||
stable = "\n".join([str(path.resolve()), str(line), repo_key, item])
|
||||
return hashlib.sha256(stable.encode("utf-8")).hexdigest()[:24]
|
||||
|
||||
|
||||
def dedupe_candidates(candidates: list[Candidate]) -> list[Candidate]:
|
||||
seen: set[tuple[str, str]] = set()
|
||||
unique: list[Candidate] = []
|
||||
for candidate in candidates:
|
||||
key = (candidate.repo_key, normalize_title(candidate.title))
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
unique.append(candidate)
|
||||
return unique
|
||||
|
||||
|
||||
def grouped_counts(candidates: list[Candidate]) -> dict[str, int]:
|
||||
counts: dict[str, int] = {}
|
||||
for candidate in candidates:
|
||||
counts[candidate.repo_key] = counts.get(candidate.repo_key, 0) + 1
|
||||
return dict(sorted(counts.items()))
|
||||
|
||||
|
||||
def load_repo_state(repo: RepoInfo, token: str, *, apply: bool) -> RepoState:
|
||||
target = infer_target(repo.root)
|
||||
client = GiteaClient(target, token)
|
||||
labels = client.paginate(repo_path(target.owner, target.repo, "/labels"), limit=50)
|
||||
label_ids = {str(label["name"]): int(label["id"]) for label in labels}
|
||||
if apply:
|
||||
for name, color, description in GENERIC_LABELS:
|
||||
if name in label_ids:
|
||||
continue
|
||||
created = client.request_json(
|
||||
"POST",
|
||||
repo_path(target.owner, target.repo, "/labels"),
|
||||
body={"name": name, "color": color, "description": description, "exclusive": name.startswith(("type/", "status/", "priority/"))},
|
||||
)
|
||||
label_ids[str(created["name"])] = int(created["id"])
|
||||
print(f"created label {target.owner}/{target.repo}:{name}")
|
||||
|
||||
missing = [name for name, _, _ in GENERIC_LABELS if name not in label_ids]
|
||||
if missing and apply:
|
||||
raise GiteaError(f"{repo.key} missing labels: {', '.join(missing)}. Re-run with --apply to create them.")
|
||||
|
||||
milestones = client.paginate(repo_path(target.owner, target.repo, "/milestones"), query={"state": "all"}, limit=50)
|
||||
milestone_ids = {str(item["title"]): int(item["id"]) for item in milestones}
|
||||
issues = client.paginate(repo_path(target.owner, target.repo, "/issues"), query={"state": "all"}, limit=50)
|
||||
fingerprints: set[str] = set()
|
||||
titles: set[str] = set()
|
||||
for issue in issues:
|
||||
body = str(issue.get("body") or "")
|
||||
titles.add(normalize_title(str(issue.get("title") or "")))
|
||||
fingerprints.update(re.findall(r"codex-(?:generic-)?backlog-fingerprint:([0-9a-f]{24})", body))
|
||||
return RepoState(target=target, client=client, label_ids=label_ids, milestone_ids=milestone_ids, fingerprints=fingerprints, titles=titles)
|
||||
|
||||
|
||||
def ensure_milestone(state: RepoState, title: str) -> int:
|
||||
if title in state.milestone_ids:
|
||||
return state.milestone_ids[title]
|
||||
created = state.client.request_json(
|
||||
"POST",
|
||||
repo_path(state.target.owner, state.target.repo, "/milestones"),
|
||||
body={"title": title, "state": "open", "description": "Created from imported backlog-like files."},
|
||||
)
|
||||
state.milestone_ids[title] = int(created["id"])
|
||||
print(f"created milestone {state.target.owner}/{state.target.repo}:{title}")
|
||||
return state.milestone_ids[title]
|
||||
|
||||
|
||||
def normalize_title(title: str) -> str:
|
||||
return re.sub(r"[^a-z0-9]+", " ", title.lower()).strip()
|
||||
|
||||
|
||||
def print_preview(candidates: list[Candidate], limit_per_repo: int = 80) -> None:
|
||||
by_repo: dict[str, list[Candidate]] = {}
|
||||
for candidate in candidates:
|
||||
by_repo.setdefault(candidate.repo_key, []).append(candidate)
|
||||
for repo_key, items in sorted(by_repo.items()):
|
||||
print(f"{repo_key}:")
|
||||
for item in items[:limit_per_repo]:
|
||||
print(f" {item.title} [{item.priority}; {item.milestone}]")
|
||||
if len(items) > limit_per_repo:
|
||||
print(f" ... {len(items) - limit_per_repo} more")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
113
scripts/gitea-install-workflow.py
Normal file
113
scripts/gitea-install-workflow.py
Normal file
@@ -0,0 +1,113 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Install Gitea issue workflow files into one or more repositories."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
from gitea_common import GiteaError, repo_root
|
||||
|
||||
|
||||
SOURCE_ROOT = pathlib.Path(__file__).resolve().parents[1]
|
||||
WORKFLOW_FILES = (
|
||||
".gitea/PULL_REQUEST_TEMPLATE.md",
|
||||
".gitea/ISSUE_TEMPLATE/bug_report.md",
|
||||
".gitea/ISSUE_TEMPLATE/config.yaml",
|
||||
".gitea/ISSUE_TEMPLATE/docs_workflow.md",
|
||||
".gitea/ISSUE_TEMPLATE/feature_request.md",
|
||||
".gitea/ISSUE_TEMPLATE/task.md",
|
||||
".gitea/ISSUE_TEMPLATE/tech_debt.md",
|
||||
)
|
||||
MODULE_LABEL_BY_REPO = {
|
||||
"govoplan-core": "module/core",
|
||||
"govoplan-access": "module/access",
|
||||
"govoplan-admin": "module/admin",
|
||||
"govoplan-tenancy": "module/tenancy",
|
||||
"govoplan-policy": "module/policy",
|
||||
"govoplan-audit": "module/audit",
|
||||
"govoplan-mail": "module/mail",
|
||||
"govoplan-files": "module/files",
|
||||
"govoplan-campaign": "module/campaign",
|
||||
"govoplan-web": "module/web",
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("targets", nargs="*", type=pathlib.Path, help="target repository roots or child paths")
|
||||
parser.add_argument("--module-label", help="module label to write into issue templates; only valid for one target")
|
||||
parser.add_argument("--include-labels-file", action="store_true", help="also copy docs/gitea-labels.json")
|
||||
parser.add_argument("--apply", action="store_true", help="write files instead of previewing changes")
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
target_roots = [repo_root(path) for path in (args.targets or [pathlib.Path.cwd()])]
|
||||
unique_roots = list(dict.fromkeys(target_roots))
|
||||
if args.module_label and len(unique_roots) != 1:
|
||||
raise GiteaError("--module-label can only be used with a single target")
|
||||
|
||||
rel_paths = list(WORKFLOW_FILES)
|
||||
if args.include_labels_file:
|
||||
rel_paths.append("docs/gitea-labels.json")
|
||||
|
||||
for target_root in unique_roots:
|
||||
module_label = args.module_label or infer_module_label(target_root)
|
||||
install_files(target_root, rel_paths, module_label, apply=args.apply)
|
||||
|
||||
if not args.apply:
|
||||
print("Dry run only. Re-run with --apply to write files.")
|
||||
return 0
|
||||
except GiteaError as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
def infer_module_label(target_root: pathlib.Path) -> str:
|
||||
repo_name = target_root.name
|
||||
try:
|
||||
repo_name = target_root.resolve().name
|
||||
except OSError:
|
||||
pass
|
||||
label = MODULE_LABEL_BY_REPO.get(repo_name)
|
||||
if label:
|
||||
return label
|
||||
if repo_name.startswith("govoplan-"):
|
||||
suffix = repo_name.removeprefix("govoplan-")
|
||||
if suffix:
|
||||
return f"module/{suffix}"
|
||||
raise GiteaError(f"cannot infer module label for {target_root}. Use --module-label module/<name>.")
|
||||
|
||||
|
||||
def install_files(target_root: pathlib.Path, rel_paths: list[str], module_label: str, *, apply: bool) -> None:
|
||||
print(f"Target: {target_root} ({module_label})")
|
||||
for rel_path in rel_paths:
|
||||
source = SOURCE_ROOT / rel_path
|
||||
target = target_root / rel_path
|
||||
if not source.exists():
|
||||
raise GiteaError(f"missing workflow source file: {source}")
|
||||
|
||||
content = source.read_text(encoding="utf-8")
|
||||
content = transform_content(rel_path, content, module_label)
|
||||
|
||||
existing = target.read_text(encoding="utf-8") if target.exists() else None
|
||||
if existing == content:
|
||||
print(f"unchanged {rel_path}")
|
||||
continue
|
||||
|
||||
action = "create" if existing is None else "update"
|
||||
print(f"{action if apply else 'would ' + action} {rel_path}")
|
||||
if apply:
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_text(content, encoding="utf-8")
|
||||
|
||||
|
||||
def transform_content(rel_path: str, content: str, module_label: str) -> str:
|
||||
if rel_path.startswith(".gitea/ISSUE_TEMPLATE/") and rel_path.endswith(".md"):
|
||||
return content.replace(" - module/core", f" - {module_label}")
|
||||
return content
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
149
scripts/gitea-sync-labels.py
Normal file
149
scripts/gitea-sync-labels.py
Normal file
@@ -0,0 +1,149 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Synchronize issue labels to a Gitea repository."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import pathlib
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
from gitea_common import (
|
||||
GiteaClient,
|
||||
GiteaError,
|
||||
infer_target,
|
||||
load_dotenv,
|
||||
load_json,
|
||||
repo_path,
|
||||
repo_root,
|
||||
require_token,
|
||||
)
|
||||
|
||||
|
||||
DEFAULT_LABELS_FILE = pathlib.Path(__file__).resolve().parents[1] / "docs" / "gitea-labels.json"
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--root", type=pathlib.Path, default=pathlib.Path.cwd(), help="repository root or child path")
|
||||
parser.add_argument("--remote", default="origin", help="git remote to use for target inference")
|
||||
parser.add_argument("--labels-file", type=pathlib.Path, default=DEFAULT_LABELS_FILE)
|
||||
parser.add_argument("--env-file", type=pathlib.Path, help="dotenv file to read before using GITEA_* values")
|
||||
parser.add_argument("--apply", action="store_true", help="create or update labels in Gitea")
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
root = repo_root(args.root)
|
||||
load_dotenv(args.env_file or root / ".env")
|
||||
target = infer_target(root, args.remote)
|
||||
labels = _load_labels(args.labels_file)
|
||||
token = require_token() if args.apply else os.environ.get("GITEA_TOKEN")
|
||||
|
||||
print(f"Target: {target.display}")
|
||||
print(f"Labels file: {args.labels_file}")
|
||||
|
||||
if not args.apply and not token:
|
||||
print("Dry run without GITEA_TOKEN: API comparison skipped.")
|
||||
for label in labels:
|
||||
print(f"would ensure {label['name']} ({label['color']})")
|
||||
return 0
|
||||
|
||||
client = GiteaClient(target, token)
|
||||
existing = _labels_by_name(client, target.owner, target.repo)
|
||||
creates: list[dict[str, Any]] = []
|
||||
updates: list[tuple[dict[str, Any], dict[str, Any]]] = []
|
||||
|
||||
for label in labels:
|
||||
current = existing.get(label["name"])
|
||||
if current is None:
|
||||
creates.append(label)
|
||||
continue
|
||||
|
||||
patch = _diff_label(current, label)
|
||||
if patch:
|
||||
updates.append((current, patch))
|
||||
|
||||
if not creates and not updates:
|
||||
print("No label changes needed.")
|
||||
return 0
|
||||
|
||||
for label in creates:
|
||||
print(f"{'create' if args.apply else 'would create'} {label['name']}")
|
||||
if args.apply:
|
||||
client.request_json(
|
||||
"POST",
|
||||
repo_path(target.owner, target.repo, "/labels"),
|
||||
body=label,
|
||||
)
|
||||
|
||||
for current, patch in updates:
|
||||
print(f"{'update' if args.apply else 'would update'} {current['name']}: {', '.join(sorted(patch))}")
|
||||
if args.apply:
|
||||
client.request_json(
|
||||
"PATCH",
|
||||
repo_path(target.owner, target.repo, f"/labels/{current['id']}"),
|
||||
body=patch,
|
||||
)
|
||||
|
||||
return 0
|
||||
except GiteaError as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
def _load_labels(path: pathlib.Path) -> list[dict[str, Any]]:
|
||||
payload = load_json(path)
|
||||
if not isinstance(payload, list):
|
||||
raise GiteaError(f"{path} must contain a JSON list")
|
||||
|
||||
labels: list[dict[str, Any]] = []
|
||||
seen: set[str] = set()
|
||||
for index, item in enumerate(payload, start=1):
|
||||
if not isinstance(item, dict):
|
||||
raise GiteaError(f"label #{index} must be an object")
|
||||
label = {
|
||||
"name": str(item.get("name", "")).strip(),
|
||||
"color": str(item.get("color", "")).strip().lstrip("#").lower(),
|
||||
"description": str(item.get("description", "")).strip(),
|
||||
"exclusive": bool(item.get("exclusive", False)),
|
||||
}
|
||||
if not label["name"]:
|
||||
raise GiteaError(f"label #{index} is missing name")
|
||||
if label["name"] in seen:
|
||||
raise GiteaError(f"duplicate label name: {label['name']}")
|
||||
if not _is_hex_color(label["color"]):
|
||||
raise GiteaError(f"{label['name']} has invalid color {label['color']!r}")
|
||||
seen.add(label["name"])
|
||||
labels.append(label)
|
||||
return labels
|
||||
|
||||
|
||||
def _labels_by_name(client: GiteaClient, owner: str, repo: str) -> dict[str, dict[str, Any]]:
|
||||
labels = client.paginate(repo_path(owner, repo, "/labels"))
|
||||
return {str(label.get("name")): label for label in labels}
|
||||
|
||||
|
||||
def _diff_label(current: dict[str, Any], desired: dict[str, Any]) -> dict[str, Any]:
|
||||
patch: dict[str, Any] = {}
|
||||
if _normalize_color(current.get("color")) != desired["color"]:
|
||||
patch["color"] = desired["color"]
|
||||
if str(current.get("description") or "").strip() != desired["description"]:
|
||||
patch["description"] = desired["description"]
|
||||
if bool(current.get("exclusive", False)) != desired["exclusive"]:
|
||||
patch["exclusive"] = desired["exclusive"]
|
||||
return patch
|
||||
|
||||
|
||||
def _normalize_color(value: Any) -> str:
|
||||
return str(value or "").strip().lstrip("#").lower()
|
||||
|
||||
|
||||
def _is_hex_color(value: str) -> bool:
|
||||
if len(value) != 6:
|
||||
return False
|
||||
return all(char in "0123456789abcdef" for char in value)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
583
scripts/gitea-sync-wiki.py
Normal file
583
scripts/gitea-sync-wiki.py
Normal file
@@ -0,0 +1,583 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Mirror project documentation files into Gitea repository wikis."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import dataclasses
|
||||
import hashlib
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.parse
|
||||
from typing import Any
|
||||
|
||||
from gitea_common import GiteaClient, GiteaError, infer_target, load_dotenv, repo_path, require_token
|
||||
|
||||
|
||||
GIT_ROOT = pathlib.Path("/mnt/DATA/git")
|
||||
PRODUCTS_ROOT = pathlib.Path("/mnt/DATA/Nextcloud/ADD ideas UG/Products")
|
||||
MANAGED_MARKER = "<!-- codex-wiki-sync:"
|
||||
TEXT_SUFFIXES = {".md", ".txt", ".csv", ".toml", ".json", ".yaml", ".yml", ".rst"}
|
||||
SENSITIVE_NAME_RE = re.compile(r"(password|passwd|secret|credential|api[_-]?keys?|token|private[_-]?key)", re.IGNORECASE)
|
||||
EXCLUDED_PARTS = {
|
||||
".git",
|
||||
".gitea",
|
||||
".venv",
|
||||
"node_modules",
|
||||
"__pycache__",
|
||||
"dist",
|
||||
"build",
|
||||
".cache",
|
||||
".module-test-build",
|
||||
}
|
||||
EXCLUDED_NAMES = {
|
||||
"package-lock.json",
|
||||
"pnpm-lock.yaml",
|
||||
"yarn.lock",
|
||||
"uv.lock",
|
||||
}
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class RepoInfo:
|
||||
root: pathlib.Path
|
||||
owner: str
|
||||
repo: str
|
||||
|
||||
@property
|
||||
def key(self) -> str:
|
||||
return self.root.name
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class WikiSource:
|
||||
repo_key: str
|
||||
path: pathlib.Path
|
||||
origin: str
|
||||
page_title: str
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--env-file", type=pathlib.Path, default=pathlib.Path("/home/zemion/.config/gitea/gitea.env"))
|
||||
parser.add_argument("--git-root", type=pathlib.Path, default=GIT_ROOT)
|
||||
parser.add_argument("--products-root", type=pathlib.Path, default=PRODUCTS_ROOT)
|
||||
parser.add_argument("--repo", action="append", help="limit to one or more local repository directory names")
|
||||
parser.add_argument("--page", action="append", help="limit to one or more generated wiki page names")
|
||||
parser.add_argument("--overwrite-unmanaged", action="store_true", help="update existing wiki pages not previously managed by this script")
|
||||
parser.add_argument("--transport", choices=("git", "api"), default="git", help="sync through the wiki git repository or the Gitea REST API")
|
||||
parser.add_argument("--wiki-cache-dir", type=pathlib.Path, default=pathlib.Path("/tmp/codex-gitea-wiki-sync"), help="local cache for wiki git checkouts")
|
||||
parser.add_argument("--commit-message", default="Sync wiki from project files", help="commit message used by git transport")
|
||||
parser.add_argument("--apply", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
load_dotenv(args.env_file)
|
||||
repos = discover_hosted_repos(args.git_root)
|
||||
if args.repo:
|
||||
requested = set(args.repo)
|
||||
repos = {key: repo for key, repo in repos.items() if key in requested}
|
||||
missing = requested - set(repos)
|
||||
if missing:
|
||||
raise GiteaError(f"requested repos are not hosted on git.add-ideas.de locally: {', '.join(sorted(missing))}")
|
||||
|
||||
sources = discover_sources(repos, args.products_root)
|
||||
if args.page:
|
||||
requested_pages = set(args.page)
|
||||
sources = [source for source in sources if source.page_title in requested_pages]
|
||||
missing_pages = requested_pages - {source.page_title for source in sources}
|
||||
if missing_pages:
|
||||
raise GiteaError(f"requested pages were not discovered: {', '.join(sorted(missing_pages))}")
|
||||
print(f"Hosted repositories: {len(repos)}")
|
||||
print(f"Wiki source files: {len(sources)}")
|
||||
for repo_key, count in grouped_counts(sources).items():
|
||||
print(f" {repo_key}: {count}")
|
||||
|
||||
if not args.apply:
|
||||
preview_sources(sources)
|
||||
print("Dry run only. Re-run with --apply to write wiki pages.")
|
||||
return 0
|
||||
|
||||
token = require_token() if args.transport == "api" else ""
|
||||
failures = 0
|
||||
for repo_key, repo_sources in group_sources(sources).items():
|
||||
repo = repos[repo_key]
|
||||
try:
|
||||
if args.transport == "git":
|
||||
sync_repo_wiki_git(
|
||||
repo,
|
||||
repo_sources,
|
||||
cache_dir=args.wiki_cache_dir,
|
||||
overwrite_unmanaged=args.overwrite_unmanaged,
|
||||
commit_message=args.commit_message,
|
||||
write_index=not bool(args.page),
|
||||
)
|
||||
else:
|
||||
sync_repo_wiki(repo, repo_sources, token, overwrite_unmanaged=args.overwrite_unmanaged, write_index=not bool(args.page))
|
||||
except GiteaError as exc:
|
||||
failures += 1
|
||||
print(f"error syncing wiki for {repo_key}: {exc}", file=sys.stderr)
|
||||
return 1 if failures else 0
|
||||
except GiteaError as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
def discover_hosted_repos(git_root: pathlib.Path) -> dict[str, RepoInfo]:
|
||||
repos: dict[str, RepoInfo] = {}
|
||||
for gitdir in sorted(git_root.glob("*/.git")):
|
||||
root = gitdir.parent
|
||||
result = subprocess.run(
|
||||
["git", "-C", str(root), "remote", "get-url", "origin"],
|
||||
check=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
text=True,
|
||||
)
|
||||
if "git.add-ideas.de" not in result.stdout:
|
||||
continue
|
||||
target = infer_target(root)
|
||||
repos[root.name] = RepoInfo(root=root, owner=target.owner, repo=target.repo)
|
||||
return repos
|
||||
|
||||
|
||||
def discover_sources(repos: dict[str, RepoInfo], products_root: pathlib.Path) -> list[WikiSource]:
|
||||
sources: list[WikiSource] = []
|
||||
for repo in repos.values():
|
||||
for path in repo.root.rglob("*"):
|
||||
if is_repo_doc(repo.root, path):
|
||||
rel = path.relative_to(repo.root)
|
||||
sources.append(
|
||||
WikiSource(
|
||||
repo_key=repo.key,
|
||||
path=path,
|
||||
origin="repository",
|
||||
page_title=title_for_path("Repo", rel),
|
||||
)
|
||||
)
|
||||
|
||||
product_map = {
|
||||
"co2api": "emission-api-lib",
|
||||
"govoplan": "govoplan-core",
|
||||
"meubility": "meubility-workbench",
|
||||
"mousehold": "mousehold",
|
||||
"prvnncr": "prvnncr-server",
|
||||
}
|
||||
if products_root.exists():
|
||||
for product_dir in sorted(path for path in products_root.iterdir() if path.is_dir()):
|
||||
repo_key = product_map.get(product_dir.name)
|
||||
if not repo_key or repo_key not in repos:
|
||||
continue
|
||||
for path in product_dir.rglob("*"):
|
||||
if is_product_doc(path):
|
||||
rel = path.relative_to(product_dir)
|
||||
sources.append(
|
||||
WikiSource(
|
||||
repo_key=repo_key,
|
||||
path=path,
|
||||
origin=f"product:{product_dir.name}",
|
||||
page_title=title_for_path(f"Product {product_dir.name}", rel),
|
||||
)
|
||||
)
|
||||
|
||||
unique: dict[tuple[str, str], WikiSource] = {}
|
||||
for source in sources:
|
||||
unique[(source.repo_key, source.page_title)] = source
|
||||
return sorted(unique.values(), key=lambda item: (item.repo_key, item.page_title))
|
||||
|
||||
|
||||
def is_repo_doc(repo_root_path: pathlib.Path, path: pathlib.Path) -> bool:
|
||||
if not path.is_file() or not is_text_file(path):
|
||||
return False
|
||||
parts = set(path.relative_to(repo_root_path).parts)
|
||||
if parts & EXCLUDED_PARTS:
|
||||
return False
|
||||
if path.name in EXCLUDED_NAMES:
|
||||
return False
|
||||
if SENSITIVE_NAME_RE.search(path.name):
|
||||
return False
|
||||
rel = path.relative_to(repo_root_path)
|
||||
if len(rel.parts) == 1 and path.name.lower().startswith(("readme", "license", "changelog", "contributing", "security")):
|
||||
return True
|
||||
if "generated" in rel.parts:
|
||||
return False
|
||||
if rel.parts[0] in {"docs", "doc", "codex"} and path.suffix.lower() in {".md", ".txt", ".csv"}:
|
||||
return True
|
||||
if re.search(r"(backlog|todo|roadmap|plan|architecture|workflow|manifest)", path.name, re.IGNORECASE):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def is_product_doc(path: pathlib.Path) -> bool:
|
||||
if not path.is_file() or not is_text_file(path):
|
||||
return False
|
||||
if path.name in EXCLUDED_NAMES:
|
||||
return False
|
||||
if SENSITIVE_NAME_RE.search(path.name):
|
||||
return False
|
||||
if any(part in {"python_backup", "bruno", "fluege"} for part in path.parts):
|
||||
return False
|
||||
if re.search(
|
||||
r"(readme|todo|roadmap|plan|concept|continuation|copyright|notes|whitepaper|pitch|request_response)",
|
||||
path.name,
|
||||
re.IGNORECASE,
|
||||
) and path.suffix.lower() in {".md", ".txt"}:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def is_text_file(path: pathlib.Path) -> bool:
|
||||
if path.suffix.lower() not in TEXT_SUFFIXES:
|
||||
return False
|
||||
try:
|
||||
chunk = path.read_bytes()[:4096]
|
||||
except OSError:
|
||||
return False
|
||||
return b"\x00" not in chunk
|
||||
|
||||
|
||||
def title_for_path(prefix: str, rel: pathlib.Path) -> str:
|
||||
stem = rel.with_suffix("").as_posix()
|
||||
stem = re.sub(r"[^A-Za-z0-9]+", "-", stem).strip("-")
|
||||
return f"{prefix}-{stem}"[:180]
|
||||
|
||||
|
||||
def group_sources(sources: list[WikiSource]) -> dict[str, list[WikiSource]]:
|
||||
grouped: dict[str, list[WikiSource]] = {}
|
||||
for source in sources:
|
||||
grouped.setdefault(source.repo_key, []).append(source)
|
||||
return dict(sorted(grouped.items()))
|
||||
|
||||
|
||||
def grouped_counts(sources: list[WikiSource]) -> dict[str, int]:
|
||||
return {repo_key: len(items) for repo_key, items in group_sources(sources).items()}
|
||||
|
||||
|
||||
def preview_sources(sources: list[WikiSource]) -> None:
|
||||
for repo_key, repo_sources in group_sources(sources).items():
|
||||
print(f"{repo_key}:")
|
||||
for source in repo_sources[:80]:
|
||||
print(f" {source.page_title} <- {source.path}")
|
||||
if len(repo_sources) > 80:
|
||||
print(f" ... {len(repo_sources) - 80} more")
|
||||
|
||||
|
||||
def sync_repo_wiki(repo: RepoInfo, sources: list[WikiSource], token: str, *, overwrite_unmanaged: bool, write_index: bool = True) -> None:
|
||||
target = infer_target(repo.root)
|
||||
client = GiteaClient(target, token)
|
||||
existing_pages = list_existing_wiki_pages(client, target.owner, target.repo)
|
||||
existing_page_names = {
|
||||
str(page.get("title")): str(page.get("sub_url") or page.get("title"))
|
||||
for page in existing_pages
|
||||
if page.get("title")
|
||||
}
|
||||
index_lines = [
|
||||
f"# {target.repo} Project Wiki Index",
|
||||
"",
|
||||
f"{MANAGED_MARKER}index -->",
|
||||
"",
|
||||
"This page is generated from repository and product-directory project files.",
|
||||
"",
|
||||
]
|
||||
|
||||
for source in sources:
|
||||
content = render_wiki_content(source)
|
||||
changed = put_wiki_page(
|
||||
client,
|
||||
target.owner,
|
||||
target.repo,
|
||||
source.page_title,
|
||||
content,
|
||||
existing_page_names,
|
||||
overwrite_unmanaged=overwrite_unmanaged,
|
||||
)
|
||||
print(f"{'updated' if changed else 'unchanged'} {target.owner}/{target.repo} wiki:{source.page_title}")
|
||||
index_lines.append(f"- [{source.page_title}]({source.page_title}) - `{source.path}`")
|
||||
|
||||
if write_index:
|
||||
put_wiki_page(
|
||||
client,
|
||||
target.owner,
|
||||
target.repo,
|
||||
"Codex-Project-Index",
|
||||
"\n".join(index_lines) + "\n",
|
||||
existing_page_names,
|
||||
overwrite_unmanaged=True,
|
||||
)
|
||||
else:
|
||||
print(f"skipped {target.owner}/{target.repo} wiki:Codex-Project-Index during page-limited sync")
|
||||
|
||||
|
||||
def sync_repo_wiki_git(
|
||||
repo: RepoInfo,
|
||||
sources: list[WikiSource],
|
||||
*,
|
||||
cache_dir: pathlib.Path,
|
||||
overwrite_unmanaged: bool,
|
||||
commit_message: str,
|
||||
write_index: bool = True,
|
||||
) -> None:
|
||||
target = infer_target(repo.root)
|
||||
wiki_root = prepare_wiki_checkout(repo, cache_dir=cache_dir)
|
||||
index_lines = [
|
||||
f"# {target.repo} Project Wiki Index",
|
||||
"",
|
||||
f"{MANAGED_MARKER}index -->",
|
||||
"",
|
||||
"This page is generated from repository and product-directory project files.",
|
||||
"",
|
||||
]
|
||||
|
||||
for source in sources:
|
||||
content = render_wiki_content(source)
|
||||
status = write_wiki_page_file(
|
||||
wiki_root,
|
||||
source.page_title,
|
||||
content,
|
||||
overwrite_unmanaged=overwrite_unmanaged,
|
||||
)
|
||||
print(f"{status} {target.owner}/{target.repo} wiki:{source.page_title}")
|
||||
index_lines.append(f"- [{source.page_title}]({source.page_title}) - `{source.path}`")
|
||||
|
||||
if write_index:
|
||||
write_wiki_page_file(
|
||||
wiki_root,
|
||||
"Codex-Project-Index",
|
||||
"\n".join(index_lines) + "\n",
|
||||
overwrite_unmanaged=True,
|
||||
)
|
||||
else:
|
||||
print(f"skipped {target.owner}/{target.repo} wiki:Codex-Project-Index during page-limited sync")
|
||||
if not git_has_changes(wiki_root):
|
||||
print(f"unchanged {target.owner}/{target.repo} wiki repository")
|
||||
return
|
||||
run_git(wiki_root, "add", "-A")
|
||||
if not git_has_staged_changes(wiki_root):
|
||||
print(f"unchanged {target.owner}/{target.repo} wiki repository")
|
||||
return
|
||||
ensure_git_identity(wiki_root)
|
||||
run_git(wiki_root, "commit", "-m", commit_message)
|
||||
run_git(wiki_root, "push")
|
||||
print(f"pushed {target.owner}/{target.repo} wiki repository")
|
||||
|
||||
|
||||
def prepare_wiki_checkout(repo: RepoInfo, *, cache_dir: pathlib.Path) -> pathlib.Path:
|
||||
remote = wiki_remote_for_repo(repo.root)
|
||||
cache_dir.mkdir(parents=True, exist_ok=True)
|
||||
checkout = cache_dir / f"{repo.owner}-{repo.repo}.wiki"
|
||||
if (checkout / ".git").exists():
|
||||
run_git(checkout, "remote", "set-url", "origin", remote)
|
||||
run_git(checkout, "fetch", "origin")
|
||||
branch = current_branch(checkout)
|
||||
if branch:
|
||||
run_git(checkout, "reset", "--hard", f"origin/{branch}")
|
||||
else:
|
||||
run_git(checkout, "reset", "--hard")
|
||||
run_git(checkout, "clean", "-fd")
|
||||
return checkout
|
||||
if checkout.exists():
|
||||
shutil.rmtree(checkout)
|
||||
result = subprocess.run(
|
||||
["git", "clone", remote, str(checkout)],
|
||||
check=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode == 0:
|
||||
return checkout
|
||||
checkout.mkdir(parents=True, exist_ok=True)
|
||||
run_git(checkout, "init")
|
||||
run_git(checkout, "remote", "add", "origin", remote)
|
||||
run_git(checkout, "checkout", "-b", "master")
|
||||
return checkout
|
||||
|
||||
|
||||
def wiki_remote_for_repo(repo_root_path: pathlib.Path) -> str:
|
||||
result = subprocess.run(
|
||||
["git", "-C", str(repo_root_path), "remote", "get-url", "origin"],
|
||||
check=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode != 0 or not result.stdout.strip():
|
||||
raise GiteaError(f"Could not read origin remote for {repo_root_path}")
|
||||
remote = result.stdout.strip()
|
||||
if remote.endswith(".git"):
|
||||
return f"{remote[:-4]}.wiki.git"
|
||||
return f"{remote}.wiki.git"
|
||||
|
||||
|
||||
def write_wiki_page_file(wiki_root: pathlib.Path, title: str, content: str, *, overwrite_unmanaged: bool) -> str:
|
||||
path = wiki_root / f"{wiki_file_stem(title)}.md"
|
||||
if path.exists():
|
||||
current = read_text(path)
|
||||
if current == content:
|
||||
return "unchanged"
|
||||
if MANAGED_MARKER not in current and not overwrite_unmanaged:
|
||||
return "skipped unmanaged"
|
||||
path.write_text(content, encoding="utf-8")
|
||||
return "updated"
|
||||
|
||||
|
||||
def wiki_file_stem(title: str) -> str:
|
||||
return re.sub(r"[/\\]+", "-", title).strip() or "Home"
|
||||
|
||||
|
||||
def git_has_changes(root: pathlib.Path) -> bool:
|
||||
result = subprocess.run(
|
||||
["git", "-C", str(root), "status", "--porcelain"],
|
||||
check=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
raise GiteaError(f"git status failed in {root}: {result.stderr.strip()}")
|
||||
return bool(result.stdout.strip())
|
||||
|
||||
|
||||
def git_has_staged_changes(root: pathlib.Path) -> bool:
|
||||
result = subprocess.run(
|
||||
["git", "-C", str(root), "diff", "--cached", "--quiet"],
|
||||
check=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode == 0:
|
||||
return False
|
||||
if result.returncode == 1:
|
||||
return True
|
||||
raise GiteaError(f"git diff --cached failed in {root}: {result.stderr.strip()}")
|
||||
|
||||
|
||||
def current_branch(root: pathlib.Path) -> str:
|
||||
result = subprocess.run(
|
||||
["git", "-C", str(root), "rev-parse", "--abbrev-ref", "HEAD"],
|
||||
check=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
branch = result.stdout.strip()
|
||||
if result.returncode != 0 or branch == "HEAD":
|
||||
return ""
|
||||
return branch
|
||||
|
||||
|
||||
def ensure_git_identity(root: pathlib.Path) -> None:
|
||||
if not git_config_value(root, "user.email"):
|
||||
run_git(root, "config", "user.email", "codex@govoplan.local")
|
||||
if not git_config_value(root, "user.name"):
|
||||
run_git(root, "config", "user.name", "Codex")
|
||||
|
||||
|
||||
def git_config_value(root: pathlib.Path, key: str) -> str:
|
||||
result = subprocess.run(
|
||||
["git", "-C", str(root), "config", "--get", key],
|
||||
check=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
return result.stdout.strip() if result.returncode == 0 else ""
|
||||
|
||||
|
||||
def run_git(root: pathlib.Path, *args: str) -> None:
|
||||
result = subprocess.run(
|
||||
["git", "-C", str(root), *args],
|
||||
check=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
command = "git -C " + str(root) + " " + " ".join(args)
|
||||
raise GiteaError(f"{command} failed: {result.stderr.strip() or result.stdout.strip()}")
|
||||
|
||||
|
||||
def list_existing_wiki_pages(client: GiteaClient, owner: str, repo: str) -> list[dict[str, Any]]:
|
||||
try:
|
||||
return client.paginate(repo_path(owner, repo, "/wiki/pages"), limit=50)
|
||||
except GiteaError as exc:
|
||||
if "HTTP 404" in str(exc) and "/wiki/pages" in str(exc):
|
||||
return []
|
||||
raise
|
||||
|
||||
|
||||
def render_wiki_content(source: WikiSource) -> str:
|
||||
raw = read_text(source.path)
|
||||
fingerprint = hashlib.sha256(raw.encode("utf-8")).hexdigest()[:24]
|
||||
header = [
|
||||
f"{MANAGED_MARKER}{fingerprint} -->",
|
||||
"",
|
||||
f"> Mirrored from `{source.path}`.",
|
||||
f"> Origin: `{source.origin}`.",
|
||||
"> Active tasks and changing state belong in Gitea issues; this wiki page is durable project context.",
|
||||
"",
|
||||
"---",
|
||||
"",
|
||||
]
|
||||
if source.path.suffix.lower() == ".csv":
|
||||
return "\n".join(header + ["```csv", raw.rstrip(), "```", ""])
|
||||
return "\n".join(header) + raw.rstrip() + "\n"
|
||||
|
||||
|
||||
def put_wiki_page(
|
||||
client: GiteaClient,
|
||||
owner: str,
|
||||
repo: str,
|
||||
title: str,
|
||||
content: str,
|
||||
existing_page_names: dict[str, str],
|
||||
*,
|
||||
overwrite_unmanaged: bool,
|
||||
) -> bool:
|
||||
body = {
|
||||
"title": title,
|
||||
"content_base64": base64.b64encode(content.encode("utf-8")).decode("ascii"),
|
||||
"message": f"Sync {title} from project files",
|
||||
}
|
||||
if title in existing_page_names:
|
||||
page_name = existing_page_names[title]
|
||||
current = client.request_json("GET", repo_path(owner, repo, f"/wiki/page/{quote_wiki_page_name(page_name)}"))
|
||||
current_content = decode_wiki_content(current)
|
||||
if current_content == content:
|
||||
return False
|
||||
if MANAGED_MARKER not in current_content and not overwrite_unmanaged:
|
||||
print(f"skip unmanaged existing wiki page {owner}/{repo}:{title}")
|
||||
return False
|
||||
client.request_json("PATCH", repo_path(owner, repo, f"/wiki/page/{quote_wiki_page_name(page_name)}"), body=body)
|
||||
return True
|
||||
client.request_json("POST", repo_path(owner, repo, "/wiki/new"), body=body)
|
||||
existing_page_names[title] = title
|
||||
return True
|
||||
|
||||
|
||||
def quote_wiki_page_name(value: str) -> str:
|
||||
return urllib.parse.quote(value, safe="+")
|
||||
|
||||
|
||||
def decode_wiki_content(page: dict[str, Any]) -> str:
|
||||
encoded = str(page.get("content_base64") or "")
|
||||
if not encoded:
|
||||
return ""
|
||||
return base64.b64decode(encoded).decode("utf-8")
|
||||
|
||||
|
||||
def read_text(path: pathlib.Path) -> str:
|
||||
try:
|
||||
return path.read_text(encoding="utf-8")
|
||||
except UnicodeDecodeError:
|
||||
return path.read_text(encoding="latin-1")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
378
scripts/gitea-todo-import.py
Normal file
378
scripts/gitea-todo-import.py
Normal file
@@ -0,0 +1,378 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Preview or import inline TODO-style markers as Gitea issues."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
from gitea_common import GiteaClient, GiteaError, infer_target, load_dotenv, repo_path, repo_root, require_token
|
||||
|
||||
|
||||
MARKER_RE = re.compile(
|
||||
r"(?P<prefix>#|//|/\*|\*|--|<!--)?\s*"
|
||||
r"\b(?P<marker>TODO|FIXME|XXX|HACK)\b"
|
||||
r"\s*(?:\((?P<context>[^)]{1,120})\))?"
|
||||
r"\s*(?P<colon>:)?\s*(?P<text>.*)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
DEFAULT_EXCLUDES = (
|
||||
"!**/.git/**",
|
||||
"!**/.venv/**",
|
||||
"!**/node_modules/**",
|
||||
"!**/__pycache__/**",
|
||||
"!**/.module-test-build/**",
|
||||
"!**/dist/**",
|
||||
"!**/build/**",
|
||||
"!**/.cache/**",
|
||||
"!.gitea/**",
|
||||
"!docs/GITEA_ISSUES.md",
|
||||
"!scripts/gitea-todo-import.py",
|
||||
"!scripts/gitea-sync-labels.py",
|
||||
"!scripts/gitea-codex-note.py",
|
||||
"!scripts/gitea_common.py",
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--root", type=pathlib.Path, default=pathlib.Path.cwd(), help="repository root or child path")
|
||||
parser.add_argument("--remote", default="origin", help="git remote to use for target inference")
|
||||
parser.add_argument("--env-file", type=pathlib.Path, help="dotenv file to read before using GITEA_* values")
|
||||
parser.add_argument("--apply", action="store_true", help="create missing Gitea issues")
|
||||
parser.add_argument("--include-linked", action="store_true", help="include markers that already reference an issue")
|
||||
parser.add_argument(
|
||||
"--module-label",
|
||||
help="project/module label to apply; defaults to a known GovOPlaN mapping or module/core",
|
||||
)
|
||||
parser.add_argument("--no-area-labels", action="store_true", help="do not infer area/* labels from paths")
|
||||
parser.add_argument("--extra-label", action="append", default=[], help="additional label name to apply")
|
||||
parser.add_argument("--limit", type=int, default=0, help="maximum number of markers to process")
|
||||
parser.add_argument("--json", action="store_true", help="print issue previews as JSON")
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
root = repo_root(args.root)
|
||||
load_dotenv(args.env_file or root / ".env")
|
||||
target = infer_target(root, args.remote)
|
||||
token = require_token() if args.apply else os.environ.get("GITEA_TOKEN")
|
||||
|
||||
markers = scan_markers(root, include_linked=args.include_linked)
|
||||
if args.limit > 0:
|
||||
markers = markers[: args.limit]
|
||||
|
||||
module_label = args.module_label or infer_module_label(root)
|
||||
previews = [
|
||||
build_issue_preview(
|
||||
target.owner,
|
||||
target.repo,
|
||||
marker,
|
||||
module_label=module_label,
|
||||
infer_areas=not args.no_area_labels,
|
||||
extra_labels=args.extra_label,
|
||||
)
|
||||
for marker in markers
|
||||
]
|
||||
|
||||
if args.json:
|
||||
print(json.dumps([preview.as_dict() for preview in previews], indent=2, sort_keys=True))
|
||||
else:
|
||||
print(f"Target: {target.display}")
|
||||
print(f"Scanned root: {root}")
|
||||
print(f"Found {len(previews)} importable marker(s).")
|
||||
for preview in previews:
|
||||
print(f"- {preview.title}")
|
||||
print(f" {preview.location}")
|
||||
print(f" labels: {', '.join(preview.labels)}")
|
||||
|
||||
if not args.apply:
|
||||
if not previews:
|
||||
return 0
|
||||
print("Dry run only. Re-run with --apply and GITEA_TOKEN to create issues.")
|
||||
return 0
|
||||
|
||||
if not previews:
|
||||
print("No issues to create.")
|
||||
return 0
|
||||
|
||||
client = GiteaClient(target, token)
|
||||
label_ids = load_label_ids(client, target.owner, target.repo)
|
||||
missing_labels = sorted({label for preview in previews for label in preview.labels if label not in label_ids})
|
||||
if missing_labels:
|
||||
joined = ", ".join(missing_labels)
|
||||
raise GiteaError(f"missing labels: {joined}. Run scripts/gitea-sync-labels.py --apply first.")
|
||||
|
||||
existing_fingerprints = load_existing_fingerprints(client, target.owner, target.repo)
|
||||
created = 0
|
||||
skipped = 0
|
||||
for preview in previews:
|
||||
if preview.fingerprint in existing_fingerprints:
|
||||
print(f"skip existing {preview.location}")
|
||||
skipped += 1
|
||||
continue
|
||||
issue = client.request_json(
|
||||
"POST",
|
||||
repo_path(target.owner, target.repo, "/issues"),
|
||||
body={
|
||||
"title": preview.title,
|
||||
"body": preview.body,
|
||||
"labels": [label_ids[label] for label in preview.labels],
|
||||
},
|
||||
)
|
||||
print(f"created #{issue.get('number') or issue.get('index')}: {preview.title}")
|
||||
created += 1
|
||||
|
||||
print(f"Created {created} issue(s), skipped {skipped} existing issue(s).")
|
||||
return 0
|
||||
except GiteaError as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
class Marker:
|
||||
def __init__(self, path: pathlib.Path, line: int, column: int, marker: str, context: str, text: str, raw: str) -> None:
|
||||
self.path = path
|
||||
self.line = line
|
||||
self.column = column
|
||||
self.marker = marker.upper()
|
||||
self.context = context
|
||||
self.text = text
|
||||
self.raw = raw.rstrip("\n")
|
||||
|
||||
@property
|
||||
def location(self) -> str:
|
||||
return f"{self.path}:{self.line}"
|
||||
|
||||
|
||||
class IssuePreview:
|
||||
def __init__(self, title: str, body: str, labels: list[str], fingerprint: str, location: str) -> None:
|
||||
self.title = title
|
||||
self.body = body
|
||||
self.labels = labels
|
||||
self.fingerprint = fingerprint
|
||||
self.location = location
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"title": self.title,
|
||||
"body": self.body,
|
||||
"labels": self.labels,
|
||||
"fingerprint": self.fingerprint,
|
||||
"location": self.location,
|
||||
}
|
||||
|
||||
|
||||
def scan_markers(root: pathlib.Path, *, include_linked: bool) -> list[Marker]:
|
||||
command = [
|
||||
"rg",
|
||||
"--json",
|
||||
"--hidden",
|
||||
"--line-number",
|
||||
"--column",
|
||||
"-e",
|
||||
r"\b(TODO|FIXME|XXX|HACK)\b",
|
||||
]
|
||||
for pattern in DEFAULT_EXCLUDES:
|
||||
command.extend(["--glob", pattern])
|
||||
command.append(str(root))
|
||||
|
||||
result = subprocess.run(
|
||||
command,
|
||||
check=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode not in {0, 1}:
|
||||
raise GiteaError(f"rg failed: {result.stderr.strip()}")
|
||||
|
||||
markers: list[Marker] = []
|
||||
for line in result.stdout.splitlines():
|
||||
event = json.loads(line)
|
||||
if event.get("type") != "match":
|
||||
continue
|
||||
data = event["data"]
|
||||
raw_line = data["lines"]["text"]
|
||||
match = MARKER_RE.search(raw_line)
|
||||
if not match:
|
||||
continue
|
||||
if not _looks_like_marker(match):
|
||||
continue
|
||||
context = (match.group("context") or "").strip()
|
||||
if not include_linked and _already_linked(context, raw_line):
|
||||
continue
|
||||
markers.append(
|
||||
Marker(
|
||||
path=pathlib.Path(data["path"]["text"]).resolve().relative_to(root),
|
||||
line=int(data["line_number"]),
|
||||
column=int(data.get("submatches", [{}])[0].get("start", 0)) + 1,
|
||||
marker=match.group("marker"),
|
||||
context=context,
|
||||
text=clean_marker_text(match.group("text") or raw_line),
|
||||
raw=raw_line,
|
||||
)
|
||||
)
|
||||
return markers
|
||||
|
||||
|
||||
def build_issue_preview(
|
||||
owner: str,
|
||||
repo: str,
|
||||
marker: Marker,
|
||||
*,
|
||||
module_label: str,
|
||||
infer_areas: bool,
|
||||
extra_labels: list[str],
|
||||
) -> IssuePreview:
|
||||
fingerprint = marker_fingerprint(owner, repo, marker)
|
||||
summary = marker.text or f"review {marker.location}"
|
||||
title = truncate_title(f"{marker.marker}: {summary} ({marker.location})")
|
||||
labels = sorted(set(default_labels(marker, module_label=module_label, infer_areas=infer_areas) + list(extra_labels)))
|
||||
body = "\n".join(
|
||||
[
|
||||
f"<!-- codex-todo-fingerprint:{fingerprint} -->",
|
||||
"",
|
||||
"Imported from an inline marker.",
|
||||
"",
|
||||
f"- Repository: `{owner}/{repo}`",
|
||||
f"- Location: `{marker.location}`",
|
||||
f"- Marker: `{marker.marker}`",
|
||||
f"- Context: `{marker.context or 'none'}`",
|
||||
"",
|
||||
"Current line:",
|
||||
"",
|
||||
"```text",
|
||||
marker.raw,
|
||||
"```",
|
||||
"",
|
||||
"Migration rule:",
|
||||
"",
|
||||
"- Keep this Gitea issue as the canonical backlog item.",
|
||||
"- When touching this code, remove the inline marker or replace it with a short issue reference.",
|
||||
]
|
||||
)
|
||||
return IssuePreview(title=title, body=body, labels=labels, fingerprint=fingerprint, location=marker.location)
|
||||
|
||||
|
||||
def clean_marker_text(text: str) -> str:
|
||||
cleaned = text.strip()
|
||||
cleaned = re.sub(r"\s*(?:#|//|/\*|\*|--)\s*$", "", cleaned).strip()
|
||||
cleaned = re.sub(r"\s*\*/\s*$", "", cleaned).strip()
|
||||
return cleaned
|
||||
|
||||
|
||||
def default_labels(marker: Marker, *, module_label: str, infer_areas: bool) -> list[str]:
|
||||
labels = ["source/todo-scan", "status/triage", module_label, marker_type_label(marker.marker)]
|
||||
area = infer_area_label(marker.path) if infer_areas else ""
|
||||
if area:
|
||||
labels.append(area)
|
||||
return labels
|
||||
|
||||
|
||||
def marker_type_label(marker: str) -> str:
|
||||
if marker == "FIXME":
|
||||
return "type/bug"
|
||||
if marker in {"HACK", "XXX"}:
|
||||
return "type/debt"
|
||||
return "type/task"
|
||||
|
||||
|
||||
def infer_area_label(path: pathlib.Path) -> str:
|
||||
text = path.as_posix()
|
||||
if text.startswith("webui/"):
|
||||
return "area/webui"
|
||||
if text.startswith("alembic/"):
|
||||
return "area/migrations"
|
||||
if text.startswith("docs/"):
|
||||
return "area/docs"
|
||||
if text.startswith("scripts/"):
|
||||
return "area/devex"
|
||||
if "/rbac" in text or "permission" in text:
|
||||
return "area/rbac"
|
||||
if "/access" in text or "auth" in text or "session" in text:
|
||||
return "area/auth"
|
||||
if "tenant" in text:
|
||||
return "area/tenancy"
|
||||
if "governance" in text or "privacy" in text or "audit" in text or "retention" in text:
|
||||
return "area/governance"
|
||||
if "module" in text:
|
||||
return "area/module-system"
|
||||
if "migration" in text:
|
||||
return "area/migrations"
|
||||
if "router" in text or "api" in text:
|
||||
return "area/api"
|
||||
if "db" in text or "model" in text or "persistence" in text:
|
||||
return "area/db"
|
||||
return ""
|
||||
|
||||
|
||||
def infer_module_label(root: pathlib.Path) -> str:
|
||||
known = {
|
||||
"govoplan-core": "module/core",
|
||||
"govoplan-access": "module/access",
|
||||
"govoplan-admin": "module/admin",
|
||||
"govoplan-tenancy": "module/tenancy",
|
||||
"govoplan-policy": "module/policy",
|
||||
"govoplan-audit": "module/audit",
|
||||
"govoplan-mail": "module/mail",
|
||||
"govoplan-files": "module/files",
|
||||
"govoplan-campaign": "module/campaign",
|
||||
"govoplan-web": "module/web",
|
||||
}
|
||||
if root.name in known:
|
||||
return known[root.name]
|
||||
if root.name.startswith("govoplan-"):
|
||||
suffix = root.name.removeprefix("govoplan-")
|
||||
if suffix:
|
||||
return f"module/{suffix}"
|
||||
return "module/core"
|
||||
|
||||
|
||||
def marker_fingerprint(owner: str, repo: str, marker: Marker) -> str:
|
||||
stable = "\n".join([owner, repo, marker.path.as_posix(), marker.marker, marker.context, marker.text])
|
||||
return hashlib.sha256(stable.encode("utf-8")).hexdigest()[:24]
|
||||
|
||||
|
||||
def truncate_title(title: str, limit: int = 180) -> str:
|
||||
if len(title) <= limit:
|
||||
return title
|
||||
return f"{title[: limit - 1].rstrip()}..."
|
||||
|
||||
|
||||
def load_label_ids(client: GiteaClient, owner: str, repo: str) -> dict[str, int]:
|
||||
labels = client.paginate(repo_path(owner, repo, "/labels"))
|
||||
return {str(label["name"]): int(label["id"]) for label in labels if "name" in label and "id" in label}
|
||||
|
||||
|
||||
def load_existing_fingerprints(client: GiteaClient, owner: str, repo: str) -> set[str]:
|
||||
issues = client.paginate(
|
||||
repo_path(owner, repo, "/issues"),
|
||||
query={"state": "all", "labels": "source/todo-scan"},
|
||||
)
|
||||
fingerprints: set[str] = set()
|
||||
for issue in issues:
|
||||
body = str(issue.get("body") or "")
|
||||
for match in re.finditer(r"(?:codex|govoplan)-todo-fingerprint:([0-9a-f]{24})", body):
|
||||
fingerprints.add(match.group(1))
|
||||
return fingerprints
|
||||
|
||||
|
||||
def _already_linked(context: str, raw_line: str) -> bool:
|
||||
linked_text = f"{context} {raw_line}".lower()
|
||||
return "gitea#" in linked_text or "issue#" in linked_text or re.search(r"#\d+", linked_text) is not None
|
||||
|
||||
|
||||
def _looks_like_marker(match: re.Match[str]) -> bool:
|
||||
text = (match.group("text") or "").strip()
|
||||
return bool(match.group("context") or match.group("colon") or (match.group("prefix") and text))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
249
scripts/gitea_common.py
Normal file
249
scripts/gitea_common.py
Normal file
@@ -0,0 +1,249 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Shared helpers for Gitea maintenance scripts."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import subprocess
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from typing import Any
|
||||
|
||||
|
||||
class GiteaError(RuntimeError):
|
||||
"""Raised when a Gitea API request fails."""
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class RepoTarget:
|
||||
base_url: str
|
||||
owner: str
|
||||
repo: str
|
||||
|
||||
@property
|
||||
def display(self) -> str:
|
||||
return f"{self.base_url.rstrip('/')}/{self.owner}/{self.repo}"
|
||||
|
||||
@property
|
||||
def api_base(self) -> str:
|
||||
return f"{self.base_url.rstrip('/')}/api/v1"
|
||||
|
||||
|
||||
def repo_root(start: pathlib.Path) -> pathlib.Path:
|
||||
result = subprocess.run(
|
||||
["git", "-C", str(start), "rev-parse", "--show-toplevel"],
|
||||
check=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
return start.resolve()
|
||||
return pathlib.Path(result.stdout.strip()).resolve()
|
||||
|
||||
|
||||
def infer_target(root: pathlib.Path, remote_name: str = "origin") -> RepoTarget:
|
||||
env_url = os.environ.get("GITEA_URL")
|
||||
env_owner = os.environ.get("GITEA_OWNER")
|
||||
env_repo = os.environ.get("GITEA_REPO")
|
||||
|
||||
remote_url = _git_remote(root, remote_name)
|
||||
inferred_url, inferred_owner, inferred_repo = _parse_remote(remote_url)
|
||||
|
||||
base_url = (env_url or inferred_url).rstrip("/")
|
||||
owner = env_owner or inferred_owner
|
||||
repo = env_repo or inferred_repo
|
||||
|
||||
missing = [
|
||||
name
|
||||
for name, value in (
|
||||
("GITEA_URL", base_url),
|
||||
("GITEA_OWNER", owner),
|
||||
("GITEA_REPO", repo),
|
||||
)
|
||||
if not value
|
||||
]
|
||||
if missing:
|
||||
joined = ", ".join(missing)
|
||||
raise GiteaError(
|
||||
f"Could not infer Gitea target. Set {joined}, or configure git remote {remote_name!r}."
|
||||
)
|
||||
return RepoTarget(base_url=base_url, owner=owner, repo=_strip_git_suffix(repo))
|
||||
|
||||
|
||||
def quote_path(value: str) -> str:
|
||||
return urllib.parse.quote(value, safe="")
|
||||
|
||||
|
||||
class GiteaClient:
|
||||
def __init__(self, target: RepoTarget, token: str | None) -> None:
|
||||
self.target = target
|
||||
self.token = token
|
||||
|
||||
def request_json(
|
||||
self,
|
||||
method: str,
|
||||
path: str,
|
||||
*,
|
||||
body: dict[str, Any] | None = None,
|
||||
query: dict[str, Any] | None = None,
|
||||
) -> Any:
|
||||
url = f"{self.target.api_base}{path}"
|
||||
if query:
|
||||
url = f"{url}?{urllib.parse.urlencode(query, doseq=True)}"
|
||||
|
||||
data = None
|
||||
headers = {
|
||||
"Accept": "application/json",
|
||||
"User-Agent": "codex-gitea-maintenance",
|
||||
}
|
||||
if self.token:
|
||||
headers["Authorization"] = f"token {self.token}"
|
||||
if body is not None:
|
||||
data = json.dumps(body).encode("utf-8")
|
||||
headers["Content-Type"] = "application/json"
|
||||
|
||||
request = urllib.request.Request(url, data=data, headers=headers, method=method)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=30) as response:
|
||||
payload = response.read().decode("utf-8")
|
||||
if not payload:
|
||||
return None
|
||||
return json.loads(payload)
|
||||
except urllib.error.HTTPError as exc:
|
||||
message = exc.read().decode("utf-8", errors="replace")
|
||||
raise GiteaError(f"{method} {url} failed with HTTP {exc.code}: {message}") from exc
|
||||
except urllib.error.URLError as exc:
|
||||
raise GiteaError(f"{method} {url} failed: {exc.reason}") from exc
|
||||
|
||||
def paginate(
|
||||
self,
|
||||
path: str,
|
||||
*,
|
||||
query: dict[str, Any] | None = None,
|
||||
limit: int = 100,
|
||||
) -> list[dict[str, Any]]:
|
||||
items: list[dict[str, Any]] = []
|
||||
page = 1
|
||||
effective_limit = min(limit, 50)
|
||||
seen_pages: set[tuple[Any, ...]] = set()
|
||||
while True:
|
||||
page_query = dict(query or {})
|
||||
page_query.update({"page": page, "limit": effective_limit})
|
||||
payload = self.request_json("GET", path, query=page_query)
|
||||
if not isinstance(payload, list):
|
||||
raise GiteaError(f"Expected a list from {path}, got {type(payload).__name__}")
|
||||
if not payload:
|
||||
return items
|
||||
signature = tuple(
|
||||
item.get("id") or item.get("number") or item.get("index") or item.get("name")
|
||||
for item in payload
|
||||
if isinstance(item, dict)
|
||||
)
|
||||
if signature in seen_pages:
|
||||
return items
|
||||
seen_pages.add(signature)
|
||||
items.extend(payload)
|
||||
if len(payload) < effective_limit:
|
||||
return items
|
||||
page += 1
|
||||
|
||||
|
||||
def load_json(path: pathlib.Path) -> Any:
|
||||
with path.open("r", encoding="utf-8") as handle:
|
||||
return json.load(handle)
|
||||
|
||||
|
||||
def load_dotenv(path: pathlib.Path | None) -> list[str]:
|
||||
if path is None or not path.exists():
|
||||
return []
|
||||
|
||||
loaded: list[str] = []
|
||||
for line_number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1):
|
||||
stripped = line.strip()
|
||||
if not stripped or stripped.startswith("#"):
|
||||
continue
|
||||
if stripped.startswith("export "):
|
||||
stripped = stripped[len("export ") :].strip()
|
||||
if "=" not in stripped:
|
||||
raise GiteaError(f"{path}:{line_number}: expected KEY=value")
|
||||
key, value = stripped.split("=", 1)
|
||||
key = key.strip()
|
||||
value = _strip_env_value(value.strip())
|
||||
if not re.match(r"^[A-Za-z_][A-Za-z0-9_]*$", key):
|
||||
raise GiteaError(f"{path}:{line_number}: invalid environment key {key!r}")
|
||||
if key not in os.environ:
|
||||
os.environ[key] = value
|
||||
loaded.append(key)
|
||||
return loaded
|
||||
|
||||
|
||||
def require_token() -> str:
|
||||
token = os.environ.get("GITEA_TOKEN")
|
||||
if not token:
|
||||
raise GiteaError(
|
||||
"GITEA_TOKEN is required when using --apply. "
|
||||
"Set it in the environment, the target .env, or --env-file."
|
||||
)
|
||||
return token
|
||||
|
||||
|
||||
def repo_path(owner: str, repo: str, suffix: str) -> str:
|
||||
return f"/repos/{quote_path(owner)}/{quote_path(repo)}{suffix}"
|
||||
|
||||
|
||||
def _git_remote(root: pathlib.Path, remote_name: str) -> str:
|
||||
result = subprocess.run(
|
||||
["git", "-C", str(root), "remote", "get-url", remote_name],
|
||||
check=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
return ""
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def _parse_remote(remote_url: str) -> tuple[str, str, str]:
|
||||
if not remote_url:
|
||||
return "", "", ""
|
||||
|
||||
parsed = urllib.parse.urlparse(remote_url)
|
||||
if parsed.scheme in {"http", "https", "ssh"} and parsed.netloc:
|
||||
path_parts = [part for part in parsed.path.strip("/").split("/") if part]
|
||||
owner, repo = _owner_repo_from_parts(path_parts)
|
||||
if parsed.scheme in {"http", "https"}:
|
||||
prefix = "/".join(path_parts[:-2])
|
||||
base_path = f"/{prefix}" if prefix else ""
|
||||
return f"{parsed.scheme}://{parsed.netloc}{base_path}", owner, repo
|
||||
return f"https://{parsed.hostname or parsed.netloc}", owner, repo
|
||||
|
||||
scp_like = re.match(r"^(?:[^@]+@)?(?P<host>[^:]+):(?P<path>.+)$", remote_url)
|
||||
if scp_like:
|
||||
path_parts = [part for part in scp_like.group("path").strip("/").split("/") if part]
|
||||
owner, repo = _owner_repo_from_parts(path_parts)
|
||||
return f"https://{scp_like.group('host')}", owner, repo
|
||||
|
||||
return "", "", ""
|
||||
|
||||
|
||||
def _owner_repo_from_parts(path_parts: list[str]) -> tuple[str, str]:
|
||||
if len(path_parts) < 2:
|
||||
return "", ""
|
||||
return path_parts[-2], _strip_git_suffix(path_parts[-1])
|
||||
|
||||
|
||||
def _strip_git_suffix(value: str) -> str:
|
||||
return value[:-4] if value.endswith(".git") else value
|
||||
|
||||
|
||||
def _strip_env_value(value: str) -> str:
|
||||
if len(value) >= 2 and value[0] == value[-1] and value[0] in {"'", '"'}:
|
||||
return value[1:-1]
|
||||
return value
|
||||
@@ -12,6 +12,10 @@ BACKEND_PORT="${GOVOPLAN_BACKEND_PORT:-8000}"
|
||||
FRONTEND_HOST="${GOVOPLAN_FRONTEND_HOST:-127.0.0.1}"
|
||||
FRONTEND_PORT="${GOVOPLAN_FRONTEND_PORT:-5173}"
|
||||
OPEN_BROWSER="${OPEN_BROWSER:-1}"
|
||||
FRONTEND_FORCE_RELOAD="${GOVOPLAN_FRONTEND_FORCE_RELOAD:-1}"
|
||||
FRONTEND_CLEAR_VITE_CACHE="${GOVOPLAN_FRONTEND_CLEAR_VITE_CACHE:-1}"
|
||||
FRONTEND_USE_POLLING="${GOVOPLAN_FRONTEND_USE_POLLING:-1}"
|
||||
FRONTEND_POLLING_INTERVAL="${GOVOPLAN_FRONTEND_POLLING_INTERVAL:-500}"
|
||||
|
||||
LOG_DIR="$ROOT/runtime/dev-launcher"
|
||||
BACKEND_LOG="$LOG_DIR/backend.log"
|
||||
@@ -84,6 +88,10 @@ mkdir -p "$LOG_DIR"
|
||||
: > "$BACKEND_LOG"
|
||||
: > "$FRONTEND_LOG"
|
||||
|
||||
if [ "$FRONTEND_CLEAR_VITE_CACHE" = "1" ]; then
|
||||
rm -rf "$WEBUI_ROOT/node_modules/.vite" "$WEBUI_ROOT/node_modules/.vite-temp"
|
||||
fi
|
||||
|
||||
port_is_free "$BACKEND_HOST" "$BACKEND_PORT" || fail "$BACKEND_URL is already in use"
|
||||
port_is_free "$FRONTEND_HOST" "$FRONTEND_PORT" || fail "$FRONTEND_URL is already in use"
|
||||
|
||||
@@ -103,9 +111,15 @@ wait_for_url "$BACKEND_URL/health" || {
|
||||
printf 'Starting GovOPlaN WebUI at %s\n' "$FRONTEND_URL"
|
||||
(
|
||||
cd "$WEBUI_ROOT"
|
||||
frontend_args=(run dev)
|
||||
if [ "$FRONTEND_FORCE_RELOAD" = "1" ]; then
|
||||
frontend_args+=(-- --force)
|
||||
fi
|
||||
PATH="$WEBUI_ROOT/node_modules/.bin:$NODE_BIN:$PATH" \
|
||||
CHOKIDAR_USEPOLLING="$FRONTEND_USE_POLLING" \
|
||||
CHOKIDAR_INTERVAL="$FRONTEND_POLLING_INTERVAL" \
|
||||
VITE_API_PROXY_TARGET="$BACKEND_URL" \
|
||||
"$NPM" run dev
|
||||
"$NPM" "${frontend_args[@]}"
|
||||
) >"$FRONTEND_LOG" 2>&1 &
|
||||
frontend_pid="$!"
|
||||
|
||||
@@ -130,6 +144,11 @@ Logs:
|
||||
Backend: $BACKEND_LOG
|
||||
WebUI: $FRONTEND_LOG
|
||||
|
||||
Frontend reload:
|
||||
Vite cache cleared: $FRONTEND_CLEAR_VITE_CACHE
|
||||
Vite --force: $FRONTEND_FORCE_RELOAD
|
||||
Watch polling: $FRONTEND_USE_POLLING (${FRONTEND_POLLING_INTERVAL}ms)
|
||||
|
||||
Press Ctrl+C to stop both processes.
|
||||
EOF
|
||||
|
||||
|
||||
259
scripts/publish-release-catalog.sh
Normal file
259
scripts/publish-release-catalog.sh
Normal file
@@ -0,0 +1,259 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
usage() {
|
||||
cat <<'USAGE'
|
||||
Usage:
|
||||
scripts/publish-release-catalog.sh --version <x.y.z> --catalog-signing-key <key-id=/path/private.pem> [options]
|
||||
|
||||
Generates a signed module package catalog for a GovOPlaN release, writes it into
|
||||
govoplan-web/public/catalogs/v1, validates it, optionally builds the website,
|
||||
and optionally commits/tags/pushes govoplan-web.
|
||||
|
||||
Options:
|
||||
--version <x.y.z> Release version without leading v.
|
||||
--channel <name> Catalog channel. Defaults to stable.
|
||||
--sequence <number> Monotonic channel sequence. Defaults to UTC timestamp.
|
||||
--expires-days <days> Catalog expiry window. Defaults to 90.
|
||||
--catalog-signing-key <key-id=/path/private.pem>
|
||||
Ed25519 private key. May be repeated for rotation.
|
||||
--core-root <path> govoplan-core checkout. Defaults to this script parent.
|
||||
--web-root <path> govoplan-web checkout. Defaults to ../govoplan-web.
|
||||
--public-base-url <url> Public website URL. Defaults to https://govoplan.add-ideas.de.
|
||||
--npm <path> npm executable used for optional web build.
|
||||
--build-web Run npm run build in govoplan-web after generating files.
|
||||
--commit Commit generated catalog files in govoplan-web.
|
||||
--tag Tag govoplan-web with catalog-v<x.y.z>. Implies --commit.
|
||||
--push Push govoplan-web branch and tag. Implies --commit.
|
||||
--remote <name> Git remote for push. Defaults to origin.
|
||||
--branch <name> Branch to push. Defaults to current govoplan-web branch.
|
||||
-n, --dry-run Print commands and validate inputs without writing git changes.
|
||||
-h, --help Show this help.
|
||||
USAGE
|
||||
}
|
||||
|
||||
fail() {
|
||||
echo "error: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
PARENT="$(dirname "$ROOT")"
|
||||
CORE_ROOT="$ROOT"
|
||||
WEB_ROOT="$PARENT/govoplan-web"
|
||||
VERSION=""
|
||||
CHANNEL="stable"
|
||||
SEQUENCE=""
|
||||
EXPIRES_DAYS="90"
|
||||
PUBLIC_BASE_URL="https://govoplan.add-ideas.de"
|
||||
REMOTE="origin"
|
||||
BRANCH=""
|
||||
BUILD_WEB=0
|
||||
COMMIT=0
|
||||
TAG_WEB=0
|
||||
PUSH=0
|
||||
DRY_RUN=0
|
||||
SIGNING_KEYS=()
|
||||
NPM_BIN="${NPM:-}"
|
||||
|
||||
if [[ -z "${PYTHON:-}" ]]; then
|
||||
if [[ -x "$ROOT/.venv/bin/python" ]]; then
|
||||
PYTHON="$ROOT/.venv/bin/python"
|
||||
else
|
||||
PYTHON="python3"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ -z "$NPM_BIN" ]]; then
|
||||
if [[ -x "/home/zemion/.nvm/versions/node/v22.22.3/bin/npm" ]]; then
|
||||
NPM_BIN="/home/zemion/.nvm/versions/node/v22.22.3/bin/npm"
|
||||
else
|
||||
NPM_BIN="npm"
|
||||
fi
|
||||
fi
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--version)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
VERSION="${2#v}"
|
||||
shift 2
|
||||
;;
|
||||
--channel)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
CHANNEL="$2"
|
||||
shift 2
|
||||
;;
|
||||
--sequence)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
SEQUENCE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--expires-days)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
EXPIRES_DAYS="$2"
|
||||
shift 2
|
||||
;;
|
||||
--catalog-signing-key)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
SIGNING_KEYS+=("$2")
|
||||
shift 2
|
||||
;;
|
||||
--core-root)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
CORE_ROOT="$(cd "$2" && pwd)"
|
||||
shift 2
|
||||
;;
|
||||
--web-root)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
WEB_ROOT="$(cd "$2" && pwd)"
|
||||
shift 2
|
||||
;;
|
||||
--public-base-url)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
PUBLIC_BASE_URL="$2"
|
||||
shift 2
|
||||
;;
|
||||
--npm)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
NPM_BIN="$2"
|
||||
shift 2
|
||||
;;
|
||||
--build-web)
|
||||
BUILD_WEB=1
|
||||
shift
|
||||
;;
|
||||
--commit)
|
||||
COMMIT=1
|
||||
shift
|
||||
;;
|
||||
--tag)
|
||||
TAG_WEB=1
|
||||
COMMIT=1
|
||||
shift
|
||||
;;
|
||||
--push)
|
||||
PUSH=1
|
||||
COMMIT=1
|
||||
shift
|
||||
;;
|
||||
--remote)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
REMOTE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--branch)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
BRANCH="$2"
|
||||
shift 2
|
||||
;;
|
||||
-n|--dry-run)
|
||||
DRY_RUN=1
|
||||
shift
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "Unknown argument: $1" >&2
|
||||
usage >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
[[ -n "$VERSION" ]] || fail "--version is required"
|
||||
[[ "$VERSION" =~ ^[0-9]+[.][0-9]+[.][0-9]+$ ]] || fail "version must be x.y.z: $VERSION"
|
||||
[[ ${#SIGNING_KEYS[@]} -gt 0 ]] || fail "at least one --catalog-signing-key is required"
|
||||
[[ -d "$CORE_ROOT/.git" ]] || fail "not a govoplan-core git repo: $CORE_ROOT"
|
||||
[[ -d "$WEB_ROOT/.git" ]] || fail "not a govoplan-web git repo: $WEB_ROOT"
|
||||
command -v "$PYTHON" >/dev/null 2>&1 || fail "Python not found: $PYTHON"
|
||||
|
||||
if [[ "$BUILD_WEB" -eq 1 ]]; then
|
||||
command -v "$NPM_BIN" >/dev/null 2>&1 || fail "npm not found: $NPM_BIN"
|
||||
fi
|
||||
|
||||
if [[ -z "$BRANCH" ]]; then
|
||||
BRANCH="$(git -C "$WEB_ROOT" symbolic-ref --quiet --short HEAD || true)"
|
||||
fi
|
||||
[[ -n "$BRANCH" ]] || fail "cannot infer govoplan-web branch; pass --branch"
|
||||
|
||||
CATALOG_PATH="$WEB_ROOT/public/catalogs/v1/channels/$CHANNEL.json"
|
||||
KEYRING_PATH="$WEB_ROOT/public/catalogs/v1/keyring.json"
|
||||
TAG_NAME="catalog-v$VERSION"
|
||||
|
||||
run() {
|
||||
printf '+'
|
||||
printf ' %q' "$@"
|
||||
printf '\n'
|
||||
if [[ "$DRY_RUN" -eq 0 ]]; then
|
||||
"$@"
|
||||
fi
|
||||
}
|
||||
|
||||
GEN_ARGS=(
|
||||
"$PYTHON" "$CORE_ROOT/scripts/generate-release-catalog.py"
|
||||
--version "$VERSION"
|
||||
--channel "$CHANNEL"
|
||||
--expires-days "$EXPIRES_DAYS"
|
||||
--catalog-output "$CATALOG_PATH"
|
||||
--keyring-output "$KEYRING_PATH"
|
||||
--public-base-url "$PUBLIC_BASE_URL"
|
||||
)
|
||||
if [[ -n "$SEQUENCE" ]]; then
|
||||
GEN_ARGS+=(--sequence "$SEQUENCE")
|
||||
fi
|
||||
for signing_key in "${SIGNING_KEYS[@]}"; do
|
||||
GEN_ARGS+=(--catalog-signing-key "$signing_key")
|
||||
done
|
||||
|
||||
run "${GEN_ARGS[@]}"
|
||||
|
||||
if [[ "$DRY_RUN" -eq 0 ]]; then
|
||||
GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE="$KEYRING_PATH" \
|
||||
"$PYTHON" -m govoplan_core.commands.module_installer \
|
||||
--validate-package-catalog "$CATALOG_PATH" \
|
||||
--require-signed-catalog \
|
||||
--approved-catalog-channel "$CHANNEL" \
|
||||
--format json >/dev/null
|
||||
else
|
||||
echo "+ GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE=$KEYRING_PATH $PYTHON -m govoplan_core.commands.module_installer --validate-package-catalog $CATALOG_PATH --require-signed-catalog --approved-catalog-channel $CHANNEL --format json"
|
||||
fi
|
||||
|
||||
if [[ "$BUILD_WEB" -eq 1 ]]; then
|
||||
run env PATH="$(dirname "$NPM_BIN"):$PATH" "$NPM_BIN" --prefix "$WEB_ROOT" run build
|
||||
fi
|
||||
|
||||
if [[ "$COMMIT" -eq 1 ]]; then
|
||||
run git -C "$WEB_ROOT" add "$CATALOG_PATH" "$KEYRING_PATH"
|
||||
if [[ "$DRY_RUN" -eq 0 ]]; then
|
||||
if git -C "$WEB_ROOT" diff --cached --quiet; then
|
||||
echo "No govoplan-web catalog changes to commit."
|
||||
else
|
||||
run git -C "$WEB_ROOT" commit -m "Publish GovOPlaN v$VERSION $CHANNEL catalog"
|
||||
fi
|
||||
else
|
||||
run git -C "$WEB_ROOT" commit -m "Publish GovOPlaN v$VERSION $CHANNEL catalog"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ "$TAG_WEB" -eq 1 ]]; then
|
||||
if git -C "$WEB_ROOT" rev-parse --quiet --verify "refs/tags/$TAG_NAME" >/dev/null; then
|
||||
fail "govoplan-web tag already exists: $TAG_NAME"
|
||||
fi
|
||||
run git -C "$WEB_ROOT" tag -a "$TAG_NAME" -m "Publish GovOPlaN v$VERSION $CHANNEL catalog"
|
||||
fi
|
||||
|
||||
if [[ "$PUSH" -eq 1 ]]; then
|
||||
if [[ "$TAG_WEB" -eq 1 ]]; then
|
||||
run git -C "$WEB_ROOT" push --atomic "$REMOTE" "HEAD:refs/heads/$BRANCH" "refs/tags/$TAG_NAME"
|
||||
else
|
||||
run git -C "$WEB_ROOT" push "$REMOTE" "HEAD:refs/heads/$BRANCH"
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "Catalog ready:"
|
||||
echo " $CATALOG_PATH"
|
||||
echo " $KEYRING_PATH"
|
||||
echo " URL: $PUBLIC_BASE_URL/catalogs/v1/channels/$CHANNEL.json"
|
||||
@@ -6,8 +6,9 @@ usage() {
|
||||
Usage:
|
||||
scripts/push-release-tag.sh [options]
|
||||
|
||||
Bumps all GovOPlaN package versions, commits all changes in all four repos,
|
||||
creates an annotated release tag in each repo, and pushes branch+tag.
|
||||
Bumps installable GovOPlaN package versions, commits pending changes in the
|
||||
GovOPlaN release/module repositories, creates an annotated release tag in each
|
||||
repo, and pushes branch+tag.
|
||||
|
||||
By default the script asks which version part to bump:
|
||||
major X.Y.Z -> (X+1).0.0
|
||||
@@ -23,16 +24,46 @@ Options:
|
||||
-m, --message <text> Commit message. Defaults to "Release v<x.y.z>".
|
||||
--tag-message <text> Annotated tag message. Defaults to the commit message.
|
||||
--skip-release-lock Do not regenerate core webui/package-lock.release.json.
|
||||
--publish-web-catalog After core/modules are pushed, publish a signed release
|
||||
catalog into govoplan-web.
|
||||
--web-root <path> govoplan-web checkout for --publish-web-catalog.
|
||||
Defaults to ../govoplan-web.
|
||||
--catalog-signing-key <key-id=/path/private.pem>
|
||||
Catalog signing key for --publish-web-catalog. May be
|
||||
repeated during key rotation.
|
||||
--catalog-channel <name>
|
||||
Catalog channel. Defaults to stable.
|
||||
--catalog-sequence <n> Monotonic catalog sequence. Defaults to UTC timestamp.
|
||||
--catalog-expires-days <days>
|
||||
Catalog expiry window. Defaults to 90.
|
||||
--catalog-public-url <url>
|
||||
Public website URL. Defaults to
|
||||
https://govoplan.add-ideas.de.
|
||||
--build-web-catalog Run npm run build in govoplan-web before committing the
|
||||
catalog update.
|
||||
--npm <path> npm executable for lockfile generation.
|
||||
-y, --yes Do not prompt before committing, tagging, and pushing.
|
||||
-n, --dry-run Print intended actions without changing files or git state.
|
||||
-h, --help Show this help.
|
||||
|
||||
Repos:
|
||||
govoplan-core
|
||||
Installable release packages:
|
||||
govoplan-access
|
||||
govoplan-admin
|
||||
govoplan-tenancy
|
||||
govoplan-policy
|
||||
govoplan-audit
|
||||
govoplan-files
|
||||
govoplan-mail
|
||||
govoplan-campaign
|
||||
govoplan-calendar
|
||||
govoplan-core
|
||||
|
||||
Roadmap/scaffold module repositories are tag-only until they have package
|
||||
metadata: addresses, appointments, cases, connectors, dms, erp,
|
||||
fit-connect, forms, identity-trust, idm, ledger, notifications, ops,
|
||||
payments, portal, reporting, scheduling, search, tasks, templates, workflow, xoev,
|
||||
xrechnung, and xta-osci.
|
||||
|
||||
Examples:
|
||||
scripts/push-release-tag.sh
|
||||
@@ -48,10 +79,47 @@ fail() {
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
PARENT="$(dirname "$ROOT")"
|
||||
REPOS=(
|
||||
PACKAGE_MODULE_REPOS=(
|
||||
"$PARENT/govoplan-access"
|
||||
"$PARENT/govoplan-admin"
|
||||
"$PARENT/govoplan-tenancy"
|
||||
"$PARENT/govoplan-policy"
|
||||
"$PARENT/govoplan-audit"
|
||||
"$PARENT/govoplan-files"
|
||||
"$PARENT/govoplan-mail"
|
||||
"$PARENT/govoplan-campaign"
|
||||
"$PARENT/govoplan-calendar"
|
||||
)
|
||||
TAG_ONLY_MODULE_REPOS=(
|
||||
"$PARENT/govoplan-addresses"
|
||||
"$PARENT/govoplan-appointments"
|
||||
"$PARENT/govoplan-cases"
|
||||
"$PARENT/govoplan-connectors"
|
||||
"$PARENT/govoplan-dms"
|
||||
"$PARENT/govoplan-erp"
|
||||
"$PARENT/govoplan-fit-connect"
|
||||
"$PARENT/govoplan-forms"
|
||||
"$PARENT/govoplan-identity-trust"
|
||||
"$PARENT/govoplan-idm"
|
||||
"$PARENT/govoplan-ledger"
|
||||
"$PARENT/govoplan-notifications"
|
||||
"$PARENT/govoplan-ops"
|
||||
"$PARENT/govoplan-payments"
|
||||
"$PARENT/govoplan-portal"
|
||||
"$PARENT/govoplan-reporting"
|
||||
"$PARENT/govoplan-scheduling"
|
||||
"$PARENT/govoplan-search"
|
||||
"$PARENT/govoplan-tasks"
|
||||
"$PARENT/govoplan-templates"
|
||||
"$PARENT/govoplan-workflow"
|
||||
"$PARENT/govoplan-xoev"
|
||||
"$PARENT/govoplan-xrechnung"
|
||||
"$PARENT/govoplan-xta-osci"
|
||||
)
|
||||
MODULE_REPOS=("${PACKAGE_MODULE_REPOS[@]}" "${TAG_ONLY_MODULE_REPOS[@]}")
|
||||
PACKAGE_REPOS=("${PACKAGE_MODULE_REPOS[@]}" "$ROOT")
|
||||
REPOS=(
|
||||
"${MODULE_REPOS[@]}"
|
||||
"$ROOT"
|
||||
)
|
||||
|
||||
@@ -65,6 +133,14 @@ DRY_RUN=0
|
||||
YES=0
|
||||
GENERATE_RELEASE_LOCK=1
|
||||
NPM_BIN="${NPM:-}"
|
||||
PUBLISH_WEB_CATALOG=0
|
||||
WEB_ROOT="$PARENT/govoplan-web"
|
||||
CATALOG_CHANNEL="stable"
|
||||
CATALOG_SEQUENCE=""
|
||||
CATALOG_EXPIRES_DAYS="90"
|
||||
CATALOG_PUBLIC_URL="https://govoplan.add-ideas.de"
|
||||
BUILD_WEB_CATALOG=0
|
||||
CATALOG_SIGNING_KEYS=()
|
||||
|
||||
if [[ -z "${PYTHON:-}" ]]; then
|
||||
if [[ -x "$ROOT/.venv/bin/python" ]]; then
|
||||
@@ -175,8 +251,8 @@ if version_count != 1:
|
||||
|
||||
if name != "govoplan-core":
|
||||
text, dependency_count = re.subn(
|
||||
r'govoplan-core>=\d+\.\d+\.\d+',
|
||||
f'govoplan-core>={new_version}',
|
||||
r'(govoplan-[A-Za-z0-9-]+>=)\d+\.\d+\.\d+',
|
||||
rf'\g<1>{new_version}',
|
||||
text,
|
||||
)
|
||||
if dependency_count < 1:
|
||||
@@ -195,7 +271,22 @@ update_manifest_version() {
|
||||
|
||||
case "$project_name" in
|
||||
govoplan-core)
|
||||
manifest_path="$repo/src/govoplan_core/access/manifest.py"
|
||||
return 0
|
||||
;;
|
||||
govoplan-access)
|
||||
manifest_path="$repo/src/govoplan_access/backend/manifest.py"
|
||||
;;
|
||||
govoplan-admin)
|
||||
manifest_path="$repo/src/govoplan_admin/backend/manifest.py"
|
||||
;;
|
||||
govoplan-tenancy)
|
||||
manifest_path="$repo/src/govoplan_tenancy/backend/manifest.py"
|
||||
;;
|
||||
govoplan-policy)
|
||||
manifest_path="$repo/src/govoplan_policy/backend/manifest.py"
|
||||
;;
|
||||
govoplan-audit)
|
||||
manifest_path="$repo/src/govoplan_audit/backend/manifest.py"
|
||||
;;
|
||||
govoplan-files)
|
||||
manifest_path="$repo/src/govoplan_files/backend/manifest.py"
|
||||
@@ -206,6 +297,9 @@ update_manifest_version() {
|
||||
govoplan-campaign)
|
||||
manifest_path="$repo/src/govoplan_campaign/backend/manifest.py"
|
||||
;;
|
||||
govoplan-calendar)
|
||||
manifest_path="$repo/src/govoplan_calendar/backend/manifest.py"
|
||||
;;
|
||||
*)
|
||||
fail "unknown project for manifest version update: $project_name"
|
||||
;;
|
||||
@@ -281,7 +375,7 @@ data = json.loads(path.read_text())
|
||||
data["version"] = new_version
|
||||
dependencies = data.setdefault("dependencies", {})
|
||||
for name, spec in list(dependencies.items()):
|
||||
if name in {"@govoplan/files-webui", "@govoplan/mail-webui", "@govoplan/campaign-webui"}:
|
||||
if name.startswith("@govoplan/") and isinstance(spec, str) and spec.startswith("git+"):
|
||||
dependencies[name] = re.sub(r"#v\d+\.\d+\.\d+$", f"#v{new_version}", spec)
|
||||
path.write_text(json.dumps(data, indent=2) + "\n")
|
||||
PYCODE
|
||||
@@ -420,6 +514,44 @@ while [[ $# -gt 0 ]]; do
|
||||
GENERATE_RELEASE_LOCK=0
|
||||
shift
|
||||
;;
|
||||
--publish-web-catalog)
|
||||
PUBLISH_WEB_CATALOG=1
|
||||
shift
|
||||
;;
|
||||
--web-root)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
WEB_ROOT="$2"
|
||||
shift 2
|
||||
;;
|
||||
--catalog-signing-key)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
CATALOG_SIGNING_KEYS+=("$2")
|
||||
shift 2
|
||||
;;
|
||||
--catalog-channel)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
CATALOG_CHANNEL="$2"
|
||||
shift 2
|
||||
;;
|
||||
--catalog-sequence)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
CATALOG_SEQUENCE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--catalog-expires-days)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
CATALOG_EXPIRES_DAYS="$2"
|
||||
shift 2
|
||||
;;
|
||||
--catalog-public-url)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
CATALOG_PUBLIC_URL="$2"
|
||||
shift 2
|
||||
;;
|
||||
--build-web-catalog)
|
||||
BUILD_WEB_CATALOG=1
|
||||
shift
|
||||
;;
|
||||
--npm)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
NPM_BIN="$2"
|
||||
@@ -448,6 +580,9 @@ done
|
||||
if [[ -n "$BUMP" && -n "$TARGET_VERSION" ]]; then
|
||||
fail "use either --bump or --version, not both"
|
||||
fi
|
||||
if [[ "$PUBLISH_WEB_CATALOG" -eq 1 && ${#CATALOG_SIGNING_KEYS[@]} -eq 0 ]]; then
|
||||
fail "--publish-web-catalog requires at least one --catalog-signing-key"
|
||||
fi
|
||||
|
||||
prompt_for_bump
|
||||
|
||||
@@ -457,19 +592,30 @@ fi
|
||||
if ! command -v "$NPM_BIN" >/dev/null 2>&1; then
|
||||
fail "npm executable not found: $NPM_BIN"
|
||||
fi
|
||||
if [[ "$PUBLISH_WEB_CATALOG" -eq 1 ]]; then
|
||||
[[ -d "$WEB_ROOT/.git" ]] || fail "govoplan-web repo not found: $WEB_ROOT"
|
||||
[[ -x "$ROOT/scripts/publish-release-catalog.sh" ]] || fail "missing executable catalog publisher: $ROOT/scripts/publish-release-catalog.sh"
|
||||
fi
|
||||
|
||||
declare -A PROJECT_NAMES
|
||||
declare -A OLD_VERSIONS
|
||||
declare -A BRANCHES
|
||||
declare -A REPO_KINDS
|
||||
|
||||
for repo in "${REPOS[@]}"; do
|
||||
[[ -d "$repo/.git" ]] || fail "not a git repo: $repo"
|
||||
[[ -f "$repo/pyproject.toml" ]] || fail "missing pyproject.toml: $repo"
|
||||
|
||||
PROJECT_NAMES["$repo"]="$(get_project_name "$repo")"
|
||||
OLD_VERSIONS["$repo"]="$(get_project_version "$repo")"
|
||||
if [[ -f "$repo/pyproject.toml" ]]; then
|
||||
REPO_KINDS["$repo"]="package"
|
||||
PROJECT_NAMES["$repo"]="$(get_project_name "$repo")"
|
||||
OLD_VERSIONS["$repo"]="$(get_project_version "$repo")"
|
||||
|
||||
validate_version "${OLD_VERSIONS[$repo]}" || fail "unsupported version ${OLD_VERSIONS[$repo]} in $repo"
|
||||
validate_version "${OLD_VERSIONS[$repo]}" || fail "unsupported version ${OLD_VERSIONS[$repo]} in $repo"
|
||||
else
|
||||
REPO_KINDS["$repo"]="tag-only"
|
||||
PROJECT_NAMES["$repo"]="$(basename "$repo")"
|
||||
OLD_VERSIONS["$repo"]="tag-only"
|
||||
fi
|
||||
|
||||
if [[ -n "$BRANCH" ]]; then
|
||||
BRANCHES["$repo"]="$BRANCH"
|
||||
@@ -482,7 +628,7 @@ for repo in "${REPOS[@]}"; do
|
||||
git -C "$repo" remote get-url "$REMOTE" >/dev/null || fail "remote $REMOTE not found in $repo"
|
||||
|
||||
upstream="$(git -C "$repo" rev-parse --abbrev-ref --symbolic-full-name '@{u}' 2>/dev/null || true)"
|
||||
if [[ -n "$upstream" ]]; then
|
||||
if [[ -n "$upstream" && "$upstream" != "@{u}" ]]; then
|
||||
behind_count="$(git -C "$repo" rev-list --count "HEAD..$upstream")"
|
||||
[[ "$behind_count" == "0" ]] || fail "$repo is behind $upstream by $behind_count commit(s); pull/rebase first"
|
||||
fi
|
||||
@@ -490,7 +636,7 @@ done
|
||||
|
||||
if [[ -z "$TARGET_VERSION" ]]; then
|
||||
BASE_VERSION="${OLD_VERSIONS[$ROOT]}"
|
||||
for repo in "${REPOS[@]}"; do
|
||||
for repo in "${PACKAGE_REPOS[@]}"; do
|
||||
if [[ "${OLD_VERSIONS[$repo]}" != "$BASE_VERSION" ]]; then
|
||||
fail "repo versions differ; pass --version x.y.z to choose an explicit target"
|
||||
fi
|
||||
@@ -500,7 +646,7 @@ fi
|
||||
|
||||
validate_version "$TARGET_VERSION" || fail "target version must be x.y.z: $TARGET_VERSION"
|
||||
|
||||
for repo in "${REPOS[@]}"; do
|
||||
for repo in "${PACKAGE_REPOS[@]}"; do
|
||||
if [[ "$TARGET_VERSION" == "${OLD_VERSIONS[$repo]}" ]]; then
|
||||
continue
|
||||
fi
|
||||
@@ -537,22 +683,33 @@ echo " version: $TARGET_VERSION"
|
||||
echo " tag: $TAG"
|
||||
echo " remote: $REMOTE"
|
||||
echo " commit message: $COMMIT_MESSAGE"
|
||||
echo " package repos: ${#PACKAGE_REPOS[@]} versioned"
|
||||
echo " tag-only module repos: ${#TAG_ONLY_MODULE_REPOS[@]} committed/tagged/pushed without package version files"
|
||||
if [[ "$GENERATE_RELEASE_LOCK" -eq 1 ]]; then
|
||||
echo " release lock: regenerate core webui/package-lock.release.json after module tags are pushed"
|
||||
else
|
||||
echo " release lock: skipped"
|
||||
fi
|
||||
if [[ "$PUBLISH_WEB_CATALOG" -eq 1 ]]; then
|
||||
echo " web catalog: publish $CATALOG_CHANNEL catalog to $WEB_ROOT after core tag is pushed"
|
||||
else
|
||||
echo " web catalog: skipped"
|
||||
fi
|
||||
echo
|
||||
|
||||
for repo in "${REPOS[@]}"; do
|
||||
echo "${PROJECT_NAMES[$repo]}: ${OLD_VERSIONS[$repo]} -> $TARGET_VERSION on ${BRANCHES[$repo]}"
|
||||
if [[ "${REPO_KINDS[$repo]}" == "package" ]]; then
|
||||
echo "${PROJECT_NAMES[$repo]}: ${OLD_VERSIONS[$repo]} -> $TARGET_VERSION on ${BRANCHES[$repo]}"
|
||||
else
|
||||
echo "${PROJECT_NAMES[$repo]}: tag-only $TAG on ${BRANCHES[$repo]}"
|
||||
fi
|
||||
git -C "$repo" status --short
|
||||
echo
|
||||
done
|
||||
|
||||
confirm_release
|
||||
|
||||
for repo in "${REPOS[@]}"; do
|
||||
for repo in "${PACKAGE_REPOS[@]}"; do
|
||||
if [[ "$DRY_RUN" -eq 1 ]]; then
|
||||
echo "Would update version files in $repo to $TARGET_VERSION"
|
||||
else
|
||||
@@ -562,20 +719,26 @@ done
|
||||
|
||||
refresh_development_webui_lock
|
||||
|
||||
for repo in "${REPOS[@]:0:3}"; do
|
||||
for repo in "${MODULE_REPOS[@]}"; do
|
||||
run git -C "$repo" add -A
|
||||
|
||||
if [[ "$DRY_RUN" -eq 0 ]]; then
|
||||
if git -C "$repo" diff --cached --quiet; then
|
||||
fail "no staged changes for ${PROJECT_NAMES[$repo]} after version bump"
|
||||
if [[ "${REPO_KINDS[$repo]}" == "package" ]]; then
|
||||
fail "no staged changes for ${PROJECT_NAMES[$repo]} after version bump"
|
||||
fi
|
||||
echo "No staged changes for ${PROJECT_NAMES[$repo]}; tagging current HEAD."
|
||||
else
|
||||
run git -C "$repo" commit -m "$COMMIT_MESSAGE"
|
||||
fi
|
||||
else
|
||||
run git -C "$repo" commit -m "$COMMIT_MESSAGE"
|
||||
fi
|
||||
|
||||
run git -C "$repo" commit -m "$COMMIT_MESSAGE"
|
||||
run git -C "$repo" tag -a "$TAG" -m "$TAG_MESSAGE"
|
||||
done
|
||||
|
||||
for repo in "${REPOS[@]:0:3}"; do
|
||||
for repo in "${MODULE_REPOS[@]}"; do
|
||||
run git -C "$repo" push --atomic "$REMOTE" "HEAD:refs/heads/${BRANCHES[$repo]}" "refs/tags/$TAG"
|
||||
done
|
||||
|
||||
@@ -593,6 +756,39 @@ run git -C "$ROOT" commit -m "$COMMIT_MESSAGE"
|
||||
run git -C "$ROOT" tag -a "$TAG" -m "$TAG_MESSAGE"
|
||||
run git -C "$ROOT" push --atomic "$REMOTE" "HEAD:refs/heads/${BRANCHES[$ROOT]}" "refs/tags/$TAG"
|
||||
|
||||
if [[ "$PUBLISH_WEB_CATALOG" -eq 1 ]]; then
|
||||
CATALOG_ARGS=(
|
||||
"$ROOT/scripts/publish-release-catalog.sh"
|
||||
--version "$TARGET_VERSION"
|
||||
--channel "$CATALOG_CHANNEL"
|
||||
--expires-days "$CATALOG_EXPIRES_DAYS"
|
||||
--core-root "$ROOT"
|
||||
--web-root "$WEB_ROOT"
|
||||
--public-base-url "$CATALOG_PUBLIC_URL"
|
||||
--npm "$NPM_BIN"
|
||||
--commit
|
||||
--tag
|
||||
--push
|
||||
--remote "$REMOTE"
|
||||
)
|
||||
if [[ -n "$BRANCH" ]]; then
|
||||
CATALOG_ARGS+=(--branch "$BRANCH")
|
||||
fi
|
||||
if [[ -n "$CATALOG_SEQUENCE" ]]; then
|
||||
CATALOG_ARGS+=(--sequence "$CATALOG_SEQUENCE")
|
||||
fi
|
||||
if [[ "$BUILD_WEB_CATALOG" -eq 1 ]]; then
|
||||
CATALOG_ARGS+=(--build-web)
|
||||
fi
|
||||
for signing_key in "${CATALOG_SIGNING_KEYS[@]}"; do
|
||||
CATALOG_ARGS+=(--catalog-signing-key "$signing_key")
|
||||
done
|
||||
if [[ "$DRY_RUN" -eq 1 ]]; then
|
||||
CATALOG_ARGS+=(--dry-run)
|
||||
fi
|
||||
run "${CATALOG_ARGS[@]}"
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" -eq 1 ]]; then
|
||||
echo "Dry run complete for $TAG across all GovOPlaN repos."
|
||||
else
|
||||
|
||||
Reference in New Issue
Block a user