Initialize GovOPlaN meta repository
This commit is contained in:
580
tools/gitea/gitea-backlog-import.py
Normal file
580
tools/gitea/gitea-backlog-import.py
Normal file
@@ -0,0 +1,580 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Import markdown backlog/planning files into Gitea issues."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import dataclasses
|
||||
import hashlib
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import sys
|
||||
from typing import Any, Iterable
|
||||
|
||||
from gitea_common import GiteaClient, GiteaError, infer_target, label_ids_by_name, load_dotenv, repo_path, repo_root, require_token
|
||||
|
||||
|
||||
DEFAULT_PRODUCT_BACKLOG = pathlib.Path("/mnt/DATA/Nextcloud/ADD ideas UG/Products/govoplan/backlog.md")
|
||||
DEFAULT_WEB_TODO = pathlib.Path("/mnt/DATA/git/addideas-govoplan-website/TODO.md")
|
||||
|
||||
REPO_ROOTS = {
|
||||
"core": pathlib.Path("/mnt/DATA/git/govoplan-core"),
|
||||
"access": pathlib.Path("/mnt/DATA/git/govoplan-access"),
|
||||
"mail": pathlib.Path("/mnt/DATA/git/govoplan-mail"),
|
||||
"files": pathlib.Path("/mnt/DATA/git/govoplan-files"),
|
||||
"campaign": pathlib.Path("/mnt/DATA/git/govoplan-campaign"),
|
||||
"web": pathlib.Path("/mnt/DATA/git/addideas-govoplan-website"),
|
||||
}
|
||||
|
||||
MODULE_LABELS = {
|
||||
"core": "module/core",
|
||||
"access": "module/access",
|
||||
"mail": "module/mail",
|
||||
"files": "module/files",
|
||||
"campaign": "module/campaign",
|
||||
"web": "area/marketing",
|
||||
}
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class Candidate:
|
||||
repo_key: str
|
||||
title: str
|
||||
body: str
|
||||
labels: tuple[str, ...]
|
||||
fingerprint: str
|
||||
source: str
|
||||
line: int
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class RepoState:
|
||||
target: Any
|
||||
client: GiteaClient
|
||||
label_ids: dict[str, int]
|
||||
fingerprints: set[str]
|
||||
titles: set[str]
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--env-file", type=pathlib.Path, help="dotenv file to read before using GITEA_* values")
|
||||
parser.add_argument("--product-backlog", type=pathlib.Path, default=DEFAULT_PRODUCT_BACKLOG)
|
||||
parser.add_argument("--access-plan", type=pathlib.Path, help="optional legacy access extraction plan to import")
|
||||
parser.add_argument("--web-todo", type=pathlib.Path, default=DEFAULT_WEB_TODO)
|
||||
parser.add_argument("--apply", action="store_true", help="create missing Gitea issues")
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
load_dotenv(args.env_file)
|
||||
candidates = list(build_candidates(args))
|
||||
candidates.sort(key=lambda item: (item.repo_key, item.source, item.line, item.title))
|
||||
|
||||
print(f"Prepared {len(candidates)} backlog issue candidate(s).")
|
||||
for repo_key, grouped in group_candidates(candidates).items():
|
||||
print(f"{repo_key}: {len(grouped)} candidate(s)")
|
||||
|
||||
token = require_token() if args.apply else os.environ.get("GITEA_TOKEN")
|
||||
if not token:
|
||||
print("Dry run without GITEA_TOKEN: duplicate comparison skipped.")
|
||||
print_preview(candidates)
|
||||
return 0
|
||||
|
||||
states = load_repo_states(token, sorted({candidate.repo_key for candidate in candidates}))
|
||||
missing = [candidate for candidate in candidates if not already_imported(candidate, states[candidate.repo_key])]
|
||||
skipped = len(candidates) - len(missing)
|
||||
|
||||
print(f"Skipped {skipped} existing issue candidate(s).")
|
||||
print(f"Missing {len(missing)} issue candidate(s).")
|
||||
print_preview(missing)
|
||||
|
||||
if not args.apply:
|
||||
print("Dry run only. Re-run with --apply to create missing issues.")
|
||||
return 0
|
||||
|
||||
created_by_repo: dict[str, list[str]] = {}
|
||||
for candidate in missing:
|
||||
state = states[candidate.repo_key]
|
||||
validate_labels(candidate, state)
|
||||
issue = state.client.request_json(
|
||||
"POST",
|
||||
repo_path(state.target.owner, state.target.repo, "/issues"),
|
||||
body={
|
||||
"title": candidate.title,
|
||||
"body": candidate.body,
|
||||
"labels": [state.label_ids[label] for label in candidate.labels],
|
||||
},
|
||||
)
|
||||
number = issue.get("number") or issue.get("index")
|
||||
created_by_repo.setdefault(candidate.repo_key, []).append(f"#{number} {candidate.title}")
|
||||
state.fingerprints.add(candidate.fingerprint)
|
||||
state.titles.add(normalize_title(candidate.title))
|
||||
|
||||
print("Created issues:")
|
||||
for repo_key, titles in created_by_repo.items():
|
||||
print(f"{repo_key}:")
|
||||
for title in titles:
|
||||
print(f" {title}")
|
||||
if not created_by_repo:
|
||||
print(" none")
|
||||
return 0
|
||||
except GiteaError as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
def build_candidates(args: argparse.Namespace) -> Iterable[Candidate]:
|
||||
if args.product_backlog.exists():
|
||||
yield from parse_product_backlog(args.product_backlog)
|
||||
if args.access_plan and args.access_plan.exists():
|
||||
yield from parse_access_plan(args.access_plan)
|
||||
if args.web_todo.exists():
|
||||
yield from parse_simple_todo(args.web_todo)
|
||||
|
||||
|
||||
def parse_product_backlog(path: pathlib.Path) -> Iterable[Candidate]:
|
||||
headings: list[tuple[int, str]] = []
|
||||
status = ""
|
||||
lines = path.read_text(encoding="utf-8").splitlines()
|
||||
for index, line in enumerate(lines):
|
||||
line_number = index + 1
|
||||
heading_match = re.match(r"^(#{2,4})\s+(.+?)\s*$", line)
|
||||
if heading_match:
|
||||
level = len(heading_match.group(1))
|
||||
title = heading_match.group(2).strip()
|
||||
if level == 2 and title == "Completed Work Archive":
|
||||
break
|
||||
headings = [(h_level, h_title) for h_level, h_title in headings if h_level < level]
|
||||
headings.append((level, title))
|
||||
status = ""
|
||||
continue
|
||||
|
||||
status_match = re.match(r"^\*\*Status:\*\*\s*(.+?)\s*$", line)
|
||||
if status_match:
|
||||
status = status_match.group(1).strip()
|
||||
continue
|
||||
|
||||
item_match = re.match(r"^\s*[-*]\s+\[\s\]\s+(.+?)\s*$", line)
|
||||
if not item_match:
|
||||
continue
|
||||
|
||||
item = collect_continued_item(lines, index, item_match.group(1))
|
||||
section = section_title(headings)
|
||||
repo_key, labels = classify_product_item(item, section)
|
||||
prefix = type_prefix(labels)
|
||||
title = format_title(prefix, item)
|
||||
source = str(path)
|
||||
body = body_for_item(
|
||||
fingerprint=make_fingerprint(source, line_number, repo_key, item),
|
||||
source=source,
|
||||
line=line_number,
|
||||
section=section,
|
||||
status=status,
|
||||
item=item,
|
||||
note="Imported from the consolidated GovOPlaN product backlog.",
|
||||
)
|
||||
yield Candidate(
|
||||
repo_key=repo_key,
|
||||
title=title,
|
||||
body=body,
|
||||
labels=tuple(sorted(set(labels))),
|
||||
fingerprint=make_fingerprint(source, line_number, repo_key, item),
|
||||
source=source,
|
||||
line=line_number,
|
||||
)
|
||||
|
||||
|
||||
def parse_access_plan(path: pathlib.Path) -> Iterable[Candidate]:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
lines = text.splitlines()
|
||||
yield from access_stage_candidates(path, lines)
|
||||
yield from access_immediate_candidates(path, lines)
|
||||
yield from access_decision_candidates(path, lines)
|
||||
|
||||
|
||||
def access_stage_candidates(path: pathlib.Path, lines: list[str]) -> Iterable[Candidate]:
|
||||
index = 0
|
||||
while index < len(lines):
|
||||
match = re.match(r"^### Stage ([1-7]):\s*(.+?)\s*$", lines[index])
|
||||
if not match:
|
||||
index += 1
|
||||
continue
|
||||
stage = match.group(1)
|
||||
name = match.group(2).strip()
|
||||
start_line = index + 1
|
||||
index += 1
|
||||
block: list[str] = []
|
||||
while index < len(lines) and not re.match(r"^### Stage \d+:", lines[index]) and not re.match(r"^## ", lines[index]):
|
||||
block.append(lines[index])
|
||||
index += 1
|
||||
item = f"Access extraction Stage {stage}: {name}"
|
||||
fingerprint = make_fingerprint(str(path), start_line, "core", item)
|
||||
body = "\n".join(
|
||||
[
|
||||
f"<!-- codex-backlog-fingerprint:{fingerprint} -->",
|
||||
"",
|
||||
"Imported from the GovOPlaN access extraction plan.",
|
||||
"",
|
||||
f"- Source: `{path}`",
|
||||
f"- Line: `{start_line}`",
|
||||
f"- Section: `Stage {stage}: {name}`",
|
||||
"",
|
||||
"Plan excerpt:",
|
||||
"",
|
||||
"```markdown",
|
||||
f"### Stage {stage}: {name}",
|
||||
*block,
|
||||
"```",
|
||||
]
|
||||
)
|
||||
yield Candidate(
|
||||
repo_key="core",
|
||||
title=format_title("[Task]", item),
|
||||
body=body,
|
||||
labels=labels("core", "type/task", "area/module-system", "module/access"),
|
||||
fingerprint=fingerprint,
|
||||
source=str(path),
|
||||
line=start_line,
|
||||
)
|
||||
|
||||
|
||||
def access_immediate_candidates(path: pathlib.Path, lines: list[str]) -> Iterable[Candidate]:
|
||||
in_section = False
|
||||
for index, line in enumerate(lines):
|
||||
line_number = index + 1
|
||||
if line == "## Immediate Backlog":
|
||||
in_section = True
|
||||
continue
|
||||
if in_section and line.startswith("## "):
|
||||
break
|
||||
if not in_section:
|
||||
continue
|
||||
item_match = re.match(r"^\s*[-*]\s+\[\s\]\s+(.+?)\s*$", line)
|
||||
if not item_match:
|
||||
continue
|
||||
item = collect_continued_item(lines, index, item_match.group(1))
|
||||
title_item = access_title_item(item)
|
||||
fingerprint = make_fingerprint(str(path), line_number, "core", item)
|
||||
body = body_for_item(
|
||||
fingerprint=fingerprint,
|
||||
source=str(path),
|
||||
line=line_number,
|
||||
section="Immediate Backlog",
|
||||
status="open",
|
||||
item=item,
|
||||
note="Imported from the GovOPlaN access extraction plan immediate backlog.",
|
||||
)
|
||||
yield Candidate(
|
||||
repo_key="core",
|
||||
title=format_title("[Task]", title_item),
|
||||
body=body,
|
||||
labels=labels("core", "type/task", "area/auth", "area/module-system", "module/access"),
|
||||
fingerprint=fingerprint,
|
||||
source=str(path),
|
||||
line=line_number,
|
||||
)
|
||||
|
||||
|
||||
def access_decision_candidates(path: pathlib.Path, lines: list[str]) -> Iterable[Candidate]:
|
||||
in_section = False
|
||||
for index, line in enumerate(lines):
|
||||
line_number = index + 1
|
||||
if line == "## Open Decisions":
|
||||
in_section = True
|
||||
continue
|
||||
if in_section and line.startswith("## "):
|
||||
break
|
||||
if not in_section:
|
||||
continue
|
||||
item_match = re.match(r"^\s*[-*]\s+(.+?)\s*$", line)
|
||||
if not item_match:
|
||||
continue
|
||||
item = collect_continued_item(lines, index, item_match.group(1))
|
||||
fingerprint = make_fingerprint(str(path), line_number, "core", item)
|
||||
body = body_for_item(
|
||||
fingerprint=fingerprint,
|
||||
source=str(path),
|
||||
line=line_number,
|
||||
section="Open Decisions",
|
||||
status="decision needed",
|
||||
item=item,
|
||||
note="Imported from the GovOPlaN access extraction plan open decisions.",
|
||||
)
|
||||
yield Candidate(
|
||||
repo_key="core",
|
||||
title=format_title("[Task]", f"Decide: {item}"),
|
||||
body=body,
|
||||
labels=labels("core", "type/task", "status/needs-info", "codex/needs-human", "area/auth", "module/access"),
|
||||
fingerprint=fingerprint,
|
||||
source=str(path),
|
||||
line=line_number,
|
||||
)
|
||||
|
||||
|
||||
def parse_simple_todo(path: pathlib.Path) -> Iterable[Candidate]:
|
||||
lines = path.read_text(encoding="utf-8").splitlines()
|
||||
for index, line in enumerate(lines):
|
||||
line_number = index + 1
|
||||
item_match = re.match(r"^\s*[-*]\s+(.+?)\s*$", line)
|
||||
if not item_match:
|
||||
continue
|
||||
item = collect_continued_item(lines, index, item_match.group(1))
|
||||
fingerprint = make_fingerprint(str(path), line_number, "web", item)
|
||||
yield Candidate(
|
||||
repo_key="web",
|
||||
title=format_title("[Task]", item),
|
||||
body=body_for_item(
|
||||
fingerprint=fingerprint,
|
||||
source=str(path),
|
||||
line=line_number,
|
||||
section="TODO",
|
||||
status="open",
|
||||
item=item,
|
||||
note="Imported from the GovOPlaN website TODO file.",
|
||||
),
|
||||
labels=labels("web", "type/task", "area/marketing"),
|
||||
fingerprint=fingerprint,
|
||||
source=str(path),
|
||||
line=line_number,
|
||||
)
|
||||
|
||||
|
||||
def classify_product_item(item: str, section: str) -> tuple[str, tuple[str, ...]]:
|
||||
text = f"{section} {item}".lower()
|
||||
repo_key = "core"
|
||||
area = "area/devex"
|
||||
issue_type = "type/task"
|
||||
|
||||
if "dsar search/export/delete" in text:
|
||||
return "core", labels("core", "type/feature", "area/governance")
|
||||
if "fully stream zip uploads" in text:
|
||||
return "files", labels("files", "type/task", "area/api")
|
||||
if "installer packaging approach" in text:
|
||||
return "core", labels("core", "type/task", "area/devex")
|
||||
if "profile reselection/revalidation" in text:
|
||||
return "campaign", labels("campaign", "type/task", "area/api", "module/mail")
|
||||
if "module boundaries" in text:
|
||||
return "core", labels("core", issue_type, "area/module-system")
|
||||
if "external storage connectors" in text:
|
||||
repo_key = "files"
|
||||
area = "area/api"
|
||||
elif any(phrase in text for phrase in ("report, evidence", "recipient import", "campaign and attachment", "collaboration and advanced governance", "address book, templates")):
|
||||
repo_key = "campaign"
|
||||
area = "area/webui" if any(word in text for word in ("page", "table", "display", "ui", "wizard", "editor", "flow", "polish")) else "area/api"
|
||||
elif "controlled sending" in text:
|
||||
repo_key = "campaign"
|
||||
area = "area/api"
|
||||
elif "mail profiles" in text:
|
||||
repo_key = "mail"
|
||||
area = "area/api"
|
||||
|
||||
if any(word in text for word in ("smtp", "imap", "mailbox", "mail profile", "deliverability", "bounce", "openpgp", "s/mime", "pop3", "jmap")):
|
||||
repo_key = "mail"
|
||||
area = "area/api"
|
||||
if any(word in text for word in ("seafile", "nextcloud", "webdav", "smb", "connector", "file rename", "managed storage", "live remote file")):
|
||||
repo_key = "files"
|
||||
area = "area/api"
|
||||
if "external storage connectors" not in text and any(
|
||||
word in text
|
||||
for word in (
|
||||
"campaign",
|
||||
"recipient",
|
||||
"attachment",
|
||||
"zip",
|
||||
"evidence",
|
||||
"wizard",
|
||||
"send",
|
||||
"archive",
|
||||
"message preview",
|
||||
"address book",
|
||||
"carddav",
|
||||
"mailing lists",
|
||||
)
|
||||
):
|
||||
repo_key = "campaign"
|
||||
area = "area/webui" if any(word in text for word in ("page", "table", "display", "ui", "wizard", "editor", "flow", "polish")) else "area/api"
|
||||
if repo_key == "core" and any(word in text for word in ("session", "device", "auth", "rbac", "role", "tenant", "governance", "policy", "retention", "dsar", "audit", "encryption", "ldap", "oidc", "saml")):
|
||||
repo_key = "core"
|
||||
area = "area/auth" if any(word in text for word in ("session", "auth", "ldap", "oidc", "saml")) else "area/governance"
|
||||
if repo_key == "core" and any(word in text for word in ("module", "manifest", "module boundaries", "installable", "activatable", "deactivatable", "uninstallable")):
|
||||
repo_key = "core"
|
||||
area = "area/module-system"
|
||||
if repo_key == "core" and any(word in text for word in ("release", "version metadata", "lockfile", "package", "tagging")):
|
||||
repo_key = "core"
|
||||
area = "area/release"
|
||||
if repo_key == "core" and not any(
|
||||
phrase in text
|
||||
for phrase in (
|
||||
"external storage connectors",
|
||||
"controlled sending",
|
||||
"report, evidence",
|
||||
"recipient import",
|
||||
"campaign and attachment",
|
||||
)
|
||||
) and any(word in text for word in ("postgres", "redis", "celery", "worker", "deployment", "installer", "backup", "monitoring", "configuration", "dev profile", ".env")):
|
||||
repo_key = "core"
|
||||
area = "area/devex"
|
||||
if any(word in text for word in ("documentation", "handbook", "docs", "runbook")):
|
||||
area = "area/docs"
|
||||
if "ui lists often start at index 2" in text:
|
||||
issue_type = "type/bug"
|
||||
area = "area/webui"
|
||||
repo_key = "core"
|
||||
if any(word in text for word in ("add ", "implement ", "build ", "provide ", "allow ", "make ", "extend ", "replace ")):
|
||||
issue_type = "type/feature"
|
||||
if any(word in text for word in ("cleanup", "remove legacy", "fragile", "deferred cleanup")):
|
||||
issue_type = "type/debt"
|
||||
|
||||
return repo_key, labels(repo_key, issue_type, area)
|
||||
|
||||
|
||||
def collect_continued_item(lines: list[str], index: int, initial: str) -> str:
|
||||
parts = [initial.strip()]
|
||||
next_index = index + 1
|
||||
while next_index < len(lines):
|
||||
line = lines[next_index]
|
||||
if not line.startswith((" ", "\t")):
|
||||
break
|
||||
stripped = line.strip()
|
||||
if not stripped:
|
||||
break
|
||||
if re.match(r"^[-*]\s+", stripped) or stripped.startswith("#"):
|
||||
break
|
||||
parts.append(stripped)
|
||||
next_index += 1
|
||||
return " ".join(parts)
|
||||
|
||||
|
||||
def labels(repo_key: str, *extra: str) -> tuple[str, ...]:
|
||||
base = {"status/triage", "source/backlog-import", "codex/ready", MODULE_LABELS[repo_key]}
|
||||
base.update(extra)
|
||||
if "status/needs-info" in base:
|
||||
base.discard("status/triage")
|
||||
return tuple(sorted(base))
|
||||
|
||||
|
||||
def type_prefix(label_set: Iterable[str]) -> str:
|
||||
labels_set = set(label_set)
|
||||
if "type/bug" in labels_set:
|
||||
return "[Bug]"
|
||||
if "type/feature" in labels_set:
|
||||
return "[Feature]"
|
||||
if "type/debt" in labels_set:
|
||||
return "[Debt]"
|
||||
if "type/docs" in labels_set:
|
||||
return "[Docs]"
|
||||
return "[Task]"
|
||||
|
||||
|
||||
def format_title(prefix: str, item: str) -> str:
|
||||
clean = re.sub(r"\s+", " ", item).strip().rstrip(".")
|
||||
title = f"{prefix} {clean}"
|
||||
if len(title) <= 180:
|
||||
return title
|
||||
return f"{title[:177].rstrip()}..."
|
||||
|
||||
|
||||
def access_title_item(item: str) -> str:
|
||||
lowered = item.lower()
|
||||
if "compatibility wrappers" in lowered:
|
||||
return "Wire core auth compatibility through access capabilities"
|
||||
return item
|
||||
|
||||
|
||||
def body_for_item(
|
||||
*,
|
||||
fingerprint: str,
|
||||
source: str,
|
||||
line: int,
|
||||
section: str,
|
||||
status: str,
|
||||
item: str,
|
||||
note: str,
|
||||
) -> str:
|
||||
lines = [
|
||||
f"<!-- codex-backlog-fingerprint:{fingerprint} -->",
|
||||
"",
|
||||
note,
|
||||
"",
|
||||
f"- Source: `{source}`",
|
||||
f"- Line: `{line}`",
|
||||
f"- Section: `{section}`",
|
||||
]
|
||||
if status:
|
||||
lines.append(f"- Source status: `{status}`")
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
"Imported backlog item:",
|
||||
"",
|
||||
"```markdown",
|
||||
f"- [ ] {item}",
|
||||
"```",
|
||||
]
|
||||
)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def section_title(headings: list[tuple[int, str]]) -> str:
|
||||
return " > ".join(title for _, title in headings)
|
||||
|
||||
|
||||
def make_fingerprint(source: str, line: int, repo_key: str, item: str) -> str:
|
||||
stable = "\n".join([source, str(line), repo_key, item])
|
||||
return hashlib.sha256(stable.encode("utf-8")).hexdigest()[:24]
|
||||
|
||||
|
||||
def group_candidates(candidates: list[Candidate]) -> dict[str, list[Candidate]]:
|
||||
grouped: dict[str, list[Candidate]] = {}
|
||||
for candidate in candidates:
|
||||
grouped.setdefault(candidate.repo_key, []).append(candidate)
|
||||
return grouped
|
||||
|
||||
|
||||
def load_repo_states(token: str, repo_keys: list[str]) -> dict[str, RepoState]:
|
||||
states: dict[str, RepoState] = {}
|
||||
for repo_key in repo_keys:
|
||||
root = repo_root(REPO_ROOTS[repo_key])
|
||||
target = infer_target(root)
|
||||
client = GiteaClient(target, token)
|
||||
label_ids = label_ids_by_name(client, target.owner, target.repo)
|
||||
issues = client.paginate(repo_path(target.owner, target.repo, "/issues"), query={"state": "all"}, limit=100)
|
||||
fingerprints: set[str] = set()
|
||||
titles: set[str] = set()
|
||||
for issue in issues:
|
||||
body = str(issue.get("body") or "")
|
||||
if issue.get("state") == "closed" and "Moved to `" in body:
|
||||
continue
|
||||
titles.add(normalize_title(str(issue.get("title") or "")))
|
||||
fingerprints.update(re.findall(r"codex-backlog-fingerprint:([0-9a-f]{24})", body))
|
||||
states[repo_key] = RepoState(target=target, client=client, label_ids=label_ids, fingerprints=fingerprints, titles=titles)
|
||||
return states
|
||||
|
||||
|
||||
def already_imported(candidate: Candidate, state: RepoState) -> bool:
|
||||
return candidate.fingerprint in state.fingerprints or normalize_title(candidate.title) in state.titles
|
||||
|
||||
|
||||
def normalize_title(title: str) -> str:
|
||||
return re.sub(r"[^a-z0-9]+", " ", title.lower()).strip()
|
||||
|
||||
|
||||
def validate_labels(candidate: Candidate, state: RepoState) -> None:
|
||||
missing = sorted(label for label in candidate.labels if label not in state.label_ids)
|
||||
if missing:
|
||||
joined = ", ".join(missing)
|
||||
raise GiteaError(f"{candidate.repo_key} is missing labels for {candidate.title!r}: {joined}")
|
||||
|
||||
|
||||
def print_preview(candidates: list[Candidate], limit: int = 160) -> None:
|
||||
for repo_key, grouped in group_candidates(candidates).items():
|
||||
print(f"{repo_key}:")
|
||||
for candidate in grouped[:limit]:
|
||||
print(f" {candidate.title}")
|
||||
if len(grouped) > limit:
|
||||
print(f" ... {len(grouped) - limit} more")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
97
tools/gitea/gitea-codex-note.py
Normal file
97
tools/gitea/gitea-codex-note.py
Normal file
@@ -0,0 +1,97 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Post a standardized Codex state update comment to a Gitea issue."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
from gitea_common import GiteaClient, GiteaError, infer_target, load_dotenv, repo_path, repo_root, require_token
|
||||
|
||||
|
||||
STATUS_LABELS = {
|
||||
"started": "status/in-progress",
|
||||
"progress": "status/in-progress",
|
||||
"blocked": "status/blocked",
|
||||
"needs-info": "status/needs-info",
|
||||
"ready": "status/ready",
|
||||
"done": "",
|
||||
"note": "",
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--root", type=pathlib.Path, default=pathlib.Path.cwd(), help="repository root or child path")
|
||||
parser.add_argument("--remote", default="origin", help="git remote to use for target inference")
|
||||
parser.add_argument("--env-file", type=pathlib.Path, help="dotenv file to read before using GITEA_* values")
|
||||
parser.add_argument("--issue", type=int, required=True, help="Gitea issue number")
|
||||
parser.add_argument("--status", choices=sorted(STATUS_LABELS), default="note")
|
||||
parser.add_argument("--summary", action="append", default=[], help="summary bullet; may be repeated")
|
||||
parser.add_argument("--changed", action="append", default=[], help="changed file path; may be repeated")
|
||||
parser.add_argument("--test", action="append", default=[], help="verification command/result; may be repeated")
|
||||
parser.add_argument("--next", action="append", default=[], help="next step or blocker; may be repeated")
|
||||
parser.add_argument("--body-file", type=pathlib.Path, help="additional Markdown body to append")
|
||||
parser.add_argument("--close", action="store_true", help="close the issue after posting the note")
|
||||
parser.add_argument("--apply", action="store_true", help="post the comment and optional state update")
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
root = repo_root(args.root)
|
||||
load_dotenv(args.env_file or root / ".env")
|
||||
target = infer_target(root, args.remote)
|
||||
token = require_token() if args.apply else os.environ.get("GITEA_TOKEN")
|
||||
body = build_body(args)
|
||||
|
||||
print(f"Target: {target.display}")
|
||||
print(f"Issue: #{args.issue}")
|
||||
print(body)
|
||||
|
||||
if not args.apply:
|
||||
print("Dry run only. Re-run with --apply and GITEA_TOKEN to post.")
|
||||
return 0
|
||||
|
||||
client = GiteaClient(target, token)
|
||||
client.request_json(
|
||||
"POST",
|
||||
repo_path(target.owner, target.repo, f"/issues/{args.issue}/comments"),
|
||||
body={"body": body},
|
||||
)
|
||||
if args.close:
|
||||
client.request_json(
|
||||
"PATCH",
|
||||
repo_path(target.owner, target.repo, f"/issues/{args.issue}"),
|
||||
body={"state": "closed"},
|
||||
)
|
||||
print("Posted Codex note.")
|
||||
return 0
|
||||
except GiteaError as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
def build_body(args: argparse.Namespace) -> str:
|
||||
lines = [f"## Codex State: {args.status}", ""]
|
||||
append_section(lines, "Summary", args.summary)
|
||||
append_section(lines, "Changed Files", [f"`{item}`" for item in args.changed])
|
||||
append_section(lines, "Verification", [f"`{item}`" for item in args.test])
|
||||
append_section(lines, "Next / Blocked", args.next)
|
||||
if STATUS_LABELS[args.status]:
|
||||
lines.extend(["", f"Suggested status label: `{STATUS_LABELS[args.status]}`"])
|
||||
if args.body_file:
|
||||
lines.extend(["", args.body_file.read_text(encoding="utf-8").strip()])
|
||||
return "\n".join(lines).rstrip() + "\n"
|
||||
|
||||
|
||||
def append_section(lines: list[str], title: str, items: list[str]) -> None:
|
||||
if not items:
|
||||
return
|
||||
lines.extend([f"### {title}", ""])
|
||||
lines.extend(f"- {item}" for item in items)
|
||||
lines.append("")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
600
tools/gitea/gitea-import-all-backlogs.py
Normal file
600
tools/gitea/gitea-import-all-backlogs.py
Normal file
@@ -0,0 +1,600 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Import backlog-like files from git.add-ideas.de repositories into Gitea."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import dataclasses
|
||||
import hashlib
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from typing import Any, Iterable
|
||||
|
||||
from gitea_common import GiteaClient, GiteaError, infer_target, label_ids_by_name, load_dotenv, repo_path, require_token
|
||||
|
||||
|
||||
GIT_ROOT = pathlib.Path("/mnt/DATA/git")
|
||||
PRODUCTS_ROOT = pathlib.Path("/mnt/DATA/Nextcloud/ADD ideas UG/Products")
|
||||
EXCLUDED_FILE_PATTERNS = (
|
||||
"/.git/",
|
||||
"/.venv/",
|
||||
"/venv/",
|
||||
"/site-packages/",
|
||||
"/node_modules/",
|
||||
"/__pycache__/",
|
||||
"/dist/",
|
||||
"/build/",
|
||||
"/.cache/",
|
||||
"/.module-test-build/",
|
||||
)
|
||||
BACKLOG_NAME_RE = re.compile(r"(backlog|todo|roadmap|plan|issue|milestone|action)", re.IGNORECASE)
|
||||
ACTION_RE = re.compile(
|
||||
r"^(add|avoid|build|convert|create|define|delay|design|document|enable|ensure|expand|extract|fetch|fix|finalize|generate|"
|
||||
r"implement|import|improve|integrate|keep|make|move|optimize|persist|precompute|promote|provide|"
|
||||
r"publish|query|replace|review|run|seed|serve|split|start|store|support|test|track|use|wire)\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
SKIP_SECTION_RE = re.compile(
|
||||
r"(completed|implemented|current state|recent fixes|recently completed|working assumptions|stable decisions|"
|
||||
r"available now|done|archive)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
ACTIVE_SECTION_RE = re.compile(
|
||||
r"(p0|p1|p2|p3|mvp|milestone|phase|todo|backlog|roadmap|tasks|testing|frontend|backend|routing|"
|
||||
r"data outputs|future|critical path|next sprint|optimization|hardening|enhancements?)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
GENERIC_LABELS = [
|
||||
("type/task", "1d76db", "Implementation, maintenance, migration, or operational work."),
|
||||
("type/feature", "0e8a16", "New user-visible behavior or product capability."),
|
||||
("type/debt", "fbca04", "Cleanup, refactoring, risk reduction, or deferred engineering work."),
|
||||
("type/docs", "5319e7", "Documentation, process, or developer workflow work."),
|
||||
("status/triage", "d4c5f9", "Needs review, ownership, priority, or acceptance criteria."),
|
||||
("source/backlog-import", "bfdadc", "Imported from markdown, text, CSV, roadmap, backlog, plan, or TODO files."),
|
||||
("codex/ready", "0e8a16", "Suitable for Codex to pick up with the existing issue context."),
|
||||
("priority/p0", "b60205", "Immediate stop-the-line priority."),
|
||||
("priority/p1", "d93f0b", "High priority for the next focused work window."),
|
||||
("priority/p2", "fbca04", "Normal planned priority."),
|
||||
("priority/p3", "c2e0c6", "Low priority or opportunistic cleanup."),
|
||||
]
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class RepoInfo:
|
||||
root: pathlib.Path
|
||||
remote: str
|
||||
owner: str
|
||||
repo: str
|
||||
|
||||
@property
|
||||
def key(self) -> str:
|
||||
return self.root.name
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class SourceFile:
|
||||
repo_key: str
|
||||
path: pathlib.Path
|
||||
source_kind: str
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class Candidate:
|
||||
repo_key: str
|
||||
title: str
|
||||
body: str
|
||||
labels: tuple[str, ...]
|
||||
priority: str
|
||||
milestone: str
|
||||
fingerprint: str
|
||||
source: str
|
||||
line: int
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--env-file", type=pathlib.Path, default=pathlib.Path("/home/zemion/.config/gitea/gitea.env"))
|
||||
parser.add_argument("--git-root", type=pathlib.Path, default=GIT_ROOT)
|
||||
parser.add_argument("--products-root", type=pathlib.Path, default=PRODUCTS_ROOT)
|
||||
parser.add_argument("--apply", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
load_dotenv(args.env_file)
|
||||
repos = discover_hosted_repos(args.git_root)
|
||||
sources = discover_sources(repos, args.products_root)
|
||||
candidates = list(build_candidates(sources))
|
||||
candidates = dedupe_candidates(candidates)
|
||||
candidates.sort(key=lambda item: (item.repo_key, item.source, item.line, item.title))
|
||||
|
||||
print(f"Hosted repositories: {len(repos)}")
|
||||
print(f"Backlog-like source files: {len(sources)}")
|
||||
print(f"Issue candidates: {len(candidates)}")
|
||||
for repo_key, count in grouped_counts(candidates).items():
|
||||
print(f" {repo_key}: {count}")
|
||||
|
||||
token = require_token() if args.apply else os.environ.get("GITEA_TOKEN")
|
||||
if not token:
|
||||
print_preview(candidates)
|
||||
print("Dry run without GITEA_TOKEN: duplicate comparison skipped.")
|
||||
return 0
|
||||
|
||||
missing_by_repo: dict[str, list[Candidate]] = {}
|
||||
skipped = 0
|
||||
states: dict[str, RepoState] = {}
|
||||
for repo_key in sorted({candidate.repo_key for candidate in candidates}):
|
||||
states[repo_key] = load_repo_state(repos[repo_key], token, apply=args.apply)
|
||||
for candidate in candidates:
|
||||
state = states[candidate.repo_key]
|
||||
if candidate.fingerprint in state.fingerprints or normalize_title(candidate.title) in state.titles:
|
||||
skipped += 1
|
||||
continue
|
||||
missing_by_repo.setdefault(candidate.repo_key, []).append(candidate)
|
||||
|
||||
missing_total = sum(len(items) for items in missing_by_repo.values())
|
||||
print(f"Skipped existing: {skipped}")
|
||||
print(f"Missing candidates: {missing_total}")
|
||||
print_preview([candidate for items in missing_by_repo.values() for candidate in items])
|
||||
|
||||
if not args.apply:
|
||||
print("Dry run only. Re-run with --apply to create missing issues.")
|
||||
return 0
|
||||
|
||||
for repo_key, repo_candidates in missing_by_repo.items():
|
||||
state = states[repo_key]
|
||||
for candidate in repo_candidates:
|
||||
label_ids = [state.label_ids[label] for label in candidate.labels]
|
||||
if candidate.priority not in candidate.labels:
|
||||
label_ids.append(state.label_ids[candidate.priority])
|
||||
milestone_id = ensure_milestone(state, candidate.milestone)
|
||||
created = state.client.request_json(
|
||||
"POST",
|
||||
repo_path(state.target.owner, state.target.repo, "/issues"),
|
||||
body={
|
||||
"title": candidate.title,
|
||||
"body": candidate.body,
|
||||
"labels": label_ids,
|
||||
"milestone": milestone_id,
|
||||
},
|
||||
)
|
||||
number = created.get("number") or created.get("index")
|
||||
print(f"created {state.target.owner}/{state.target.repo}#{number}: {candidate.title}")
|
||||
state.fingerprints.add(candidate.fingerprint)
|
||||
state.titles.add(normalize_title(candidate.title))
|
||||
return 0
|
||||
except GiteaError as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class RepoState:
|
||||
target: Any
|
||||
client: GiteaClient
|
||||
label_ids: dict[str, int]
|
||||
milestone_ids: dict[str, int]
|
||||
fingerprints: set[str]
|
||||
titles: set[str]
|
||||
|
||||
|
||||
def discover_hosted_repos(git_root: pathlib.Path) -> dict[str, RepoInfo]:
|
||||
repos: dict[str, RepoInfo] = {}
|
||||
for gitdir in sorted(git_root.glob("*/.git")):
|
||||
root = gitdir.parent
|
||||
result = subprocess.run(
|
||||
["git", "-C", str(root), "remote", "get-url", "origin"],
|
||||
check=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
text=True,
|
||||
)
|
||||
remote = result.stdout.strip()
|
||||
if "git.add-ideas.de" not in remote:
|
||||
continue
|
||||
target = infer_target(root)
|
||||
repos[root.name] = RepoInfo(root=root, remote=remote, owner=target.owner, repo=target.repo)
|
||||
return repos
|
||||
|
||||
|
||||
def discover_sources(repos: dict[str, RepoInfo], products_root: pathlib.Path) -> list[SourceFile]:
|
||||
sources: list[SourceFile] = []
|
||||
for repo in repos.values():
|
||||
for path in repo.root.rglob("*"):
|
||||
if not path.is_file() or not BACKLOG_NAME_RE.search(path.name):
|
||||
continue
|
||||
path_text = path.as_posix()
|
||||
if any(pattern in path_text for pattern in EXCLUDED_FILE_PATTERNS):
|
||||
continue
|
||||
if is_excluded_repo_file(path):
|
||||
continue
|
||||
if is_text_backlog(path):
|
||||
sources.append(SourceFile(repo_key=repo.key, path=path, source_kind="repo"))
|
||||
|
||||
product_map = {
|
||||
"co2api": "emission-api-lib",
|
||||
"govoplan": "govoplan-core",
|
||||
"meubility": "meubility-workbench",
|
||||
"mousehold": "mousehold",
|
||||
"prvnncr": "prvnncr-server",
|
||||
}
|
||||
if products_root.exists():
|
||||
for product_dir in sorted(path for path in products_root.iterdir() if path.is_dir()):
|
||||
repo_key = product_map.get(product_dir.name)
|
||||
if not repo_key or repo_key not in repos:
|
||||
continue
|
||||
for path in product_dir.rglob("*"):
|
||||
if path.is_file() and BACKLOG_NAME_RE.search(path.name) and is_text_backlog(path):
|
||||
sources.append(SourceFile(repo_key=repo_key, path=path, source_kind="product"))
|
||||
|
||||
unique: dict[tuple[str, pathlib.Path], SourceFile] = {}
|
||||
for source in sources:
|
||||
unique[(source.repo_key, source.path.resolve())] = source
|
||||
return sorted(unique.values(), key=lambda item: (item.repo_key, str(item.path)))
|
||||
|
||||
|
||||
def is_excluded_repo_file(path: pathlib.Path) -> bool:
|
||||
text = path.as_posix()
|
||||
name = path.name.lower()
|
||||
if (
|
||||
"/tools/gitea/gitea-" in text
|
||||
or text.endswith("/tools/gitea/gitea_common.py")
|
||||
):
|
||||
return True
|
||||
if "testing_plan" in name or "test_plan" in name:
|
||||
return True
|
||||
if text.endswith("/docs/GITEA_ISSUES.md"):
|
||||
return True
|
||||
if text.endswith("/docs/GOVOPLAN_MASTER_ROADMAP.md"):
|
||||
return True
|
||||
if text.endswith("/addideas-govoplan-website/TODO.md"):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def is_text_backlog(path: pathlib.Path) -> bool:
|
||||
if path.suffix.lower() not in {".md", ".txt", ".csv"}:
|
||||
return False
|
||||
try:
|
||||
chunk = path.read_bytes()[:4096]
|
||||
except OSError:
|
||||
return False
|
||||
if b"\x00" in chunk:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def build_candidates(sources: list[SourceFile]) -> Iterable[Candidate]:
|
||||
for source in sources:
|
||||
suffix = source.path.suffix.lower()
|
||||
if suffix == ".csv":
|
||||
yield from parse_csv_source(source)
|
||||
else:
|
||||
yield from parse_text_source(source)
|
||||
|
||||
|
||||
def parse_csv_source(source: SourceFile) -> Iterable[Candidate]:
|
||||
with source.path.open("r", encoding="utf-8-sig", newline="") as handle:
|
||||
reader = csv.DictReader(handle)
|
||||
for index, row in enumerate(reader, start=2):
|
||||
story = (row.get("story") or row.get("title") or row.get("name") or "").strip()
|
||||
if not story:
|
||||
continue
|
||||
identifier = (row.get("id") or "").strip()
|
||||
priority = normalize_priority(row.get("priority") or "")
|
||||
milestone = (row.get("epic") or "Backlog").strip()
|
||||
title = format_title("[Feature]", f"{identifier}: {story}" if identifier else story)
|
||||
fingerprint = make_fingerprint(source.path, index, source.repo_key, story)
|
||||
body = "\n".join(
|
||||
[
|
||||
f"<!-- codex-generic-backlog-fingerprint:{fingerprint} -->",
|
||||
"",
|
||||
"Imported from a CSV backlog file.",
|
||||
"",
|
||||
f"- Source: `{source.path}`",
|
||||
f"- Line: `{index}`",
|
||||
f"- Source kind: `{source.source_kind}`",
|
||||
f"- Milestone: `{milestone}`",
|
||||
"",
|
||||
"CSV row:",
|
||||
"",
|
||||
"```text",
|
||||
", ".join(f"{key}={value}" for key, value in row.items()),
|
||||
"```",
|
||||
]
|
||||
)
|
||||
yield Candidate(
|
||||
repo_key=source.repo_key,
|
||||
title=title,
|
||||
body=body,
|
||||
labels=("type/feature", "status/triage", "source/backlog-import", "codex/ready"),
|
||||
priority=priority,
|
||||
milestone=milestone,
|
||||
fingerprint=fingerprint,
|
||||
source=str(source.path),
|
||||
line=index,
|
||||
)
|
||||
|
||||
|
||||
def parse_text_source(source: SourceFile) -> Iterable[Candidate]:
|
||||
try:
|
||||
lines = source.path.read_text(encoding="utf-8").splitlines()
|
||||
except UnicodeDecodeError:
|
||||
lines = source.path.read_text(encoding="latin-1").splitlines()
|
||||
|
||||
headings: list[tuple[int, str]] = []
|
||||
in_tasks = False
|
||||
for index, line in enumerate(lines):
|
||||
line_number = index + 1
|
||||
heading = parse_heading(line)
|
||||
if heading:
|
||||
level, title = heading
|
||||
headings = [(h_level, h_title) for h_level, h_title in headings if h_level < level]
|
||||
headings.append((level, title))
|
||||
in_tasks = False
|
||||
continue
|
||||
|
||||
if re.match(r"^\s*(tasks|action items|immediate todos?|recommended next sprint)\s*:?\s*$", line, re.IGNORECASE):
|
||||
in_tasks = True
|
||||
continue
|
||||
|
||||
item = parse_open_item(line)
|
||||
if item is None and is_action_context(source.path, headings, in_tasks):
|
||||
item = parse_plain_action_item(line)
|
||||
if item is None:
|
||||
continue
|
||||
if not item or should_skip_item(item, headings):
|
||||
continue
|
||||
|
||||
section = section_title(headings)
|
||||
priority = priority_from_context(section, item)
|
||||
milestone = milestone_from_context(source.path, section)
|
||||
prefix = "[Feature]" if ACTION_RE.match(item) else "[Task]"
|
||||
title = format_title(prefix, item)
|
||||
fingerprint = make_fingerprint(source.path, line_number, source.repo_key, item)
|
||||
body = "\n".join(
|
||||
[
|
||||
f"<!-- codex-generic-backlog-fingerprint:{fingerprint} -->",
|
||||
"",
|
||||
"Imported from a backlog-like text file.",
|
||||
"",
|
||||
f"- Source: `{source.path}`",
|
||||
f"- Line: `{line_number}`",
|
||||
f"- Source kind: `{source.source_kind}`",
|
||||
f"- Section: `{section or 'none'}`",
|
||||
"",
|
||||
"Imported item:",
|
||||
"",
|
||||
"```text",
|
||||
item,
|
||||
"```",
|
||||
]
|
||||
)
|
||||
yield Candidate(
|
||||
repo_key=source.repo_key,
|
||||
title=title,
|
||||
body=body,
|
||||
labels=("type/feature" if prefix == "[Feature]" else "type/task", "status/triage", "source/backlog-import", "codex/ready"),
|
||||
priority=priority,
|
||||
milestone=milestone,
|
||||
fingerprint=fingerprint,
|
||||
source=str(source.path),
|
||||
line=line_number,
|
||||
)
|
||||
|
||||
|
||||
def parse_heading(line: str) -> tuple[int, str] | None:
|
||||
markdown = re.match(r"^(#{1,6})\s+(.+?)\s*$", line)
|
||||
if markdown:
|
||||
return len(markdown.group(1)), clean_text(markdown.group(2))
|
||||
underlined = re.match(r"^(.+?)\s*$", line)
|
||||
if underlined and line.strip() and len(line.strip()) < 120:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def parse_open_item(line: str) -> str | None:
|
||||
patterns = [
|
||||
r"^(?P<indent>\s*)[-*]\s+\[\s\]\s+(?P<text>.+?)\s*$",
|
||||
r"^(?P<indent>\s*)[-*]\s+☐\s+(?P<text>.+?)\s*$",
|
||||
r"^(?P<indent>\s*)\d+[.)]\s+☐\s+(?P<text>.+?)\s*$",
|
||||
r"^(?P<indent>\s*)[-*]\s+\[(?!x\]|X\])(?:[^\]]+)\]\s+(?P<text>.+?)\s*$",
|
||||
]
|
||||
for pattern in patterns:
|
||||
match = re.match(pattern, line)
|
||||
if match and leading_width(match.group("indent")) == 0:
|
||||
return clean_text(match.group("text"))
|
||||
return None
|
||||
|
||||
|
||||
def parse_plain_action_item(line: str) -> str | None:
|
||||
match = re.match(r"^(?P<indent>\s*)[-*]\s+(?P<text>.+?)\s*$", line)
|
||||
if match and leading_width(match.group("indent")) == 0:
|
||||
text = clean_text(match.group("text"))
|
||||
if "✅" in text or text.startswith(("✅", "[x]", "[X]")):
|
||||
return None
|
||||
if ACTION_RE.match(text) or is_short_noun_task(text):
|
||||
return text
|
||||
return None
|
||||
|
||||
|
||||
def is_action_context(path: pathlib.Path, headings: list[tuple[int, str]], in_tasks: bool) -> bool:
|
||||
if in_tasks:
|
||||
return True
|
||||
context = section_title(headings)
|
||||
if not context:
|
||||
return False
|
||||
if SKIP_SECTION_RE.search(context):
|
||||
return False
|
||||
if ACTIVE_SECTION_RE.search(context):
|
||||
return True
|
||||
return path.name.lower() in {"todo.txt", "todo.md"}
|
||||
|
||||
|
||||
def should_skip_item(item: str, headings: list[tuple[int, str]]) -> bool:
|
||||
context = section_title(headings)
|
||||
if "✅" in item or item.startswith(("✅", "[x]", "[X]")):
|
||||
return True
|
||||
if item.endswith((",", ";")):
|
||||
return True
|
||||
if SKIP_SECTION_RE.search(context):
|
||||
return True
|
||||
if len(item) < 4:
|
||||
return True
|
||||
if item.endswith(":") and len(item.split()) <= 6:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def is_short_noun_task(text: str) -> bool:
|
||||
return len(text.split()) <= 8 and not text.endswith(".") and not re.search(r"://", text)
|
||||
|
||||
|
||||
def clean_text(text: str) -> str:
|
||||
text = text.strip()
|
||||
text = re.sub(r"^☐\s*", "", text)
|
||||
text = re.sub(r"^✅\s*", "", text)
|
||||
text = re.sub(r"\s+", " ", text)
|
||||
return text.strip(" -")
|
||||
|
||||
|
||||
def leading_width(indent: str) -> int:
|
||||
return len(indent.replace("\t", " "))
|
||||
|
||||
|
||||
def section_title(headings: list[tuple[int, str]]) -> str:
|
||||
return " > ".join(title for _, title in headings)
|
||||
|
||||
|
||||
def priority_from_context(section: str, item: str) -> str:
|
||||
text = f"{section} {item}".lower()
|
||||
if "p0" in text or "critical path" in text:
|
||||
return "priority/p0"
|
||||
if "p1" in text or "immediate" in text or "next sprint" in text:
|
||||
return "priority/p1"
|
||||
if "p2" in text:
|
||||
return "priority/p2"
|
||||
if "p3" in text or "future" in text or "later" in text:
|
||||
return "priority/p3"
|
||||
return "priority/p2"
|
||||
|
||||
|
||||
def normalize_priority(value: str) -> str:
|
||||
value = value.strip().lower()
|
||||
if value == "p0":
|
||||
return "priority/p0"
|
||||
if value == "p1":
|
||||
return "priority/p1"
|
||||
if value == "p2":
|
||||
return "priority/p2"
|
||||
if value == "p3":
|
||||
return "priority/p3"
|
||||
return "priority/p2"
|
||||
|
||||
|
||||
def milestone_from_context(path: pathlib.Path, section: str) -> str:
|
||||
if section:
|
||||
parts = [part for part in section.split(" > ") if ACTIVE_SECTION_RE.search(part)]
|
||||
if parts:
|
||||
return parts[-1][:120]
|
||||
return path.stem.replace("_", " ").replace("-", " ").title()
|
||||
|
||||
|
||||
def format_title(prefix: str, item: str) -> str:
|
||||
title = f"{prefix} {clean_text(item).rstrip('.')}"
|
||||
if len(title) <= 180:
|
||||
return title
|
||||
return f"{title[:177].rstrip()}..."
|
||||
|
||||
|
||||
def make_fingerprint(path: pathlib.Path, line: int, repo_key: str, item: str) -> str:
|
||||
stable = "\n".join([str(path.resolve()), str(line), repo_key, item])
|
||||
return hashlib.sha256(stable.encode("utf-8")).hexdigest()[:24]
|
||||
|
||||
|
||||
def dedupe_candidates(candidates: list[Candidate]) -> list[Candidate]:
|
||||
seen: set[tuple[str, str]] = set()
|
||||
unique: list[Candidate] = []
|
||||
for candidate in candidates:
|
||||
key = (candidate.repo_key, normalize_title(candidate.title))
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
unique.append(candidate)
|
||||
return unique
|
||||
|
||||
|
||||
def grouped_counts(candidates: list[Candidate]) -> dict[str, int]:
|
||||
counts: dict[str, int] = {}
|
||||
for candidate in candidates:
|
||||
counts[candidate.repo_key] = counts.get(candidate.repo_key, 0) + 1
|
||||
return dict(sorted(counts.items()))
|
||||
|
||||
|
||||
def load_repo_state(repo: RepoInfo, token: str, *, apply: bool) -> RepoState:
|
||||
target = infer_target(repo.root)
|
||||
client = GiteaClient(target, token)
|
||||
label_ids = label_ids_by_name(client, target.owner, target.repo)
|
||||
if apply:
|
||||
for name, color, description in GENERIC_LABELS:
|
||||
if name in label_ids:
|
||||
continue
|
||||
created = client.request_json(
|
||||
"POST",
|
||||
repo_path(target.owner, target.repo, "/labels"),
|
||||
body={"name": name, "color": color, "description": description, "exclusive": name.startswith(("type/", "status/", "priority/"))},
|
||||
)
|
||||
label_ids[str(created["name"])] = int(created["id"])
|
||||
print(f"created label {target.owner}/{target.repo}:{name}")
|
||||
|
||||
missing = [name for name, _, _ in GENERIC_LABELS if name not in label_ids]
|
||||
if missing and apply:
|
||||
raise GiteaError(f"{repo.key} missing labels: {', '.join(missing)}. Re-run with --apply to create them.")
|
||||
|
||||
milestones = client.paginate(repo_path(target.owner, target.repo, "/milestones"), query={"state": "all"}, limit=50)
|
||||
milestone_ids = {str(item["title"]): int(item["id"]) for item in milestones}
|
||||
issues = client.paginate(repo_path(target.owner, target.repo, "/issues"), query={"state": "all"}, limit=50)
|
||||
fingerprints: set[str] = set()
|
||||
titles: set[str] = set()
|
||||
for issue in issues:
|
||||
body = str(issue.get("body") or "")
|
||||
titles.add(normalize_title(str(issue.get("title") or "")))
|
||||
fingerprints.update(re.findall(r"codex-(?:generic-)?backlog-fingerprint:([0-9a-f]{24})", body))
|
||||
return RepoState(target=target, client=client, label_ids=label_ids, milestone_ids=milestone_ids, fingerprints=fingerprints, titles=titles)
|
||||
|
||||
|
||||
def ensure_milestone(state: RepoState, title: str) -> int:
|
||||
if title in state.milestone_ids:
|
||||
return state.milestone_ids[title]
|
||||
created = state.client.request_json(
|
||||
"POST",
|
||||
repo_path(state.target.owner, state.target.repo, "/milestones"),
|
||||
body={"title": title, "state": "open", "description": "Created from imported backlog-like files."},
|
||||
)
|
||||
state.milestone_ids[title] = int(created["id"])
|
||||
print(f"created milestone {state.target.owner}/{state.target.repo}:{title}")
|
||||
return state.milestone_ids[title]
|
||||
|
||||
|
||||
def normalize_title(title: str) -> str:
|
||||
return re.sub(r"[^a-z0-9]+", " ", title.lower()).strip()
|
||||
|
||||
|
||||
def print_preview(candidates: list[Candidate], limit_per_repo: int = 80) -> None:
|
||||
by_repo: dict[str, list[Candidate]] = {}
|
||||
for candidate in candidates:
|
||||
by_repo.setdefault(candidate.repo_key, []).append(candidate)
|
||||
for repo_key, items in sorted(by_repo.items()):
|
||||
print(f"{repo_key}:")
|
||||
for item in items[:limit_per_repo]:
|
||||
print(f" {item.title} [{item.priority}; {item.milestone}]")
|
||||
if len(items) > limit_per_repo:
|
||||
print(f" ... {len(items) - limit_per_repo} more")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
113
tools/gitea/gitea-install-workflow.py
Normal file
113
tools/gitea/gitea-install-workflow.py
Normal file
@@ -0,0 +1,113 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Install Gitea issue workflow files into one or more repositories."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
from gitea_common import GiteaError, repo_root
|
||||
|
||||
|
||||
SOURCE_ROOT = pathlib.Path(__file__).resolve().parents[2]
|
||||
WORKFLOW_FILES = (
|
||||
".gitea/PULL_REQUEST_TEMPLATE.md",
|
||||
".gitea/ISSUE_TEMPLATE/bug_report.md",
|
||||
".gitea/ISSUE_TEMPLATE/config.yaml",
|
||||
".gitea/ISSUE_TEMPLATE/docs_workflow.md",
|
||||
".gitea/ISSUE_TEMPLATE/feature_request.md",
|
||||
".gitea/ISSUE_TEMPLATE/task.md",
|
||||
".gitea/ISSUE_TEMPLATE/tech_debt.md",
|
||||
)
|
||||
MODULE_LABEL_BY_REPO = {
|
||||
"govoplan-core": "module/core",
|
||||
"govoplan-access": "module/access",
|
||||
"govoplan-admin": "module/admin",
|
||||
"govoplan-tenancy": "module/tenancy",
|
||||
"govoplan-policy": "module/policy",
|
||||
"govoplan-audit": "module/audit",
|
||||
"govoplan-mail": "module/mail",
|
||||
"govoplan-files": "module/files",
|
||||
"govoplan-campaign": "module/campaign",
|
||||
"addideas-govoplan-website": "area/marketing",
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("targets", nargs="*", type=pathlib.Path, help="target repository roots or child paths")
|
||||
parser.add_argument("--module-label", help="module label to write into issue templates; only valid for one target")
|
||||
parser.add_argument("--include-labels-file", action="store_true", help="also copy docs/gitea-labels.json")
|
||||
parser.add_argument("--apply", action="store_true", help="write files instead of previewing changes")
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
target_roots = [repo_root(path) for path in (args.targets or [pathlib.Path.cwd()])]
|
||||
unique_roots = list(dict.fromkeys(target_roots))
|
||||
if args.module_label and len(unique_roots) != 1:
|
||||
raise GiteaError("--module-label can only be used with a single target")
|
||||
|
||||
rel_paths = list(WORKFLOW_FILES)
|
||||
if args.include_labels_file:
|
||||
rel_paths.append("docs/gitea-labels.json")
|
||||
|
||||
for target_root in unique_roots:
|
||||
module_label = args.module_label or infer_module_label(target_root)
|
||||
install_files(target_root, rel_paths, module_label, apply=args.apply)
|
||||
|
||||
if not args.apply:
|
||||
print("Dry run only. Re-run with --apply to write files.")
|
||||
return 0
|
||||
except GiteaError as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
def infer_module_label(target_root: pathlib.Path) -> str:
|
||||
repo_name = target_root.name
|
||||
try:
|
||||
repo_name = target_root.resolve().name
|
||||
except OSError:
|
||||
pass
|
||||
label = MODULE_LABEL_BY_REPO.get(repo_name)
|
||||
if label:
|
||||
return label
|
||||
if repo_name.startswith("govoplan-"):
|
||||
suffix = repo_name.removeprefix("govoplan-")
|
||||
if suffix:
|
||||
return f"module/{suffix}"
|
||||
raise GiteaError(f"cannot infer repository label for {target_root}. Use --module-label module/<name> or area/<name>.")
|
||||
|
||||
|
||||
def install_files(target_root: pathlib.Path, rel_paths: list[str], module_label: str, *, apply: bool) -> None:
|
||||
print(f"Target: {target_root} ({module_label})")
|
||||
for rel_path in rel_paths:
|
||||
source = SOURCE_ROOT / rel_path
|
||||
target = target_root / rel_path
|
||||
if not source.exists():
|
||||
raise GiteaError(f"missing workflow source file: {source}")
|
||||
|
||||
content = source.read_text(encoding="utf-8")
|
||||
content = transform_content(rel_path, content, module_label)
|
||||
|
||||
existing = target.read_text(encoding="utf-8") if target.exists() else None
|
||||
if existing == content:
|
||||
print(f"unchanged {rel_path}")
|
||||
continue
|
||||
|
||||
action = "create" if existing is None else "update"
|
||||
print(f"{action if apply else 'would ' + action} {rel_path}")
|
||||
if apply:
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_text(content, encoding="utf-8")
|
||||
|
||||
|
||||
def transform_content(rel_path: str, content: str, module_label: str) -> str:
|
||||
if rel_path.startswith(".gitea/ISSUE_TEMPLATE/") and rel_path.endswith(".md"):
|
||||
return content.replace(" - module/core", f" - {module_label}")
|
||||
return content
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
250
tools/gitea/gitea-migrate-org-labels.py
Normal file
250
tools/gitea/gitea-migrate-org-labels.py
Normal file
@@ -0,0 +1,250 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Move issue labels from repository-local labels to organization labels."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
from gitea_common import (
|
||||
GiteaClient,
|
||||
GiteaError,
|
||||
infer_target,
|
||||
load_dotenv,
|
||||
org_path,
|
||||
repo_path,
|
||||
repo_root,
|
||||
require_token,
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--root", type=pathlib.Path, default=pathlib.Path.cwd(), help="repository root used for Gitea URL inference")
|
||||
parser.add_argument("--remote", default="origin", help="git remote to use for target inference")
|
||||
parser.add_argument("--env-file", type=pathlib.Path, default=pathlib.Path("/home/zemion/.config/gitea/gitea.env"))
|
||||
parser.add_argument("--org", help="organization owner; defaults to inferred owner")
|
||||
parser.add_argument("--repo", action="append", default=[], help="repository name to process; may be repeated")
|
||||
parser.add_argument("--repo-regex", default=".*", help="only process repositories whose name matches this regex")
|
||||
parser.add_argument(
|
||||
"--issue-mode",
|
||||
choices=("replace", "delete-add"),
|
||||
default="replace",
|
||||
help="replace labels in one request, or delete local ids before adding org ids",
|
||||
)
|
||||
parser.add_argument("--delete-repo-labels", action="store_true", help="delete matching repository-local labels after issue and PR migration")
|
||||
parser.add_argument("--apply", action="store_true", help="apply changes; omit for dry-run")
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
root = repo_root(args.root)
|
||||
load_dotenv(args.env_file or root / ".env")
|
||||
target = infer_target(root, args.remote)
|
||||
owner = args.org or target.owner
|
||||
client = GiteaClient(target, require_token() if args.apply else os.environ.get("GITEA_TOKEN"))
|
||||
if client.token is None:
|
||||
raise GiteaError("GITEA_TOKEN is required for dry-run and apply because migration inspects live issue labels.")
|
||||
|
||||
repo_filter = re.compile(args.repo_regex)
|
||||
org_labels = _labels_by_name(client.paginate(org_path(owner, "/labels"), limit=100))
|
||||
if not org_labels:
|
||||
raise GiteaError(f"{owner} has no organization labels")
|
||||
|
||||
repos = _selected_repositories(client, owner, args.repo, repo_filter)
|
||||
totals = Totals()
|
||||
print(f"Organization: {owner}")
|
||||
print(f"Repositories: {len(repos)}")
|
||||
print(f"Mode: {'apply' if args.apply else 'dry-run'}")
|
||||
print(f"Delete repository labels: {args.delete_repo_labels}")
|
||||
|
||||
for index, repo in enumerate(repos, start=1):
|
||||
repo_result = migrate_repository(
|
||||
client,
|
||||
owner=owner,
|
||||
repo=repo,
|
||||
org_labels=org_labels,
|
||||
issue_mode=args.issue_mode,
|
||||
apply=args.apply,
|
||||
delete_repo_labels=args.delete_repo_labels,
|
||||
)
|
||||
totals.add(repo_result)
|
||||
if repo_result.has_work:
|
||||
print(
|
||||
f"[{index}/{len(repos)}] {repo}: "
|
||||
f"issue-labels={repo_result.issue_label_migrations}, "
|
||||
f"delete-labels={repo_result.repo_label_deletions}, "
|
||||
f"errors={len(repo_result.errors)}"
|
||||
)
|
||||
for error in repo_result.errors:
|
||||
print(f" error: {error}")
|
||||
else:
|
||||
print(f"[{index}/{len(repos)}] {repo}: no matching local labels")
|
||||
|
||||
print("Summary:")
|
||||
print(f" repositories processed: {totals.repositories}")
|
||||
print(f" repositories with matching local labels: {totals.repositories_with_work}")
|
||||
print(f" issue/PR local labels migrated: {totals.issue_label_migrations}")
|
||||
print(f" repository labels deleted: {totals.repo_label_deletions}")
|
||||
print(f" errors: {totals.errors}")
|
||||
if totals.errors:
|
||||
return 1
|
||||
return 0
|
||||
except GiteaError as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
class RepoResult:
|
||||
def __init__(self) -> None:
|
||||
self.issue_label_migrations = 0
|
||||
self.repo_label_deletions = 0
|
||||
self.errors: list[str] = []
|
||||
self.matched_local_labels = 0
|
||||
|
||||
@property
|
||||
def has_work(self) -> bool:
|
||||
return bool(self.matched_local_labels or self.issue_label_migrations or self.repo_label_deletions or self.errors)
|
||||
|
||||
|
||||
class Totals:
|
||||
def __init__(self) -> None:
|
||||
self.repositories = 0
|
||||
self.repositories_with_work = 0
|
||||
self.issue_label_migrations = 0
|
||||
self.repo_label_deletions = 0
|
||||
self.errors = 0
|
||||
|
||||
def add(self, result: RepoResult) -> None:
|
||||
self.repositories += 1
|
||||
if result.has_work:
|
||||
self.repositories_with_work += 1
|
||||
self.issue_label_migrations += result.issue_label_migrations
|
||||
self.repo_label_deletions += result.repo_label_deletions
|
||||
self.errors += len(result.errors)
|
||||
|
||||
|
||||
def migrate_repository(
|
||||
client: GiteaClient,
|
||||
*,
|
||||
owner: str,
|
||||
repo: str,
|
||||
org_labels: dict[str, dict[str, Any]],
|
||||
issue_mode: str,
|
||||
apply: bool,
|
||||
delete_repo_labels: bool,
|
||||
) -> RepoResult:
|
||||
result = RepoResult()
|
||||
repo_labels = client.paginate(repo_path(owner, repo, "/labels"), limit=100)
|
||||
local_labels = {
|
||||
str(label.get("name") or ""): label
|
||||
for label in repo_labels
|
||||
if str(label.get("name") or "") in org_labels
|
||||
}
|
||||
result.matched_local_labels = len(local_labels)
|
||||
if not local_labels:
|
||||
return result
|
||||
|
||||
local_by_id = {_label_id(label): label for label in local_labels.values() if _label_id(label) is not None}
|
||||
org_by_name = {name: _label_id(label) for name, label in org_labels.items()}
|
||||
|
||||
for issue_type in ("issues", "pulls"):
|
||||
issues = client.paginate(
|
||||
repo_path(owner, repo, "/issues"),
|
||||
query={"state": "all", "type": issue_type},
|
||||
limit=100,
|
||||
)
|
||||
for issue in issues:
|
||||
issue_number = int(issue.get("number") or issue.get("index"))
|
||||
issue_label_ids = [_label_id(label) for label in issue.get("labels") or []]
|
||||
issue_label_ids = [label_id for label_id in issue_label_ids if label_id is not None]
|
||||
final_label_ids = list(issue_label_ids)
|
||||
changed = False
|
||||
migrated_count = 0
|
||||
local_ids_to_remove: list[int] = []
|
||||
org_ids_to_add: list[int] = []
|
||||
for label in issue.get("labels") or []:
|
||||
local_id = _label_id(label)
|
||||
if local_id is None or local_id not in local_by_id:
|
||||
continue
|
||||
name = str(label.get("name") or "")
|
||||
org_id = org_by_name.get(name)
|
||||
if org_id is None:
|
||||
continue
|
||||
local_ids_to_remove.append(local_id)
|
||||
if org_id not in issue_label_ids and org_id not in org_ids_to_add:
|
||||
org_ids_to_add.append(org_id)
|
||||
final_label_ids = [label_id for label_id in final_label_ids if label_id != local_id]
|
||||
if org_id not in final_label_ids:
|
||||
final_label_ids.append(org_id)
|
||||
changed = True
|
||||
migrated_count += 1
|
||||
if changed:
|
||||
if apply:
|
||||
if issue_mode == "delete-add":
|
||||
for local_id in local_ids_to_remove:
|
||||
_remove_issue_label(client, owner, repo, issue_number, local_id)
|
||||
if org_ids_to_add:
|
||||
client.request_json(
|
||||
"POST",
|
||||
repo_path(owner, repo, f"/issues/{issue_number}/labels"),
|
||||
body={"labels": org_ids_to_add},
|
||||
)
|
||||
else:
|
||||
client.request_json(
|
||||
"PUT",
|
||||
repo_path(owner, repo, f"/issues/{issue_number}/labels"),
|
||||
body={"labels": final_label_ids},
|
||||
)
|
||||
result.issue_label_migrations += migrated_count
|
||||
|
||||
if delete_repo_labels:
|
||||
for name, label in sorted(local_labels.items()):
|
||||
label_id = _label_id(label)
|
||||
if label_id is None:
|
||||
result.errors.append(f"{name} has no label id")
|
||||
continue
|
||||
if apply:
|
||||
try:
|
||||
client.request_json("DELETE", repo_path(owner, repo, f"/labels/{label_id}"))
|
||||
except GiteaError as exc:
|
||||
result.errors.append(f"delete repository label {name}: {exc}")
|
||||
continue
|
||||
result.repo_label_deletions += 1
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def _selected_repositories(client: GiteaClient, owner: str, explicit: list[str], repo_filter: re.Pattern[str]) -> list[str]:
|
||||
if explicit:
|
||||
return sorted(set(explicit))
|
||||
repos = client.paginate(org_path(owner, "/repos"), query={"type": "all"}, limit=100)
|
||||
names = sorted(str(repo.get("name") or "") for repo in repos if repo.get("name"))
|
||||
return [name for name in names if repo_filter.search(name)]
|
||||
|
||||
|
||||
def _labels_by_name(labels: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
|
||||
return {str(label.get("name")): label for label in labels if label.get("name") and _label_id(label) is not None}
|
||||
|
||||
|
||||
def _label_id(label: dict[str, Any]) -> int | None:
|
||||
value = label.get("id") or label.get("index")
|
||||
if value is None:
|
||||
return None
|
||||
return int(value)
|
||||
|
||||
|
||||
def _remove_issue_label(client: GiteaClient, owner: str, repo: str, issue_number: int, label_id: int) -> None:
|
||||
try:
|
||||
client.request_json("DELETE", repo_path(owner, repo, f"/issues/{issue_number}/labels/{label_id}"))
|
||||
except GiteaError as exc:
|
||||
if "HTTP 404" in str(exc):
|
||||
return
|
||||
raise
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
180
tools/gitea/gitea-sync-labels.py
Normal file
180
tools/gitea/gitea-sync-labels.py
Normal file
@@ -0,0 +1,180 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Synchronize issue labels to a Gitea repository."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import pathlib
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
from gitea_common import (
|
||||
GiteaClient,
|
||||
GiteaError,
|
||||
infer_target,
|
||||
load_dotenv,
|
||||
load_json,
|
||||
org_path,
|
||||
repo_path,
|
||||
repo_root,
|
||||
require_token,
|
||||
)
|
||||
|
||||
|
||||
DEFAULT_LABELS_FILE = pathlib.Path(__file__).resolve().parents[2] / "docs" / "gitea-labels.json"
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--root", type=pathlib.Path, default=pathlib.Path.cwd(), help="repository root or child path")
|
||||
parser.add_argument("--remote", default="origin", help="git remote to use for target inference")
|
||||
parser.add_argument("--labels-file", type=pathlib.Path, default=DEFAULT_LABELS_FILE)
|
||||
parser.add_argument("--env-file", type=pathlib.Path, help="dotenv file to read before using GITEA_* values")
|
||||
parser.add_argument(
|
||||
"--scope",
|
||||
choices=("repository", "organization"),
|
||||
default="repository",
|
||||
help="sync repository labels or organization labels",
|
||||
)
|
||||
parser.add_argument("--org", help="organization name for --scope organization; defaults to inferred owner")
|
||||
parser.add_argument("--apply", action="store_true", help="create or update labels in Gitea")
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
root = repo_root(args.root)
|
||||
load_dotenv(args.env_file or root / ".env")
|
||||
target = infer_target(root, args.remote)
|
||||
labels = _load_labels(args.labels_file)
|
||||
token = require_token() if args.apply else os.environ.get("GITEA_TOKEN")
|
||||
org_name = args.org or target.owner
|
||||
|
||||
if args.scope == "organization":
|
||||
print(f"Target: {target.base_url.rstrip('/')}/{org_name} organization labels")
|
||||
else:
|
||||
print(f"Target: {target.display}")
|
||||
print(f"Labels file: {args.labels_file}")
|
||||
|
||||
if not args.apply and not token:
|
||||
print("Dry run without GITEA_TOKEN: API comparison skipped.")
|
||||
for label in labels:
|
||||
print(f"would ensure {label['name']} ({label['color']})")
|
||||
return 0
|
||||
|
||||
client = GiteaClient(target, token)
|
||||
if args.scope == "organization":
|
||||
existing = _org_labels_by_name(client, org_name)
|
||||
else:
|
||||
existing = _repo_labels_by_name(client, target.owner, target.repo)
|
||||
creates: list[dict[str, Any]] = []
|
||||
updates: list[tuple[dict[str, Any], dict[str, Any]]] = []
|
||||
|
||||
for label in labels:
|
||||
current = existing.get(label["name"])
|
||||
if current is None:
|
||||
creates.append(label)
|
||||
continue
|
||||
|
||||
patch = _diff_label(current, label)
|
||||
if patch:
|
||||
updates.append((current, patch))
|
||||
|
||||
if not creates and not updates:
|
||||
print("No label changes needed.")
|
||||
return 0
|
||||
|
||||
for label in creates:
|
||||
print(f"{'create' if args.apply else 'would create'} {label['name']}")
|
||||
if args.apply:
|
||||
client.request_json("POST", _create_path(args.scope, org_name, target.owner, target.repo), body=label)
|
||||
|
||||
for current, patch in updates:
|
||||
print(f"{'update' if args.apply else 'would update'} {current['name']}: {', '.join(sorted(patch))}")
|
||||
if args.apply:
|
||||
label_id = current.get("id") or current.get("index")
|
||||
if label_id is None:
|
||||
raise GiteaError(f"{current['name']} has no label id in API response")
|
||||
client.request_json(
|
||||
"PATCH",
|
||||
_update_path(args.scope, org_name, target.owner, target.repo, int(label_id)),
|
||||
body=patch,
|
||||
)
|
||||
|
||||
return 0
|
||||
except GiteaError as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
def _load_labels(path: pathlib.Path) -> list[dict[str, Any]]:
|
||||
payload = load_json(path)
|
||||
if not isinstance(payload, list):
|
||||
raise GiteaError(f"{path} must contain a JSON list")
|
||||
|
||||
labels: list[dict[str, Any]] = []
|
||||
seen: set[str] = set()
|
||||
for index, item in enumerate(payload, start=1):
|
||||
if not isinstance(item, dict):
|
||||
raise GiteaError(f"label #{index} must be an object")
|
||||
label = {
|
||||
"name": str(item.get("name", "")).strip(),
|
||||
"color": str(item.get("color", "")).strip().lstrip("#").lower(),
|
||||
"description": str(item.get("description", "")).strip(),
|
||||
"exclusive": bool(item.get("exclusive", False)),
|
||||
}
|
||||
if not label["name"]:
|
||||
raise GiteaError(f"label #{index} is missing name")
|
||||
if label["name"] in seen:
|
||||
raise GiteaError(f"duplicate label name: {label['name']}")
|
||||
if not _is_hex_color(label["color"]):
|
||||
raise GiteaError(f"{label['name']} has invalid color {label['color']!r}")
|
||||
seen.add(label["name"])
|
||||
labels.append(label)
|
||||
return labels
|
||||
|
||||
|
||||
def _repo_labels_by_name(client: GiteaClient, owner: str, repo: str) -> dict[str, dict[str, Any]]:
|
||||
labels = client.paginate(repo_path(owner, repo, "/labels"))
|
||||
return {str(label.get("name")): label for label in labels}
|
||||
|
||||
|
||||
def _org_labels_by_name(client: GiteaClient, owner: str) -> dict[str, dict[str, Any]]:
|
||||
labels = client.paginate(org_path(owner, "/labels"))
|
||||
return {str(label.get("name")): label for label in labels}
|
||||
|
||||
|
||||
def _create_path(scope: str, org: str, owner: str, repo: str) -> str:
|
||||
if scope == "organization":
|
||||
return org_path(org, "/labels")
|
||||
return repo_path(owner, repo, "/labels")
|
||||
|
||||
|
||||
def _update_path(scope: str, org: str, owner: str, repo: str, label_id: int) -> str:
|
||||
if scope == "organization":
|
||||
return org_path(org, f"/labels/{label_id}")
|
||||
return repo_path(owner, repo, f"/labels/{label_id}")
|
||||
|
||||
|
||||
def _diff_label(current: dict[str, Any], desired: dict[str, Any]) -> dict[str, Any]:
|
||||
patch: dict[str, Any] = {}
|
||||
if _normalize_color(current.get("color")) != desired["color"]:
|
||||
patch["color"] = desired["color"]
|
||||
if str(current.get("description") or "").strip() != desired["description"]:
|
||||
patch["description"] = desired["description"]
|
||||
if bool(current.get("exclusive", False)) != desired["exclusive"]:
|
||||
patch["exclusive"] = desired["exclusive"]
|
||||
return patch
|
||||
|
||||
|
||||
def _normalize_color(value: Any) -> str:
|
||||
return str(value or "").strip().lstrip("#").lower()
|
||||
|
||||
|
||||
def _is_hex_color(value: str) -> bool:
|
||||
if len(value) != 6:
|
||||
return False
|
||||
return all(char in "0123456789abcdef" for char in value)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
637
tools/gitea/gitea-sync-wiki.py
Normal file
637
tools/gitea/gitea-sync-wiki.py
Normal file
@@ -0,0 +1,637 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Mirror project documentation files into Gitea repository wikis."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import dataclasses
|
||||
import hashlib
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.parse
|
||||
from typing import Any
|
||||
|
||||
from gitea_common import GiteaClient, GiteaError, infer_target, load_dotenv, repo_path, require_token
|
||||
|
||||
|
||||
GIT_ROOT = pathlib.Path("/mnt/DATA/git")
|
||||
PRODUCTS_ROOT = pathlib.Path("/mnt/DATA/Nextcloud/ADD ideas UG/Products")
|
||||
MANAGED_MARKER = "<!-- codex-wiki-sync:"
|
||||
TEXT_SUFFIXES = {".md", ".txt", ".csv", ".toml", ".json", ".yaml", ".yml", ".rst"}
|
||||
SENSITIVE_NAME_RE = re.compile(r"(password|passwd|secret|credential|api[_-]?keys?|token|private[_-]?key)", re.IGNORECASE)
|
||||
EXCLUDED_PARTS = {
|
||||
".git",
|
||||
".gitea",
|
||||
".venv",
|
||||
"node_modules",
|
||||
"__pycache__",
|
||||
"dist",
|
||||
"build",
|
||||
".cache",
|
||||
".module-test-build",
|
||||
}
|
||||
EXCLUDED_NAMES = {
|
||||
"package-lock.json",
|
||||
"pnpm-lock.yaml",
|
||||
"yarn.lock",
|
||||
"uv.lock",
|
||||
}
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class RepoInfo:
|
||||
root: pathlib.Path
|
||||
owner: str
|
||||
repo: str
|
||||
|
||||
@property
|
||||
def key(self) -> str:
|
||||
return self.root.name
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class WikiSource:
|
||||
repo_key: str
|
||||
path: pathlib.Path
|
||||
origin: str
|
||||
page_title: str
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--env-file", type=pathlib.Path, default=pathlib.Path("/home/zemion/.config/gitea/gitea.env"))
|
||||
parser.add_argument("--git-root", type=pathlib.Path, default=GIT_ROOT)
|
||||
parser.add_argument("--products-root", type=pathlib.Path, default=PRODUCTS_ROOT)
|
||||
parser.add_argument("--repo", action="append", help="limit to one or more local repository directory names")
|
||||
parser.add_argument("--page", action="append", help="limit to one or more generated wiki page names")
|
||||
parser.add_argument("--overwrite-unmanaged", action="store_true", help="update existing wiki pages not previously managed by this script")
|
||||
parser.add_argument("--transport", choices=("git", "api"), default="git", help="sync through the wiki git repository or the Gitea REST API")
|
||||
parser.add_argument("--wiki-cache-dir", type=pathlib.Path, default=pathlib.Path("/tmp/codex-gitea-wiki-sync"), help="local cache for wiki git checkouts")
|
||||
parser.add_argument("--commit-message", default="Sync wiki from project files", help="commit message used by git transport")
|
||||
parser.add_argument("--prune-managed", action="store_true", help="delete managed wiki pages whose source files are no longer discovered")
|
||||
parser.add_argument("--apply", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
load_dotenv(args.env_file)
|
||||
repos = discover_hosted_repos(args.git_root)
|
||||
if args.repo:
|
||||
requested = set(args.repo)
|
||||
repos = {key: repo for key, repo in repos.items() if key in requested}
|
||||
missing = requested - set(repos)
|
||||
if missing:
|
||||
raise GiteaError(f"requested repos are not hosted on git.add-ideas.de locally: {', '.join(sorted(missing))}")
|
||||
|
||||
sources = discover_sources(repos, args.products_root)
|
||||
if args.page:
|
||||
requested_pages = set(args.page)
|
||||
sources = [source for source in sources if source.page_title in requested_pages]
|
||||
missing_pages = requested_pages - {source.page_title for source in sources}
|
||||
if missing_pages:
|
||||
raise GiteaError(f"requested pages were not discovered: {', '.join(sorted(missing_pages))}")
|
||||
print(f"Hosted repositories: {len(repos)}")
|
||||
print(f"Wiki source files: {len(sources)}")
|
||||
for repo_key, count in grouped_counts(sources).items():
|
||||
print(f" {repo_key}: {count}")
|
||||
|
||||
if not args.apply:
|
||||
preview_sources(sources)
|
||||
print("Dry run only. Re-run with --apply to write wiki pages.")
|
||||
return 0
|
||||
|
||||
token = require_token() if args.transport == "api" else ""
|
||||
failures = 0
|
||||
for repo_key, repo_sources in group_sources(sources).items():
|
||||
repo = repos[repo_key]
|
||||
try:
|
||||
if args.transport == "git":
|
||||
sync_repo_wiki_git(
|
||||
repo,
|
||||
repo_sources,
|
||||
cache_dir=args.wiki_cache_dir,
|
||||
overwrite_unmanaged=args.overwrite_unmanaged,
|
||||
commit_message=args.commit_message,
|
||||
write_index=not bool(args.page),
|
||||
prune_managed=args.prune_managed and not bool(args.page),
|
||||
)
|
||||
else:
|
||||
sync_repo_wiki(
|
||||
repo,
|
||||
repo_sources,
|
||||
token,
|
||||
overwrite_unmanaged=args.overwrite_unmanaged,
|
||||
write_index=not bool(args.page),
|
||||
prune_managed=args.prune_managed and not bool(args.page),
|
||||
)
|
||||
except GiteaError as exc:
|
||||
failures += 1
|
||||
print(f"error syncing wiki for {repo_key}: {exc}", file=sys.stderr)
|
||||
return 1 if failures else 0
|
||||
except GiteaError as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
def discover_hosted_repos(git_root: pathlib.Path) -> dict[str, RepoInfo]:
|
||||
repos: dict[str, RepoInfo] = {}
|
||||
for gitdir in sorted(git_root.glob("*/.git")):
|
||||
root = gitdir.parent
|
||||
result = subprocess.run(
|
||||
["git", "-C", str(root), "remote", "get-url", "origin"],
|
||||
check=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
text=True,
|
||||
)
|
||||
if "git.add-ideas.de" not in result.stdout:
|
||||
continue
|
||||
target = infer_target(root)
|
||||
repos[root.name] = RepoInfo(root=root, owner=target.owner, repo=target.repo)
|
||||
return repos
|
||||
|
||||
|
||||
def discover_sources(repos: dict[str, RepoInfo], products_root: pathlib.Path) -> list[WikiSource]:
|
||||
sources: list[WikiSource] = []
|
||||
for repo in repos.values():
|
||||
for path in repo.root.rglob("*"):
|
||||
if is_repo_doc(repo.root, path):
|
||||
rel = path.relative_to(repo.root)
|
||||
sources.append(
|
||||
WikiSource(
|
||||
repo_key=repo.key,
|
||||
path=path,
|
||||
origin="repository",
|
||||
page_title=title_for_path("Repo", rel),
|
||||
)
|
||||
)
|
||||
|
||||
product_map = {
|
||||
"co2api": "emission-api-lib",
|
||||
"govoplan": "govoplan-core",
|
||||
"meubility": "meubility-workbench",
|
||||
"mousehold": "mousehold",
|
||||
"prvnncr": "prvnncr-server",
|
||||
}
|
||||
if products_root.exists():
|
||||
for product_dir in sorted(path for path in products_root.iterdir() if path.is_dir()):
|
||||
repo_key = product_map.get(product_dir.name)
|
||||
if not repo_key or repo_key not in repos:
|
||||
continue
|
||||
for path in product_dir.rglob("*"):
|
||||
if is_product_doc(path):
|
||||
rel = path.relative_to(product_dir)
|
||||
sources.append(
|
||||
WikiSource(
|
||||
repo_key=repo_key,
|
||||
path=path,
|
||||
origin=f"product:{product_dir.name}",
|
||||
page_title=title_for_path(f"Product {product_dir.name}", rel),
|
||||
)
|
||||
)
|
||||
|
||||
unique: dict[tuple[str, str], WikiSource] = {}
|
||||
for source in sources:
|
||||
unique[(source.repo_key, source.page_title)] = source
|
||||
return sorted(unique.values(), key=lambda item: (item.repo_key, item.page_title))
|
||||
|
||||
|
||||
def is_repo_doc(repo_root_path: pathlib.Path, path: pathlib.Path) -> bool:
|
||||
if not path.is_file() or not is_text_file(path):
|
||||
return False
|
||||
parts = set(path.relative_to(repo_root_path).parts)
|
||||
if parts & EXCLUDED_PARTS:
|
||||
return False
|
||||
if path.name in EXCLUDED_NAMES:
|
||||
return False
|
||||
if SENSITIVE_NAME_RE.search(path.name):
|
||||
return False
|
||||
rel = path.relative_to(repo_root_path)
|
||||
if len(rel.parts) == 1 and path.name.lower().startswith(("readme", "license", "changelog", "contributing", "security")):
|
||||
return True
|
||||
if "generated" in rel.parts:
|
||||
return False
|
||||
if rel.parts[0] in {"docs", "doc", "codex"} and path.suffix.lower() in {".md", ".txt", ".csv"}:
|
||||
return True
|
||||
if re.search(r"(backlog|todo|roadmap|plan|architecture|workflow|manifest)", path.name, re.IGNORECASE):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def is_product_doc(path: pathlib.Path) -> bool:
|
||||
if not path.is_file() or not is_text_file(path):
|
||||
return False
|
||||
if path.name in EXCLUDED_NAMES:
|
||||
return False
|
||||
if SENSITIVE_NAME_RE.search(path.name):
|
||||
return False
|
||||
if any(part in {"python_backup", "bruno", "fluege"} for part in path.parts):
|
||||
return False
|
||||
if re.search(
|
||||
r"(readme|todo|roadmap|plan|concept|continuation|copyright|notes|whitepaper|pitch|request_response)",
|
||||
path.name,
|
||||
re.IGNORECASE,
|
||||
) and path.suffix.lower() in {".md", ".txt"}:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def is_text_file(path: pathlib.Path) -> bool:
|
||||
if path.suffix.lower() not in TEXT_SUFFIXES:
|
||||
return False
|
||||
try:
|
||||
chunk = path.read_bytes()[:4096]
|
||||
except OSError:
|
||||
return False
|
||||
return b"\x00" not in chunk
|
||||
|
||||
|
||||
def title_for_path(prefix: str, rel: pathlib.Path) -> str:
|
||||
stem = rel.with_suffix("").as_posix()
|
||||
stem = re.sub(r"[^A-Za-z0-9]+", "-", stem).strip("-")
|
||||
return f"{prefix}-{stem}"[:180]
|
||||
|
||||
|
||||
def group_sources(sources: list[WikiSource]) -> dict[str, list[WikiSource]]:
|
||||
grouped: dict[str, list[WikiSource]] = {}
|
||||
for source in sources:
|
||||
grouped.setdefault(source.repo_key, []).append(source)
|
||||
return dict(sorted(grouped.items()))
|
||||
|
||||
|
||||
def grouped_counts(sources: list[WikiSource]) -> dict[str, int]:
|
||||
return {repo_key: len(items) for repo_key, items in group_sources(sources).items()}
|
||||
|
||||
|
||||
def preview_sources(sources: list[WikiSource]) -> None:
|
||||
for repo_key, repo_sources in group_sources(sources).items():
|
||||
print(f"{repo_key}:")
|
||||
for source in repo_sources[:80]:
|
||||
print(f" {source.page_title} <- {source.path}")
|
||||
if len(repo_sources) > 80:
|
||||
print(f" ... {len(repo_sources) - 80} more")
|
||||
|
||||
|
||||
def sync_repo_wiki(
|
||||
repo: RepoInfo,
|
||||
sources: list[WikiSource],
|
||||
token: str,
|
||||
*,
|
||||
overwrite_unmanaged: bool,
|
||||
write_index: bool = True,
|
||||
prune_managed: bool = False,
|
||||
) -> None:
|
||||
target = infer_target(repo.root)
|
||||
client = GiteaClient(target, token)
|
||||
existing_pages = list_existing_wiki_pages(client, target.owner, target.repo)
|
||||
existing_page_names = {
|
||||
str(page.get("title")): str(page.get("sub_url") or page.get("title"))
|
||||
for page in existing_pages
|
||||
if page.get("title")
|
||||
}
|
||||
index_lines = [
|
||||
f"# {target.repo} Project Wiki Index",
|
||||
"",
|
||||
f"{MANAGED_MARKER}index -->",
|
||||
"",
|
||||
"This page is generated from repository and product-directory project files.",
|
||||
"",
|
||||
]
|
||||
|
||||
for source in sources:
|
||||
content = render_wiki_content(source)
|
||||
changed = put_wiki_page(
|
||||
client,
|
||||
target.owner,
|
||||
target.repo,
|
||||
source.page_title,
|
||||
content,
|
||||
existing_page_names,
|
||||
overwrite_unmanaged=overwrite_unmanaged,
|
||||
)
|
||||
print(f"{'updated' if changed else 'unchanged'} {target.owner}/{target.repo} wiki:{source.page_title}")
|
||||
index_lines.append(f"- [{source.page_title}]({source.page_title}) - `{source.path}`")
|
||||
|
||||
if write_index:
|
||||
put_wiki_page(
|
||||
client,
|
||||
target.owner,
|
||||
target.repo,
|
||||
"Codex-Project-Index",
|
||||
"\n".join(index_lines) + "\n",
|
||||
existing_page_names,
|
||||
overwrite_unmanaged=True,
|
||||
)
|
||||
else:
|
||||
print(f"skipped {target.owner}/{target.repo} wiki:Codex-Project-Index during page-limited sync")
|
||||
if prune_managed and write_index:
|
||||
prune_managed_wiki_pages_api(client, target.owner, target.repo, existing_page_names, sources)
|
||||
|
||||
|
||||
def sync_repo_wiki_git(
|
||||
repo: RepoInfo,
|
||||
sources: list[WikiSource],
|
||||
*,
|
||||
cache_dir: pathlib.Path,
|
||||
overwrite_unmanaged: bool,
|
||||
commit_message: str,
|
||||
write_index: bool = True,
|
||||
prune_managed: bool = False,
|
||||
) -> None:
|
||||
target = infer_target(repo.root)
|
||||
wiki_root = prepare_wiki_checkout(repo, cache_dir=cache_dir)
|
||||
index_lines = [
|
||||
f"# {target.repo} Project Wiki Index",
|
||||
"",
|
||||
f"{MANAGED_MARKER}index -->",
|
||||
"",
|
||||
"This page is generated from repository and product-directory project files.",
|
||||
"",
|
||||
]
|
||||
|
||||
for source in sources:
|
||||
content = render_wiki_content(source)
|
||||
status = write_wiki_page_file(
|
||||
wiki_root,
|
||||
source.page_title,
|
||||
content,
|
||||
overwrite_unmanaged=overwrite_unmanaged,
|
||||
)
|
||||
print(f"{status} {target.owner}/{target.repo} wiki:{source.page_title}")
|
||||
index_lines.append(f"- [{source.page_title}]({source.page_title}) - `{source.path}`")
|
||||
|
||||
if write_index:
|
||||
write_wiki_page_file(
|
||||
wiki_root,
|
||||
"Codex-Project-Index",
|
||||
"\n".join(index_lines) + "\n",
|
||||
overwrite_unmanaged=True,
|
||||
)
|
||||
else:
|
||||
print(f"skipped {target.owner}/{target.repo} wiki:Codex-Project-Index during page-limited sync")
|
||||
if prune_managed and write_index:
|
||||
prune_managed_wiki_pages_git(wiki_root, sources)
|
||||
if not git_has_changes(wiki_root):
|
||||
print(f"unchanged {target.owner}/{target.repo} wiki repository")
|
||||
return
|
||||
run_git(wiki_root, "add", "-A")
|
||||
if not git_has_staged_changes(wiki_root):
|
||||
print(f"unchanged {target.owner}/{target.repo} wiki repository")
|
||||
return
|
||||
ensure_git_identity(wiki_root)
|
||||
run_git(wiki_root, "commit", "-m", commit_message)
|
||||
run_git(wiki_root, "push")
|
||||
print(f"pushed {target.owner}/{target.repo} wiki repository")
|
||||
|
||||
|
||||
def prepare_wiki_checkout(repo: RepoInfo, *, cache_dir: pathlib.Path) -> pathlib.Path:
|
||||
remote = wiki_remote_for_repo(repo.root)
|
||||
cache_dir.mkdir(parents=True, exist_ok=True)
|
||||
checkout = cache_dir / f"{repo.owner}-{repo.repo}.wiki"
|
||||
if (checkout / ".git").exists():
|
||||
run_git(checkout, "remote", "set-url", "origin", remote)
|
||||
run_git(checkout, "fetch", "origin")
|
||||
branch = current_branch(checkout)
|
||||
if branch:
|
||||
run_git(checkout, "reset", "--hard", f"origin/{branch}")
|
||||
else:
|
||||
run_git(checkout, "reset", "--hard")
|
||||
run_git(checkout, "clean", "-fd")
|
||||
return checkout
|
||||
if checkout.exists():
|
||||
shutil.rmtree(checkout)
|
||||
result = subprocess.run(
|
||||
["git", "clone", remote, str(checkout)],
|
||||
check=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode == 0:
|
||||
return checkout
|
||||
checkout.mkdir(parents=True, exist_ok=True)
|
||||
run_git(checkout, "init")
|
||||
run_git(checkout, "remote", "add", "origin", remote)
|
||||
run_git(checkout, "checkout", "-b", "master")
|
||||
return checkout
|
||||
|
||||
|
||||
def wiki_remote_for_repo(repo_root_path: pathlib.Path) -> str:
|
||||
result = subprocess.run(
|
||||
["git", "-C", str(repo_root_path), "remote", "get-url", "origin"],
|
||||
check=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode != 0 or not result.stdout.strip():
|
||||
raise GiteaError(f"Could not read origin remote for {repo_root_path}")
|
||||
remote = result.stdout.strip()
|
||||
if remote.endswith(".git"):
|
||||
return f"{remote[:-4]}.wiki.git"
|
||||
return f"{remote}.wiki.git"
|
||||
|
||||
|
||||
def write_wiki_page_file(wiki_root: pathlib.Path, title: str, content: str, *, overwrite_unmanaged: bool) -> str:
|
||||
path = wiki_root / f"{wiki_file_stem(title)}.md"
|
||||
if path.exists():
|
||||
current = read_text(path)
|
||||
if current == content:
|
||||
return "unchanged"
|
||||
if MANAGED_MARKER not in current and not overwrite_unmanaged:
|
||||
return "skipped unmanaged"
|
||||
path.write_text(content, encoding="utf-8")
|
||||
return "updated"
|
||||
|
||||
|
||||
def prune_managed_wiki_pages_git(wiki_root: pathlib.Path, sources: list[WikiSource]) -> None:
|
||||
desired = {f"{wiki_file_stem(source.page_title)}.md" for source in sources}
|
||||
desired.add(f"{wiki_file_stem('Codex-Project-Index')}.md")
|
||||
for path in sorted(wiki_root.glob("*.md")):
|
||||
if path.name in desired:
|
||||
continue
|
||||
current = read_text(path)
|
||||
if MANAGED_MARKER not in current:
|
||||
continue
|
||||
path.unlink()
|
||||
print(f"deleted stale managed wiki page {path.name}")
|
||||
|
||||
|
||||
def prune_managed_wiki_pages_api(
|
||||
client: GiteaClient,
|
||||
owner: str,
|
||||
repo: str,
|
||||
existing_page_names: dict[str, str],
|
||||
sources: list[WikiSource],
|
||||
) -> None:
|
||||
desired = {source.page_title for source in sources}
|
||||
desired.add("Codex-Project-Index")
|
||||
for title, page_name in sorted(existing_page_names.items()):
|
||||
if title in desired:
|
||||
continue
|
||||
current = client.request_json("GET", repo_path(owner, repo, f"/wiki/page/{quote_wiki_page_name(page_name)}"))
|
||||
if MANAGED_MARKER not in decode_wiki_content(current):
|
||||
continue
|
||||
client.request_json("DELETE", repo_path(owner, repo, f"/wiki/page/{quote_wiki_page_name(page_name)}"))
|
||||
print(f"deleted stale managed wiki page {owner}/{repo}:{title}")
|
||||
|
||||
|
||||
def wiki_file_stem(title: str) -> str:
|
||||
return re.sub(r"[/\\]+", "-", title).strip() or "Home"
|
||||
|
||||
|
||||
def git_has_changes(root: pathlib.Path) -> bool:
|
||||
result = subprocess.run(
|
||||
["git", "-C", str(root), "status", "--porcelain"],
|
||||
check=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
raise GiteaError(f"git status failed in {root}: {result.stderr.strip()}")
|
||||
return bool(result.stdout.strip())
|
||||
|
||||
|
||||
def git_has_staged_changes(root: pathlib.Path) -> bool:
|
||||
result = subprocess.run(
|
||||
["git", "-C", str(root), "diff", "--cached", "--quiet"],
|
||||
check=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode == 0:
|
||||
return False
|
||||
if result.returncode == 1:
|
||||
return True
|
||||
raise GiteaError(f"git diff --cached failed in {root}: {result.stderr.strip()}")
|
||||
|
||||
|
||||
def current_branch(root: pathlib.Path) -> str:
|
||||
result = subprocess.run(
|
||||
["git", "-C", str(root), "rev-parse", "--abbrev-ref", "HEAD"],
|
||||
check=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
branch = result.stdout.strip()
|
||||
if result.returncode != 0 or branch == "HEAD":
|
||||
return ""
|
||||
return branch
|
||||
|
||||
|
||||
def ensure_git_identity(root: pathlib.Path) -> None:
|
||||
if not git_config_value(root, "user.email"):
|
||||
run_git(root, "config", "user.email", "codex@govoplan.local")
|
||||
if not git_config_value(root, "user.name"):
|
||||
run_git(root, "config", "user.name", "Codex")
|
||||
|
||||
|
||||
def git_config_value(root: pathlib.Path, key: str) -> str:
|
||||
result = subprocess.run(
|
||||
["git", "-C", str(root), "config", "--get", key],
|
||||
check=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
return result.stdout.strip() if result.returncode == 0 else ""
|
||||
|
||||
|
||||
def run_git(root: pathlib.Path, *args: str) -> None:
|
||||
result = subprocess.run(
|
||||
["git", "-C", str(root), *args],
|
||||
check=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
command = "git -C " + str(root) + " " + " ".join(args)
|
||||
raise GiteaError(f"{command} failed: {result.stderr.strip() or result.stdout.strip()}")
|
||||
|
||||
|
||||
def list_existing_wiki_pages(client: GiteaClient, owner: str, repo: str) -> list[dict[str, Any]]:
|
||||
try:
|
||||
return client.paginate(repo_path(owner, repo, "/wiki/pages"), limit=50)
|
||||
except GiteaError as exc:
|
||||
if "HTTP 404" in str(exc) and "/wiki/pages" in str(exc):
|
||||
return []
|
||||
raise
|
||||
|
||||
|
||||
def render_wiki_content(source: WikiSource) -> str:
|
||||
raw = read_text(source.path)
|
||||
fingerprint = hashlib.sha256(raw.encode("utf-8")).hexdigest()[:24]
|
||||
header = [
|
||||
f"{MANAGED_MARKER}{fingerprint} -->",
|
||||
"",
|
||||
f"> Mirrored from `{source.path}`.",
|
||||
f"> Origin: `{source.origin}`.",
|
||||
"> Active tasks and changing state belong in Gitea issues; this wiki page is durable project context.",
|
||||
"",
|
||||
"---",
|
||||
"",
|
||||
]
|
||||
if source.path.suffix.lower() == ".csv":
|
||||
return "\n".join(header + ["```csv", raw.rstrip(), "```", ""])
|
||||
return "\n".join(header) + raw.rstrip() + "\n"
|
||||
|
||||
|
||||
def put_wiki_page(
|
||||
client: GiteaClient,
|
||||
owner: str,
|
||||
repo: str,
|
||||
title: str,
|
||||
content: str,
|
||||
existing_page_names: dict[str, str],
|
||||
*,
|
||||
overwrite_unmanaged: bool,
|
||||
) -> bool:
|
||||
body = {
|
||||
"title": title,
|
||||
"content_base64": base64.b64encode(content.encode("utf-8")).decode("ascii"),
|
||||
"message": f"Sync {title} from project files",
|
||||
}
|
||||
if title in existing_page_names:
|
||||
page_name = existing_page_names[title]
|
||||
current = client.request_json("GET", repo_path(owner, repo, f"/wiki/page/{quote_wiki_page_name(page_name)}"))
|
||||
current_content = decode_wiki_content(current)
|
||||
if current_content == content:
|
||||
return False
|
||||
if MANAGED_MARKER not in current_content and not overwrite_unmanaged:
|
||||
print(f"skip unmanaged existing wiki page {owner}/{repo}:{title}")
|
||||
return False
|
||||
client.request_json("PATCH", repo_path(owner, repo, f"/wiki/page/{quote_wiki_page_name(page_name)}"), body=body)
|
||||
return True
|
||||
client.request_json("POST", repo_path(owner, repo, "/wiki/new"), body=body)
|
||||
existing_page_names[title] = title
|
||||
return True
|
||||
|
||||
|
||||
def quote_wiki_page_name(value: str) -> str:
|
||||
return urllib.parse.quote(value, safe="+")
|
||||
|
||||
|
||||
def decode_wiki_content(page: dict[str, Any]) -> str:
|
||||
encoded = str(page.get("content_base64") or "")
|
||||
if not encoded:
|
||||
return ""
|
||||
return base64.b64decode(encoded).decode("utf-8")
|
||||
|
||||
|
||||
def read_text(path: pathlib.Path) -> str:
|
||||
try:
|
||||
return path.read_text(encoding="utf-8")
|
||||
except UnicodeDecodeError:
|
||||
return path.read_text(encoding="latin-1")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
377
tools/gitea/gitea-todo-import.py
Normal file
377
tools/gitea/gitea-todo-import.py
Normal file
@@ -0,0 +1,377 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Preview or import inline TODO-style markers as Gitea issues."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
from gitea_common import GiteaClient, GiteaError, infer_target, label_ids_by_name, load_dotenv, repo_path, repo_root, require_token
|
||||
|
||||
|
||||
MARKER_RE = re.compile(
|
||||
r"(?P<prefix>#|//|/\*|\*|--|<!--)?\s*"
|
||||
r"\b(?P<marker>TODO|FIXME|XXX|HACK)\b"
|
||||
r"\s*(?:\((?P<context>[^)]{1,120})\))?"
|
||||
r"\s*(?P<colon>:)?\s*(?P<text>.*)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
DEFAULT_EXCLUDES = (
|
||||
"!**/.git/**",
|
||||
"!**/.venv/**",
|
||||
"!**/node_modules/**",
|
||||
"!**/__pycache__/**",
|
||||
"!**/.module-test-build/**",
|
||||
"!**/dist/**",
|
||||
"!**/build/**",
|
||||
"!**/.cache/**",
|
||||
"!.gitea/**",
|
||||
"!docs/GITEA_ISSUES.md",
|
||||
"!tools/gitea/gitea-todo-import.py",
|
||||
"!tools/gitea/gitea-sync-labels.py",
|
||||
"!tools/gitea/gitea-codex-note.py",
|
||||
"!tools/gitea/gitea_common.py",
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--root", type=pathlib.Path, default=pathlib.Path.cwd(), help="repository root or child path")
|
||||
parser.add_argument("--remote", default="origin", help="git remote to use for target inference")
|
||||
parser.add_argument("--env-file", type=pathlib.Path, help="dotenv file to read before using GITEA_* values")
|
||||
parser.add_argument("--apply", action="store_true", help="create missing Gitea issues")
|
||||
parser.add_argument("--include-linked", action="store_true", help="include markers that already reference an issue")
|
||||
parser.add_argument(
|
||||
"--module-label",
|
||||
help="project/module label to apply; defaults to a known GovOPlaN mapping or module/core",
|
||||
)
|
||||
parser.add_argument("--no-area-labels", action="store_true", help="do not infer area/* labels from paths")
|
||||
parser.add_argument("--extra-label", action="append", default=[], help="additional label name to apply")
|
||||
parser.add_argument("--limit", type=int, default=0, help="maximum number of markers to process")
|
||||
parser.add_argument("--json", action="store_true", help="print issue previews as JSON")
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
root = repo_root(args.root)
|
||||
load_dotenv(args.env_file or root / ".env")
|
||||
target = infer_target(root, args.remote)
|
||||
token = require_token() if args.apply else os.environ.get("GITEA_TOKEN")
|
||||
|
||||
markers = scan_markers(root, include_linked=args.include_linked)
|
||||
if args.limit > 0:
|
||||
markers = markers[: args.limit]
|
||||
|
||||
module_label = args.module_label or infer_module_label(root)
|
||||
previews = [
|
||||
build_issue_preview(
|
||||
target.owner,
|
||||
target.repo,
|
||||
marker,
|
||||
module_label=module_label,
|
||||
infer_areas=not args.no_area_labels,
|
||||
extra_labels=args.extra_label,
|
||||
)
|
||||
for marker in markers
|
||||
]
|
||||
|
||||
if args.json:
|
||||
print(json.dumps([preview.as_dict() for preview in previews], indent=2, sort_keys=True))
|
||||
else:
|
||||
print(f"Target: {target.display}")
|
||||
print(f"Scanned root: {root}")
|
||||
print(f"Found {len(previews)} importable marker(s).")
|
||||
for preview in previews:
|
||||
print(f"- {preview.title}")
|
||||
print(f" {preview.location}")
|
||||
print(f" labels: {', '.join(preview.labels)}")
|
||||
|
||||
if not args.apply:
|
||||
if not previews:
|
||||
return 0
|
||||
print("Dry run only. Re-run with --apply and GITEA_TOKEN to create issues.")
|
||||
return 0
|
||||
|
||||
if not previews:
|
||||
print("No issues to create.")
|
||||
return 0
|
||||
|
||||
client = GiteaClient(target, token)
|
||||
label_ids = load_label_ids(client, target.owner, target.repo)
|
||||
missing_labels = sorted({label for preview in previews for label in preview.labels if label not in label_ids})
|
||||
if missing_labels:
|
||||
joined = ", ".join(missing_labels)
|
||||
raise GiteaError(f"missing labels: {joined}. Run tools/gitea/gitea-sync-labels.py --apply first.")
|
||||
|
||||
existing_fingerprints = load_existing_fingerprints(client, target.owner, target.repo)
|
||||
created = 0
|
||||
skipped = 0
|
||||
for preview in previews:
|
||||
if preview.fingerprint in existing_fingerprints:
|
||||
print(f"skip existing {preview.location}")
|
||||
skipped += 1
|
||||
continue
|
||||
issue = client.request_json(
|
||||
"POST",
|
||||
repo_path(target.owner, target.repo, "/issues"),
|
||||
body={
|
||||
"title": preview.title,
|
||||
"body": preview.body,
|
||||
"labels": [label_ids[label] for label in preview.labels],
|
||||
},
|
||||
)
|
||||
print(f"created #{issue.get('number') or issue.get('index')}: {preview.title}")
|
||||
created += 1
|
||||
|
||||
print(f"Created {created} issue(s), skipped {skipped} existing issue(s).")
|
||||
return 0
|
||||
except GiteaError as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
class Marker:
|
||||
def __init__(self, path: pathlib.Path, line: int, column: int, marker: str, context: str, text: str, raw: str) -> None:
|
||||
self.path = path
|
||||
self.line = line
|
||||
self.column = column
|
||||
self.marker = marker.upper()
|
||||
self.context = context
|
||||
self.text = text
|
||||
self.raw = raw.rstrip("\n")
|
||||
|
||||
@property
|
||||
def location(self) -> str:
|
||||
return f"{self.path}:{self.line}"
|
||||
|
||||
|
||||
class IssuePreview:
|
||||
def __init__(self, title: str, body: str, labels: list[str], fingerprint: str, location: str) -> None:
|
||||
self.title = title
|
||||
self.body = body
|
||||
self.labels = labels
|
||||
self.fingerprint = fingerprint
|
||||
self.location = location
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"title": self.title,
|
||||
"body": self.body,
|
||||
"labels": self.labels,
|
||||
"fingerprint": self.fingerprint,
|
||||
"location": self.location,
|
||||
}
|
||||
|
||||
|
||||
def scan_markers(root: pathlib.Path, *, include_linked: bool) -> list[Marker]:
|
||||
command = [
|
||||
"rg",
|
||||
"--json",
|
||||
"--hidden",
|
||||
"--line-number",
|
||||
"--column",
|
||||
"-e",
|
||||
r"\b(TODO|FIXME|XXX|HACK)\b",
|
||||
]
|
||||
for pattern in DEFAULT_EXCLUDES:
|
||||
command.extend(["--glob", pattern])
|
||||
command.append(str(root))
|
||||
|
||||
result = subprocess.run(
|
||||
command,
|
||||
check=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode not in {0, 1}:
|
||||
raise GiteaError(f"rg failed: {result.stderr.strip()}")
|
||||
|
||||
markers: list[Marker] = []
|
||||
for line in result.stdout.splitlines():
|
||||
event = json.loads(line)
|
||||
if event.get("type") != "match":
|
||||
continue
|
||||
data = event["data"]
|
||||
raw_line = data["lines"]["text"]
|
||||
match = MARKER_RE.search(raw_line)
|
||||
if not match:
|
||||
continue
|
||||
if not _looks_like_marker(match):
|
||||
continue
|
||||
context = (match.group("context") or "").strip()
|
||||
if not include_linked and _already_linked(context, raw_line):
|
||||
continue
|
||||
markers.append(
|
||||
Marker(
|
||||
path=pathlib.Path(data["path"]["text"]).resolve().relative_to(root),
|
||||
line=int(data["line_number"]),
|
||||
column=int(data.get("submatches", [{}])[0].get("start", 0)) + 1,
|
||||
marker=match.group("marker"),
|
||||
context=context,
|
||||
text=clean_marker_text(match.group("text") or raw_line),
|
||||
raw=raw_line,
|
||||
)
|
||||
)
|
||||
return markers
|
||||
|
||||
|
||||
def build_issue_preview(
|
||||
owner: str,
|
||||
repo: str,
|
||||
marker: Marker,
|
||||
*,
|
||||
module_label: str,
|
||||
infer_areas: bool,
|
||||
extra_labels: list[str],
|
||||
) -> IssuePreview:
|
||||
fingerprint = marker_fingerprint(owner, repo, marker)
|
||||
summary = marker.text or f"review {marker.location}"
|
||||
title = truncate_title(f"{marker.marker}: {summary} ({marker.location})")
|
||||
labels = sorted(set(default_labels(marker, module_label=module_label, infer_areas=infer_areas) + list(extra_labels)))
|
||||
body = "\n".join(
|
||||
[
|
||||
f"<!-- codex-todo-fingerprint:{fingerprint} -->",
|
||||
"",
|
||||
"Imported from an inline marker.",
|
||||
"",
|
||||
f"- Repository: `{owner}/{repo}`",
|
||||
f"- Location: `{marker.location}`",
|
||||
f"- Marker: `{marker.marker}`",
|
||||
f"- Context: `{marker.context or 'none'}`",
|
||||
"",
|
||||
"Current line:",
|
||||
"",
|
||||
"```text",
|
||||
marker.raw,
|
||||
"```",
|
||||
"",
|
||||
"Migration rule:",
|
||||
"",
|
||||
"- Keep this Gitea issue as the canonical backlog item.",
|
||||
"- When touching this code, remove the inline marker or replace it with a short issue reference.",
|
||||
]
|
||||
)
|
||||
return IssuePreview(title=title, body=body, labels=labels, fingerprint=fingerprint, location=marker.location)
|
||||
|
||||
|
||||
def clean_marker_text(text: str) -> str:
|
||||
cleaned = text.strip()
|
||||
cleaned = re.sub(r"\s*(?:#|//|/\*|\*|--)\s*$", "", cleaned).strip()
|
||||
cleaned = re.sub(r"\s*\*/\s*$", "", cleaned).strip()
|
||||
return cleaned
|
||||
|
||||
|
||||
def default_labels(marker: Marker, *, module_label: str, infer_areas: bool) -> list[str]:
|
||||
labels = ["source/todo-scan", "status/triage", module_label, marker_type_label(marker.marker)]
|
||||
area = infer_area_label(marker.path) if infer_areas else ""
|
||||
if area:
|
||||
labels.append(area)
|
||||
return labels
|
||||
|
||||
|
||||
def marker_type_label(marker: str) -> str:
|
||||
if marker == "FIXME":
|
||||
return "type/bug"
|
||||
if marker in {"HACK", "XXX"}:
|
||||
return "type/debt"
|
||||
return "type/task"
|
||||
|
||||
|
||||
def infer_area_label(path: pathlib.Path) -> str:
|
||||
text = path.as_posix()
|
||||
if text.startswith("webui/"):
|
||||
return "area/webui"
|
||||
if text.startswith("alembic/"):
|
||||
return "area/migrations"
|
||||
if text.startswith("docs/"):
|
||||
return "area/docs"
|
||||
if text.startswith("scripts/") or text.startswith("tools/"):
|
||||
return "area/devex"
|
||||
if "/rbac" in text or "permission" in text:
|
||||
return "area/rbac"
|
||||
if "/access" in text or "auth" in text or "session" in text:
|
||||
return "area/auth"
|
||||
if "tenant" in text:
|
||||
return "area/tenancy"
|
||||
if "governance" in text or "privacy" in text or "audit" in text or "retention" in text:
|
||||
return "area/governance"
|
||||
if "module" in text:
|
||||
return "area/module-system"
|
||||
if "migration" in text:
|
||||
return "area/migrations"
|
||||
if "router" in text or "api" in text:
|
||||
return "area/api"
|
||||
if "db" in text or "model" in text or "persistence" in text:
|
||||
return "area/db"
|
||||
return ""
|
||||
|
||||
|
||||
def infer_module_label(root: pathlib.Path) -> str:
|
||||
known = {
|
||||
"govoplan-core": "module/core",
|
||||
"govoplan-access": "module/access",
|
||||
"govoplan-admin": "module/admin",
|
||||
"govoplan-tenancy": "module/tenancy",
|
||||
"govoplan-policy": "module/policy",
|
||||
"govoplan-audit": "module/audit",
|
||||
"govoplan-mail": "module/mail",
|
||||
"govoplan-files": "module/files",
|
||||
"govoplan-campaign": "module/campaign",
|
||||
"addideas-govoplan-website": "area/marketing",
|
||||
}
|
||||
if root.name in known:
|
||||
return known[root.name]
|
||||
if root.name.startswith("govoplan-"):
|
||||
suffix = root.name.removeprefix("govoplan-")
|
||||
if suffix:
|
||||
return f"module/{suffix}"
|
||||
return "module/core"
|
||||
|
||||
|
||||
def marker_fingerprint(owner: str, repo: str, marker: Marker) -> str:
|
||||
stable = "\n".join([owner, repo, marker.path.as_posix(), marker.marker, marker.context, marker.text])
|
||||
return hashlib.sha256(stable.encode("utf-8")).hexdigest()[:24]
|
||||
|
||||
|
||||
def truncate_title(title: str, limit: int = 180) -> str:
|
||||
if len(title) <= limit:
|
||||
return title
|
||||
return f"{title[: limit - 1].rstrip()}..."
|
||||
|
||||
|
||||
def load_label_ids(client: GiteaClient, owner: str, repo: str) -> dict[str, int]:
|
||||
return label_ids_by_name(client, owner, repo)
|
||||
|
||||
|
||||
def load_existing_fingerprints(client: GiteaClient, owner: str, repo: str) -> set[str]:
|
||||
issues = client.paginate(
|
||||
repo_path(owner, repo, "/issues"),
|
||||
query={"state": "all", "labels": "source/todo-scan"},
|
||||
)
|
||||
fingerprints: set[str] = set()
|
||||
for issue in issues:
|
||||
body = str(issue.get("body") or "")
|
||||
for match in re.finditer(r"(?:codex|govoplan)-todo-fingerprint:([0-9a-f]{24})", body):
|
||||
fingerprints.add(match.group(1))
|
||||
return fingerprints
|
||||
|
||||
|
||||
def _already_linked(context: str, raw_line: str) -> bool:
|
||||
linked_text = f"{context} {raw_line}".lower()
|
||||
return "gitea#" in linked_text or "issue#" in linked_text or re.search(r"#\d+", linked_text) is not None
|
||||
|
||||
|
||||
def _looks_like_marker(match: re.Match[str]) -> bool:
|
||||
text = (match.group("text") or "").strip()
|
||||
return bool(match.group("context") or match.group("colon") or (match.group("prefix") and text))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
290
tools/gitea/gitea_common.py
Normal file
290
tools/gitea/gitea_common.py
Normal file
@@ -0,0 +1,290 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Shared helpers for Gitea maintenance scripts."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import subprocess
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from typing import Any
|
||||
|
||||
|
||||
class GiteaError(RuntimeError):
|
||||
"""Raised when a Gitea API request fails."""
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class RepoTarget:
|
||||
base_url: str
|
||||
owner: str
|
||||
repo: str
|
||||
|
||||
@property
|
||||
def display(self) -> str:
|
||||
return f"{self.base_url.rstrip('/')}/{self.owner}/{self.repo}"
|
||||
|
||||
@property
|
||||
def api_base(self) -> str:
|
||||
return f"{self.base_url.rstrip('/')}/api/v1"
|
||||
|
||||
|
||||
def repo_root(start: pathlib.Path) -> pathlib.Path:
|
||||
result = subprocess.run(
|
||||
["git", "-C", str(start), "rev-parse", "--show-toplevel"],
|
||||
check=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
return start.resolve()
|
||||
return pathlib.Path(result.stdout.strip()).resolve()
|
||||
|
||||
|
||||
def infer_target(root: pathlib.Path, remote_name: str = "origin") -> RepoTarget:
|
||||
env_url = os.environ.get("GITEA_URL")
|
||||
env_owner = os.environ.get("GITEA_OWNER")
|
||||
env_repo = os.environ.get("GITEA_REPO")
|
||||
|
||||
remote_url = _git_remote(root, remote_name)
|
||||
inferred_url, inferred_owner, inferred_repo = _parse_remote(remote_url)
|
||||
|
||||
base_url = (env_url or inferred_url).rstrip("/")
|
||||
owner = env_owner or inferred_owner
|
||||
repo = env_repo or inferred_repo
|
||||
|
||||
missing = [
|
||||
name
|
||||
for name, value in (
|
||||
("GITEA_URL", base_url),
|
||||
("GITEA_OWNER", owner),
|
||||
("GITEA_REPO", repo),
|
||||
)
|
||||
if not value
|
||||
]
|
||||
if missing:
|
||||
joined = ", ".join(missing)
|
||||
raise GiteaError(
|
||||
f"Could not infer Gitea target. Set {joined}, or configure git remote {remote_name!r}."
|
||||
)
|
||||
return RepoTarget(base_url=base_url, owner=owner, repo=_strip_git_suffix(repo))
|
||||
|
||||
|
||||
def quote_path(value: str) -> str:
|
||||
return urllib.parse.quote(value, safe="")
|
||||
|
||||
|
||||
class GiteaClient:
|
||||
def __init__(self, target: RepoTarget, token: str | None) -> None:
|
||||
self.target = target
|
||||
self.token = token
|
||||
|
||||
def request_json(
|
||||
self,
|
||||
method: str,
|
||||
path: str,
|
||||
*,
|
||||
body: dict[str, Any] | None = None,
|
||||
query: dict[str, Any] | None = None,
|
||||
) -> Any:
|
||||
url = f"{self.target.api_base}{path}"
|
||||
if query:
|
||||
url = f"{url}?{urllib.parse.urlencode(query, doseq=True)}"
|
||||
|
||||
data = None
|
||||
headers = {
|
||||
"Accept": "application/json",
|
||||
"User-Agent": "codex-gitea-maintenance",
|
||||
}
|
||||
if self.token:
|
||||
headers["Authorization"] = f"token {self.token}"
|
||||
if body is not None:
|
||||
data = json.dumps(body).encode("utf-8")
|
||||
headers["Content-Type"] = "application/json"
|
||||
|
||||
request = urllib.request.Request(url, data=data, headers=headers, method=method)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=30) as response:
|
||||
payload = response.read().decode("utf-8")
|
||||
if not payload:
|
||||
return None
|
||||
return json.loads(payload)
|
||||
except urllib.error.HTTPError as exc:
|
||||
message = exc.read().decode("utf-8", errors="replace")
|
||||
raise GiteaError(f"{method} {url} failed with HTTP {exc.code}: {message}") from exc
|
||||
except urllib.error.URLError as exc:
|
||||
raise GiteaError(f"{method} {url} failed: {exc.reason}") from exc
|
||||
|
||||
def paginate(
|
||||
self,
|
||||
path: str,
|
||||
*,
|
||||
query: dict[str, Any] | None = None,
|
||||
limit: int = 100,
|
||||
) -> list[dict[str, Any]]:
|
||||
items: list[dict[str, Any]] = []
|
||||
page = 1
|
||||
effective_limit = min(limit, 50)
|
||||
seen_pages: set[tuple[Any, ...]] = set()
|
||||
while True:
|
||||
page_query = dict(query or {})
|
||||
page_query.update({"page": page, "limit": effective_limit})
|
||||
payload = self.request_json("GET", path, query=page_query)
|
||||
if not isinstance(payload, list):
|
||||
raise GiteaError(f"Expected a list from {path}, got {type(payload).__name__}")
|
||||
if not payload:
|
||||
return items
|
||||
signature = tuple(
|
||||
item.get("id") or item.get("number") or item.get("index") or item.get("name")
|
||||
for item in payload
|
||||
if isinstance(item, dict)
|
||||
)
|
||||
if signature in seen_pages:
|
||||
return items
|
||||
seen_pages.add(signature)
|
||||
items.extend(payload)
|
||||
if len(payload) < effective_limit:
|
||||
return items
|
||||
page += 1
|
||||
|
||||
|
||||
def load_json(path: pathlib.Path) -> Any:
|
||||
with path.open("r", encoding="utf-8") as handle:
|
||||
return json.load(handle)
|
||||
|
||||
|
||||
def load_dotenv(path: pathlib.Path | None) -> list[str]:
|
||||
if path is None or not path.exists():
|
||||
return []
|
||||
|
||||
loaded: list[str] = []
|
||||
for line_number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1):
|
||||
stripped = line.strip()
|
||||
if not stripped or stripped.startswith("#"):
|
||||
continue
|
||||
if stripped.startswith("export "):
|
||||
stripped = stripped[len("export ") :].strip()
|
||||
if "=" not in stripped:
|
||||
raise GiteaError(f"{path}:{line_number}: expected KEY=value")
|
||||
key, value = stripped.split("=", 1)
|
||||
key = key.strip()
|
||||
value = _strip_env_value(value.strip())
|
||||
if not re.match(r"^[A-Za-z_][A-Za-z0-9_]*$", key):
|
||||
raise GiteaError(f"{path}:{line_number}: invalid environment key {key!r}")
|
||||
if key not in os.environ:
|
||||
os.environ[key] = value
|
||||
loaded.append(key)
|
||||
return loaded
|
||||
|
||||
|
||||
def require_token() -> str:
|
||||
token = os.environ.get("GITEA_TOKEN")
|
||||
if not token:
|
||||
raise GiteaError(
|
||||
"GITEA_TOKEN is required when using --apply. "
|
||||
"Set it in the environment, the target .env, or --env-file."
|
||||
)
|
||||
return token
|
||||
|
||||
|
||||
def repo_path(owner: str, repo: str, suffix: str) -> str:
|
||||
return f"/repos/{quote_path(owner)}/{quote_path(repo)}{suffix}"
|
||||
|
||||
|
||||
def org_path(owner: str, suffix: str) -> str:
|
||||
return f"/orgs/{quote_path(owner)}{suffix}"
|
||||
|
||||
|
||||
def labels_by_name(client: GiteaClient, owner: str, repo: str | None = None) -> dict[str, dict[str, Any]]:
|
||||
"""Return org labels plus optional repository labels, keyed by label name.
|
||||
|
||||
Gitea stores organization labels separately from repository labels. Issue
|
||||
APIs can use label ids from both scopes on supported instances, so issue
|
||||
creation helpers should resolve both. Repository labels intentionally win
|
||||
when a repo carries a local label with the same name.
|
||||
"""
|
||||
|
||||
labels: dict[str, dict[str, Any]] = {}
|
||||
try:
|
||||
for label in client.paginate(org_path(owner, "/labels")):
|
||||
name = str(label.get("name") or "")
|
||||
if name:
|
||||
labels[name] = label
|
||||
except GiteaError:
|
||||
# User-owned repositories or older instances may not expose org labels.
|
||||
pass
|
||||
|
||||
if repo:
|
||||
for label in client.paginate(repo_path(owner, repo, "/labels")):
|
||||
name = str(label.get("name") or "")
|
||||
if name:
|
||||
labels[name] = label
|
||||
return labels
|
||||
|
||||
|
||||
def label_ids_by_name(client: GiteaClient, owner: str, repo: str | None = None) -> dict[str, int]:
|
||||
ids: dict[str, int] = {}
|
||||
for name, label in labels_by_name(client, owner, repo).items():
|
||||
label_id = label.get("id") or label.get("index")
|
||||
if label_id is None:
|
||||
continue
|
||||
ids[name] = int(label_id)
|
||||
return ids
|
||||
|
||||
|
||||
def _git_remote(root: pathlib.Path, remote_name: str) -> str:
|
||||
result = subprocess.run(
|
||||
["git", "-C", str(root), "remote", "get-url", remote_name],
|
||||
check=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
return ""
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def _parse_remote(remote_url: str) -> tuple[str, str, str]:
|
||||
if not remote_url:
|
||||
return "", "", ""
|
||||
|
||||
parsed = urllib.parse.urlparse(remote_url)
|
||||
if parsed.scheme in {"http", "https", "ssh"} and parsed.netloc:
|
||||
path_parts = [part for part in parsed.path.strip("/").split("/") if part]
|
||||
owner, repo = _owner_repo_from_parts(path_parts)
|
||||
if parsed.scheme in {"http", "https"}:
|
||||
prefix = "/".join(path_parts[:-2])
|
||||
base_path = f"/{prefix}" if prefix else ""
|
||||
return f"{parsed.scheme}://{parsed.netloc}{base_path}", owner, repo
|
||||
return f"https://{parsed.hostname or parsed.netloc}", owner, repo
|
||||
|
||||
scp_like = re.match(r"^(?:[^@]+@)?(?P<host>[^:]+):(?P<path>.+)$", remote_url)
|
||||
if scp_like:
|
||||
path_parts = [part for part in scp_like.group("path").strip("/").split("/") if part]
|
||||
owner, repo = _owner_repo_from_parts(path_parts)
|
||||
return f"https://{scp_like.group('host')}", owner, repo
|
||||
|
||||
return "", "", ""
|
||||
|
||||
|
||||
def _owner_repo_from_parts(path_parts: list[str]) -> tuple[str, str]:
|
||||
if len(path_parts) < 2:
|
||||
return "", ""
|
||||
return path_parts[-2], _strip_git_suffix(path_parts[-1])
|
||||
|
||||
|
||||
def _strip_git_suffix(value: str) -> str:
|
||||
return value[:-4] if value.endswith(".git") else value
|
||||
|
||||
|
||||
def _strip_env_value(value: str) -> str:
|
||||
if len(value) >= 2 and value[0] == value[-1] and value[0] in {"'", '"'}:
|
||||
return value[1:-1]
|
||||
return value
|
||||
Reference in New Issue
Block a user