Files
govoplan-core/scripts/gitea-import-all-backlogs.py
2026-07-07 16:00:38 +02:00

601 lines
22 KiB
Python

#!/usr/bin/env python3
"""Import backlog-like files from git.add-ideas.de repositories into Gitea."""
from __future__ import annotations
import argparse
import csv
import dataclasses
import hashlib
import os
import pathlib
import re
import subprocess
import sys
from typing import Any, Iterable
from gitea_common import GiteaClient, GiteaError, infer_target, load_dotenv, repo_path, require_token
GIT_ROOT = pathlib.Path("/mnt/DATA/git")
PRODUCTS_ROOT = pathlib.Path("/mnt/DATA/Nextcloud/ADD ideas UG/Products")
EXCLUDED_FILE_PATTERNS = (
"/.git/",
"/.venv/",
"/venv/",
"/site-packages/",
"/node_modules/",
"/__pycache__/",
"/dist/",
"/build/",
"/.cache/",
"/.module-test-build/",
)
BACKLOG_NAME_RE = re.compile(r"(backlog|todo|roadmap|plan|issue|milestone|action)", re.IGNORECASE)
ACTION_RE = re.compile(
r"^(add|avoid|build|convert|create|define|delay|design|document|enable|ensure|expand|extract|fetch|fix|finalize|generate|"
r"implement|import|improve|integrate|keep|make|move|optimize|persist|precompute|promote|provide|"
r"publish|query|replace|review|run|seed|serve|split|start|store|support|test|track|use|wire)\b",
re.IGNORECASE,
)
SKIP_SECTION_RE = re.compile(
r"(completed|implemented|current state|recent fixes|recently completed|working assumptions|stable decisions|"
r"available now|done|archive)",
re.IGNORECASE,
)
ACTIVE_SECTION_RE = re.compile(
r"(p0|p1|p2|p3|mvp|milestone|phase|todo|backlog|roadmap|tasks|testing|frontend|backend|routing|"
r"data outputs|future|critical path|next sprint|optimization|hardening|enhancements?)",
re.IGNORECASE,
)
GENERIC_LABELS = [
("type/task", "1d76db", "Implementation, maintenance, migration, or operational work."),
("type/feature", "0e8a16", "New user-visible behavior or product capability."),
("type/debt", "fbca04", "Cleanup, refactoring, risk reduction, or deferred engineering work."),
("type/docs", "5319e7", "Documentation, process, or developer workflow work."),
("status/triage", "d4c5f9", "Needs review, ownership, priority, or acceptance criteria."),
("source/backlog-import", "bfdadc", "Imported from markdown, text, CSV, roadmap, backlog, plan, or TODO files."),
("codex/ready", "0e8a16", "Suitable for Codex to pick up with the existing issue context."),
("priority/p0", "b60205", "Immediate stop-the-line priority."),
("priority/p1", "d93f0b", "High priority for the next focused work window."),
("priority/p2", "fbca04", "Normal planned priority."),
("priority/p3", "c2e0c6", "Low priority or opportunistic cleanup."),
]
@dataclasses.dataclass(frozen=True)
class RepoInfo:
root: pathlib.Path
remote: str
owner: str
repo: str
@property
def key(self) -> str:
return self.root.name
@dataclasses.dataclass(frozen=True)
class SourceFile:
repo_key: str
path: pathlib.Path
source_kind: str
@dataclasses.dataclass(frozen=True)
class Candidate:
repo_key: str
title: str
body: str
labels: tuple[str, ...]
priority: str
milestone: str
fingerprint: str
source: str
line: int
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--env-file", type=pathlib.Path, default=pathlib.Path("/home/zemion/.config/gitea/gitea.env"))
parser.add_argument("--git-root", type=pathlib.Path, default=GIT_ROOT)
parser.add_argument("--products-root", type=pathlib.Path, default=PRODUCTS_ROOT)
parser.add_argument("--apply", action="store_true")
args = parser.parse_args()
try:
load_dotenv(args.env_file)
repos = discover_hosted_repos(args.git_root)
sources = discover_sources(repos, args.products_root)
candidates = list(build_candidates(sources))
candidates = dedupe_candidates(candidates)
candidates.sort(key=lambda item: (item.repo_key, item.source, item.line, item.title))
print(f"Hosted repositories: {len(repos)}")
print(f"Backlog-like source files: {len(sources)}")
print(f"Issue candidates: {len(candidates)}")
for repo_key, count in grouped_counts(candidates).items():
print(f" {repo_key}: {count}")
token = require_token() if args.apply else os.environ.get("GITEA_TOKEN")
if not token:
print_preview(candidates)
print("Dry run without GITEA_TOKEN: duplicate comparison skipped.")
return 0
missing_by_repo: dict[str, list[Candidate]] = {}
skipped = 0
states: dict[str, RepoState] = {}
for repo_key in sorted({candidate.repo_key for candidate in candidates}):
states[repo_key] = load_repo_state(repos[repo_key], token, apply=args.apply)
for candidate in candidates:
state = states[candidate.repo_key]
if candidate.fingerprint in state.fingerprints or normalize_title(candidate.title) in state.titles:
skipped += 1
continue
missing_by_repo.setdefault(candidate.repo_key, []).append(candidate)
missing_total = sum(len(items) for items in missing_by_repo.values())
print(f"Skipped existing: {skipped}")
print(f"Missing candidates: {missing_total}")
print_preview([candidate for items in missing_by_repo.values() for candidate in items])
if not args.apply:
print("Dry run only. Re-run with --apply to create missing issues.")
return 0
for repo_key, repo_candidates in missing_by_repo.items():
state = states[repo_key]
for candidate in repo_candidates:
label_ids = [state.label_ids[label] for label in candidate.labels]
if candidate.priority not in candidate.labels:
label_ids.append(state.label_ids[candidate.priority])
milestone_id = ensure_milestone(state, candidate.milestone)
created = state.client.request_json(
"POST",
repo_path(state.target.owner, state.target.repo, "/issues"),
body={
"title": candidate.title,
"body": candidate.body,
"labels": label_ids,
"milestone": milestone_id,
},
)
number = created.get("number") or created.get("index")
print(f"created {state.target.owner}/{state.target.repo}#{number}: {candidate.title}")
state.fingerprints.add(candidate.fingerprint)
state.titles.add(normalize_title(candidate.title))
return 0
except GiteaError as exc:
print(f"error: {exc}", file=sys.stderr)
return 1
@dataclasses.dataclass
class RepoState:
target: Any
client: GiteaClient
label_ids: dict[str, int]
milestone_ids: dict[str, int]
fingerprints: set[str]
titles: set[str]
def discover_hosted_repos(git_root: pathlib.Path) -> dict[str, RepoInfo]:
repos: dict[str, RepoInfo] = {}
for gitdir in sorted(git_root.glob("*/.git")):
root = gitdir.parent
result = subprocess.run(
["git", "-C", str(root), "remote", "get-url", "origin"],
check=False,
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL,
text=True,
)
remote = result.stdout.strip()
if "git.add-ideas.de" not in remote:
continue
target = infer_target(root)
repos[root.name] = RepoInfo(root=root, remote=remote, owner=target.owner, repo=target.repo)
return repos
def discover_sources(repos: dict[str, RepoInfo], products_root: pathlib.Path) -> list[SourceFile]:
sources: list[SourceFile] = []
for repo in repos.values():
for path in repo.root.rglob("*"):
if not path.is_file() or not BACKLOG_NAME_RE.search(path.name):
continue
path_text = path.as_posix()
if any(pattern in path_text for pattern in EXCLUDED_FILE_PATTERNS):
continue
if is_excluded_repo_file(path):
continue
if is_text_backlog(path):
sources.append(SourceFile(repo_key=repo.key, path=path, source_kind="repo"))
product_map = {
"co2api": "emission-api-lib",
"govoplan": "govoplan-core",
"meubility": "meubility-workbench",
"mousehold": "mousehold",
"prvnncr": "prvnncr-server",
}
if products_root.exists():
for product_dir in sorted(path for path in products_root.iterdir() if path.is_dir()):
repo_key = product_map.get(product_dir.name)
if not repo_key or repo_key not in repos:
continue
for path in product_dir.rglob("*"):
if path.is_file() and BACKLOG_NAME_RE.search(path.name) and is_text_backlog(path):
sources.append(SourceFile(repo_key=repo_key, path=path, source_kind="product"))
unique: dict[tuple[str, pathlib.Path], SourceFile] = {}
for source in sources:
unique[(source.repo_key, source.path.resolve())] = source
return sorted(unique.values(), key=lambda item: (item.repo_key, str(item.path)))
def is_excluded_repo_file(path: pathlib.Path) -> bool:
text = path.as_posix()
name = path.name.lower()
if "/scripts/gitea-" in text or text.endswith("/scripts/gitea_common.py"):
return True
if "testing_plan" in name or "test_plan" in name:
return True
if text.endswith("/docs/GITEA_ISSUES.md"):
return True
if text.endswith("/docs/GOVOPLAN_MODULE_ROADMAP.md"):
return True
if text.endswith("/docs/ACCESS_EXTRACTION_PLAN.md"):
return True
if text.endswith("/govoplan-web/TODO.md"):
return True
return False
def is_text_backlog(path: pathlib.Path) -> bool:
if path.suffix.lower() not in {".md", ".txt", ".csv"}:
return False
try:
chunk = path.read_bytes()[:4096]
except OSError:
return False
if b"\x00" in chunk:
return False
return True
def build_candidates(sources: list[SourceFile]) -> Iterable[Candidate]:
for source in sources:
suffix = source.path.suffix.lower()
if suffix == ".csv":
yield from parse_csv_source(source)
else:
yield from parse_text_source(source)
def parse_csv_source(source: SourceFile) -> Iterable[Candidate]:
with source.path.open("r", encoding="utf-8-sig", newline="") as handle:
reader = csv.DictReader(handle)
for index, row in enumerate(reader, start=2):
story = (row.get("story") or row.get("title") or row.get("name") or "").strip()
if not story:
continue
identifier = (row.get("id") or "").strip()
priority = normalize_priority(row.get("priority") or "")
milestone = (row.get("epic") or "Backlog").strip()
title = format_title("[Feature]", f"{identifier}: {story}" if identifier else story)
fingerprint = make_fingerprint(source.path, index, source.repo_key, story)
body = "\n".join(
[
f"<!-- codex-generic-backlog-fingerprint:{fingerprint} -->",
"",
"Imported from a CSV backlog file.",
"",
f"- Source: `{source.path}`",
f"- Line: `{index}`",
f"- Source kind: `{source.source_kind}`",
f"- Milestone: `{milestone}`",
"",
"CSV row:",
"",
"```text",
", ".join(f"{key}={value}" for key, value in row.items()),
"```",
]
)
yield Candidate(
repo_key=source.repo_key,
title=title,
body=body,
labels=("type/feature", "status/triage", "source/backlog-import", "codex/ready"),
priority=priority,
milestone=milestone,
fingerprint=fingerprint,
source=str(source.path),
line=index,
)
def parse_text_source(source: SourceFile) -> Iterable[Candidate]:
try:
lines = source.path.read_text(encoding="utf-8").splitlines()
except UnicodeDecodeError:
lines = source.path.read_text(encoding="latin-1").splitlines()
headings: list[tuple[int, str]] = []
in_tasks = False
for index, line in enumerate(lines):
line_number = index + 1
heading = parse_heading(line)
if heading:
level, title = heading
headings = [(h_level, h_title) for h_level, h_title in headings if h_level < level]
headings.append((level, title))
in_tasks = False
continue
if re.match(r"^\s*(tasks|action items|immediate todos?|recommended next sprint)\s*:?\s*$", line, re.IGNORECASE):
in_tasks = True
continue
item = parse_open_item(line)
if item is None and is_action_context(source.path, headings, in_tasks):
item = parse_plain_action_item(line)
if item is None:
continue
if not item or should_skip_item(item, headings):
continue
section = section_title(headings)
priority = priority_from_context(section, item)
milestone = milestone_from_context(source.path, section)
prefix = "[Feature]" if ACTION_RE.match(item) else "[Task]"
title = format_title(prefix, item)
fingerprint = make_fingerprint(source.path, line_number, source.repo_key, item)
body = "\n".join(
[
f"<!-- codex-generic-backlog-fingerprint:{fingerprint} -->",
"",
"Imported from a backlog-like text file.",
"",
f"- Source: `{source.path}`",
f"- Line: `{line_number}`",
f"- Source kind: `{source.source_kind}`",
f"- Section: `{section or 'none'}`",
"",
"Imported item:",
"",
"```text",
item,
"```",
]
)
yield Candidate(
repo_key=source.repo_key,
title=title,
body=body,
labels=("type/feature" if prefix == "[Feature]" else "type/task", "status/triage", "source/backlog-import", "codex/ready"),
priority=priority,
milestone=milestone,
fingerprint=fingerprint,
source=str(source.path),
line=line_number,
)
def parse_heading(line: str) -> tuple[int, str] | None:
markdown = re.match(r"^(#{1,6})\s+(.+?)\s*$", line)
if markdown:
return len(markdown.group(1)), clean_text(markdown.group(2))
underlined = re.match(r"^(.+?)\s*$", line)
if underlined and line.strip() and len(line.strip()) < 120:
return None
return None
def parse_open_item(line: str) -> str | None:
patterns = [
r"^(?P<indent>\s*)[-*]\s+\[\s\]\s+(?P<text>.+?)\s*$",
r"^(?P<indent>\s*)[-*]\s+☐\s+(?P<text>.+?)\s*$",
r"^(?P<indent>\s*)\d+[.)]\s+☐\s+(?P<text>.+?)\s*$",
r"^(?P<indent>\s*)[-*]\s+\[(?!x\]|X\])(?:[^\]]+)\]\s+(?P<text>.+?)\s*$",
]
for pattern in patterns:
match = re.match(pattern, line)
if match and leading_width(match.group("indent")) == 0:
return clean_text(match.group("text"))
return None
def parse_plain_action_item(line: str) -> str | None:
match = re.match(r"^(?P<indent>\s*)[-*]\s+(?P<text>.+?)\s*$", line)
if match and leading_width(match.group("indent")) == 0:
text = clean_text(match.group("text"))
if "" in text or text.startswith(("", "[x]", "[X]")):
return None
if ACTION_RE.match(text) or is_short_noun_task(text):
return text
return None
def is_action_context(path: pathlib.Path, headings: list[tuple[int, str]], in_tasks: bool) -> bool:
if in_tasks:
return True
context = section_title(headings)
if not context:
return False
if SKIP_SECTION_RE.search(context):
return False
if ACTIVE_SECTION_RE.search(context):
return True
return path.name.lower() in {"todo.txt", "todo.md"}
def should_skip_item(item: str, headings: list[tuple[int, str]]) -> bool:
context = section_title(headings)
if "" in item or item.startswith(("", "[x]", "[X]")):
return True
if item.endswith((",", ";")):
return True
if SKIP_SECTION_RE.search(context):
return True
if len(item) < 4:
return True
if item.endswith(":") and len(item.split()) <= 6:
return True
return False
def is_short_noun_task(text: str) -> bool:
return len(text.split()) <= 8 and not text.endswith(".") and not re.search(r"://", text)
def clean_text(text: str) -> str:
text = text.strip()
text = re.sub(r"^☐\s*", "", text)
text = re.sub(r"^✅\s*", "", text)
text = re.sub(r"\s+", " ", text)
return text.strip(" -")
def leading_width(indent: str) -> int:
return len(indent.replace("\t", " "))
def section_title(headings: list[tuple[int, str]]) -> str:
return " > ".join(title for _, title in headings)
def priority_from_context(section: str, item: str) -> str:
text = f"{section} {item}".lower()
if "p0" in text or "critical path" in text:
return "priority/p0"
if "p1" in text or "immediate" in text or "next sprint" in text:
return "priority/p1"
if "p2" in text:
return "priority/p2"
if "p3" in text or "future" in text or "later" in text:
return "priority/p3"
return "priority/p2"
def normalize_priority(value: str) -> str:
value = value.strip().lower()
if value == "p0":
return "priority/p0"
if value == "p1":
return "priority/p1"
if value == "p2":
return "priority/p2"
if value == "p3":
return "priority/p3"
return "priority/p2"
def milestone_from_context(path: pathlib.Path, section: str) -> str:
if section:
parts = [part for part in section.split(" > ") if ACTIVE_SECTION_RE.search(part)]
if parts:
return parts[-1][:120]
return path.stem.replace("_", " ").replace("-", " ").title()
def format_title(prefix: str, item: str) -> str:
title = f"{prefix} {clean_text(item).rstrip('.')}"
if len(title) <= 180:
return title
return f"{title[:177].rstrip()}..."
def make_fingerprint(path: pathlib.Path, line: int, repo_key: str, item: str) -> str:
stable = "\n".join([str(path.resolve()), str(line), repo_key, item])
return hashlib.sha256(stable.encode("utf-8")).hexdigest()[:24]
def dedupe_candidates(candidates: list[Candidate]) -> list[Candidate]:
seen: set[tuple[str, str]] = set()
unique: list[Candidate] = []
for candidate in candidates:
key = (candidate.repo_key, normalize_title(candidate.title))
if key in seen:
continue
seen.add(key)
unique.append(candidate)
return unique
def grouped_counts(candidates: list[Candidate]) -> dict[str, int]:
counts: dict[str, int] = {}
for candidate in candidates:
counts[candidate.repo_key] = counts.get(candidate.repo_key, 0) + 1
return dict(sorted(counts.items()))
def load_repo_state(repo: RepoInfo, token: str, *, apply: bool) -> RepoState:
target = infer_target(repo.root)
client = GiteaClient(target, token)
labels = client.paginate(repo_path(target.owner, target.repo, "/labels"), limit=50)
label_ids = {str(label["name"]): int(label["id"]) for label in labels}
if apply:
for name, color, description in GENERIC_LABELS:
if name in label_ids:
continue
created = client.request_json(
"POST",
repo_path(target.owner, target.repo, "/labels"),
body={"name": name, "color": color, "description": description, "exclusive": name.startswith(("type/", "status/", "priority/"))},
)
label_ids[str(created["name"])] = int(created["id"])
print(f"created label {target.owner}/{target.repo}:{name}")
missing = [name for name, _, _ in GENERIC_LABELS if name not in label_ids]
if missing and apply:
raise GiteaError(f"{repo.key} missing labels: {', '.join(missing)}. Re-run with --apply to create them.")
milestones = client.paginate(repo_path(target.owner, target.repo, "/milestones"), query={"state": "all"}, limit=50)
milestone_ids = {str(item["title"]): int(item["id"]) for item in milestones}
issues = client.paginate(repo_path(target.owner, target.repo, "/issues"), query={"state": "all"}, limit=50)
fingerprints: set[str] = set()
titles: set[str] = set()
for issue in issues:
body = str(issue.get("body") or "")
titles.add(normalize_title(str(issue.get("title") or "")))
fingerprints.update(re.findall(r"codex-(?:generic-)?backlog-fingerprint:([0-9a-f]{24})", body))
return RepoState(target=target, client=client, label_ids=label_ids, milestone_ids=milestone_ids, fingerprints=fingerprints, titles=titles)
def ensure_milestone(state: RepoState, title: str) -> int:
if title in state.milestone_ids:
return state.milestone_ids[title]
created = state.client.request_json(
"POST",
repo_path(state.target.owner, state.target.repo, "/milestones"),
body={"title": title, "state": "open", "description": "Created from imported backlog-like files."},
)
state.milestone_ids[title] = int(created["id"])
print(f"created milestone {state.target.owner}/{state.target.repo}:{title}")
return state.milestone_ids[title]
def normalize_title(title: str) -> str:
return re.sub(r"[^a-z0-9]+", " ", title.lower()).strip()
def print_preview(candidates: list[Candidate], limit_per_repo: int = 80) -> None:
by_repo: dict[str, list[Candidate]] = {}
for candidate in candidates:
by_repo.setdefault(candidate.repo_key, []).append(candidate)
for repo_key, items in sorted(by_repo.items()):
print(f"{repo_key}:")
for item in items[:limit_per_repo]:
print(f" {item.title} [{item.priority}; {item.milestone}]")
if len(items) > limit_per_repo:
print(f" ... {len(items) - limit_per_repo} more")
if __name__ == "__main__":
raise SystemExit(main())