581 lines
21 KiB
Python
581 lines
21 KiB
Python
#!/usr/bin/env python3
|
|
"""Import markdown backlog/planning files into Gitea issues."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import dataclasses
|
|
import hashlib
|
|
import os
|
|
import pathlib
|
|
import re
|
|
import sys
|
|
from typing import Any, Iterable
|
|
|
|
from gitea_common import GiteaClient, GiteaError, infer_target, label_ids_by_name, load_dotenv, repo_path, repo_root, require_token
|
|
|
|
|
|
DEFAULT_PRODUCT_BACKLOG = pathlib.Path("/mnt/DATA/Nextcloud/ADD ideas UG/Products/govoplan/backlog.md")
|
|
DEFAULT_WEB_TODO = pathlib.Path("/mnt/DATA/git/addideas-govoplan-website/TODO.md")
|
|
|
|
REPO_ROOTS = {
|
|
"core": pathlib.Path("/mnt/DATA/git/govoplan-core"),
|
|
"access": pathlib.Path("/mnt/DATA/git/govoplan-access"),
|
|
"mail": pathlib.Path("/mnt/DATA/git/govoplan-mail"),
|
|
"files": pathlib.Path("/mnt/DATA/git/govoplan-files"),
|
|
"campaign": pathlib.Path("/mnt/DATA/git/govoplan-campaign"),
|
|
"web": pathlib.Path("/mnt/DATA/git/addideas-govoplan-website"),
|
|
}
|
|
|
|
MODULE_LABELS = {
|
|
"core": "module/core",
|
|
"access": "module/access",
|
|
"mail": "module/mail",
|
|
"files": "module/files",
|
|
"campaign": "module/campaign",
|
|
"web": "area/marketing",
|
|
}
|
|
|
|
|
|
@dataclasses.dataclass(frozen=True)
|
|
class Candidate:
|
|
repo_key: str
|
|
title: str
|
|
body: str
|
|
labels: tuple[str, ...]
|
|
fingerprint: str
|
|
source: str
|
|
line: int
|
|
|
|
|
|
@dataclasses.dataclass
|
|
class RepoState:
|
|
target: Any
|
|
client: GiteaClient
|
|
label_ids: dict[str, int]
|
|
fingerprints: set[str]
|
|
titles: set[str]
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--env-file", type=pathlib.Path, help="dotenv file to read before using GITEA_* values")
|
|
parser.add_argument("--product-backlog", type=pathlib.Path, default=DEFAULT_PRODUCT_BACKLOG)
|
|
parser.add_argument("--access-plan", type=pathlib.Path, help="optional legacy access extraction plan to import")
|
|
parser.add_argument("--web-todo", type=pathlib.Path, default=DEFAULT_WEB_TODO)
|
|
parser.add_argument("--apply", action="store_true", help="create missing Gitea issues")
|
|
args = parser.parse_args()
|
|
|
|
try:
|
|
load_dotenv(args.env_file)
|
|
candidates = list(build_candidates(args))
|
|
candidates.sort(key=lambda item: (item.repo_key, item.source, item.line, item.title))
|
|
|
|
print(f"Prepared {len(candidates)} backlog issue candidate(s).")
|
|
for repo_key, grouped in group_candidates(candidates).items():
|
|
print(f"{repo_key}: {len(grouped)} candidate(s)")
|
|
|
|
token = require_token() if args.apply else os.environ.get("GITEA_TOKEN")
|
|
if not token:
|
|
print("Dry run without GITEA_TOKEN: duplicate comparison skipped.")
|
|
print_preview(candidates)
|
|
return 0
|
|
|
|
states = load_repo_states(token, sorted({candidate.repo_key for candidate in candidates}))
|
|
missing = [candidate for candidate in candidates if not already_imported(candidate, states[candidate.repo_key])]
|
|
skipped = len(candidates) - len(missing)
|
|
|
|
print(f"Skipped {skipped} existing issue candidate(s).")
|
|
print(f"Missing {len(missing)} issue candidate(s).")
|
|
print_preview(missing)
|
|
|
|
if not args.apply:
|
|
print("Dry run only. Re-run with --apply to create missing issues.")
|
|
return 0
|
|
|
|
created_by_repo: dict[str, list[str]] = {}
|
|
for candidate in missing:
|
|
state = states[candidate.repo_key]
|
|
validate_labels(candidate, state)
|
|
issue = state.client.request_json(
|
|
"POST",
|
|
repo_path(state.target.owner, state.target.repo, "/issues"),
|
|
body={
|
|
"title": candidate.title,
|
|
"body": candidate.body,
|
|
"labels": [state.label_ids[label] for label in candidate.labels],
|
|
},
|
|
)
|
|
number = issue.get("number") or issue.get("index")
|
|
created_by_repo.setdefault(candidate.repo_key, []).append(f"#{number} {candidate.title}")
|
|
state.fingerprints.add(candidate.fingerprint)
|
|
state.titles.add(normalize_title(candidate.title))
|
|
|
|
print("Created issues:")
|
|
for repo_key, titles in created_by_repo.items():
|
|
print(f"{repo_key}:")
|
|
for title in titles:
|
|
print(f" {title}")
|
|
if not created_by_repo:
|
|
print(" none")
|
|
return 0
|
|
except GiteaError as exc:
|
|
print(f"error: {exc}", file=sys.stderr)
|
|
return 1
|
|
|
|
|
|
def build_candidates(args: argparse.Namespace) -> Iterable[Candidate]:
|
|
if args.product_backlog.exists():
|
|
yield from parse_product_backlog(args.product_backlog)
|
|
if args.access_plan and args.access_plan.exists():
|
|
yield from parse_access_plan(args.access_plan)
|
|
if args.web_todo.exists():
|
|
yield from parse_simple_todo(args.web_todo)
|
|
|
|
|
|
def parse_product_backlog(path: pathlib.Path) -> Iterable[Candidate]:
|
|
headings: list[tuple[int, str]] = []
|
|
status = ""
|
|
lines = path.read_text(encoding="utf-8").splitlines()
|
|
for index, line in enumerate(lines):
|
|
line_number = index + 1
|
|
heading_match = re.match(r"^(#{2,4})\s+(.+?)\s*$", line)
|
|
if heading_match:
|
|
level = len(heading_match.group(1))
|
|
title = heading_match.group(2).strip()
|
|
if level == 2 and title == "Completed Work Archive":
|
|
break
|
|
headings = [(h_level, h_title) for h_level, h_title in headings if h_level < level]
|
|
headings.append((level, title))
|
|
status = ""
|
|
continue
|
|
|
|
status_match = re.match(r"^\*\*Status:\*\*\s*(.+?)\s*$", line)
|
|
if status_match:
|
|
status = status_match.group(1).strip()
|
|
continue
|
|
|
|
item_match = re.match(r"^\s*[-*]\s+\[\s\]\s+(.+?)\s*$", line)
|
|
if not item_match:
|
|
continue
|
|
|
|
item = collect_continued_item(lines, index, item_match.group(1))
|
|
section = section_title(headings)
|
|
repo_key, labels = classify_product_item(item, section)
|
|
prefix = type_prefix(labels)
|
|
title = format_title(prefix, item)
|
|
source = str(path)
|
|
body = body_for_item(
|
|
fingerprint=make_fingerprint(source, line_number, repo_key, item),
|
|
source=source,
|
|
line=line_number,
|
|
section=section,
|
|
status=status,
|
|
item=item,
|
|
note="Imported from the consolidated GovOPlaN product backlog.",
|
|
)
|
|
yield Candidate(
|
|
repo_key=repo_key,
|
|
title=title,
|
|
body=body,
|
|
labels=tuple(sorted(set(labels))),
|
|
fingerprint=make_fingerprint(source, line_number, repo_key, item),
|
|
source=source,
|
|
line=line_number,
|
|
)
|
|
|
|
|
|
def parse_access_plan(path: pathlib.Path) -> Iterable[Candidate]:
|
|
text = path.read_text(encoding="utf-8")
|
|
lines = text.splitlines()
|
|
yield from access_stage_candidates(path, lines)
|
|
yield from access_immediate_candidates(path, lines)
|
|
yield from access_decision_candidates(path, lines)
|
|
|
|
|
|
def access_stage_candidates(path: pathlib.Path, lines: list[str]) -> Iterable[Candidate]:
|
|
index = 0
|
|
while index < len(lines):
|
|
match = re.match(r"^### Stage ([1-7]):\s*(.+?)\s*$", lines[index])
|
|
if not match:
|
|
index += 1
|
|
continue
|
|
stage = match.group(1)
|
|
name = match.group(2).strip()
|
|
start_line = index + 1
|
|
index += 1
|
|
block: list[str] = []
|
|
while index < len(lines) and not re.match(r"^### Stage \d+:", lines[index]) and not re.match(r"^## ", lines[index]):
|
|
block.append(lines[index])
|
|
index += 1
|
|
item = f"Access extraction Stage {stage}: {name}"
|
|
fingerprint = make_fingerprint(str(path), start_line, "core", item)
|
|
body = "\n".join(
|
|
[
|
|
f"<!-- codex-backlog-fingerprint:{fingerprint} -->",
|
|
"",
|
|
"Imported from the GovOPlaN access extraction plan.",
|
|
"",
|
|
f"- Source: `{path}`",
|
|
f"- Line: `{start_line}`",
|
|
f"- Section: `Stage {stage}: {name}`",
|
|
"",
|
|
"Plan excerpt:",
|
|
"",
|
|
"```markdown",
|
|
f"### Stage {stage}: {name}",
|
|
*block,
|
|
"```",
|
|
]
|
|
)
|
|
yield Candidate(
|
|
repo_key="core",
|
|
title=format_title("[Task]", item),
|
|
body=body,
|
|
labels=labels("core", "type/task", "area/module-system", "module/access"),
|
|
fingerprint=fingerprint,
|
|
source=str(path),
|
|
line=start_line,
|
|
)
|
|
|
|
|
|
def access_immediate_candidates(path: pathlib.Path, lines: list[str]) -> Iterable[Candidate]:
|
|
in_section = False
|
|
for index, line in enumerate(lines):
|
|
line_number = index + 1
|
|
if line == "## Immediate Backlog":
|
|
in_section = True
|
|
continue
|
|
if in_section and line.startswith("## "):
|
|
break
|
|
if not in_section:
|
|
continue
|
|
item_match = re.match(r"^\s*[-*]\s+\[\s\]\s+(.+?)\s*$", line)
|
|
if not item_match:
|
|
continue
|
|
item = collect_continued_item(lines, index, item_match.group(1))
|
|
title_item = access_title_item(item)
|
|
fingerprint = make_fingerprint(str(path), line_number, "core", item)
|
|
body = body_for_item(
|
|
fingerprint=fingerprint,
|
|
source=str(path),
|
|
line=line_number,
|
|
section="Immediate Backlog",
|
|
status="open",
|
|
item=item,
|
|
note="Imported from the GovOPlaN access extraction plan immediate backlog.",
|
|
)
|
|
yield Candidate(
|
|
repo_key="core",
|
|
title=format_title("[Task]", title_item),
|
|
body=body,
|
|
labels=labels("core", "type/task", "area/auth", "area/module-system", "module/access"),
|
|
fingerprint=fingerprint,
|
|
source=str(path),
|
|
line=line_number,
|
|
)
|
|
|
|
|
|
def access_decision_candidates(path: pathlib.Path, lines: list[str]) -> Iterable[Candidate]:
|
|
in_section = False
|
|
for index, line in enumerate(lines):
|
|
line_number = index + 1
|
|
if line == "## Open Decisions":
|
|
in_section = True
|
|
continue
|
|
if in_section and line.startswith("## "):
|
|
break
|
|
if not in_section:
|
|
continue
|
|
item_match = re.match(r"^\s*[-*]\s+(.+?)\s*$", line)
|
|
if not item_match:
|
|
continue
|
|
item = collect_continued_item(lines, index, item_match.group(1))
|
|
fingerprint = make_fingerprint(str(path), line_number, "core", item)
|
|
body = body_for_item(
|
|
fingerprint=fingerprint,
|
|
source=str(path),
|
|
line=line_number,
|
|
section="Open Decisions",
|
|
status="decision needed",
|
|
item=item,
|
|
note="Imported from the GovOPlaN access extraction plan open decisions.",
|
|
)
|
|
yield Candidate(
|
|
repo_key="core",
|
|
title=format_title("[Task]", f"Decide: {item}"),
|
|
body=body,
|
|
labels=labels("core", "type/task", "status/needs-info", "codex/needs-human", "area/auth", "module/access"),
|
|
fingerprint=fingerprint,
|
|
source=str(path),
|
|
line=line_number,
|
|
)
|
|
|
|
|
|
def parse_simple_todo(path: pathlib.Path) -> Iterable[Candidate]:
|
|
lines = path.read_text(encoding="utf-8").splitlines()
|
|
for index, line in enumerate(lines):
|
|
line_number = index + 1
|
|
item_match = re.match(r"^\s*[-*]\s+(.+?)\s*$", line)
|
|
if not item_match:
|
|
continue
|
|
item = collect_continued_item(lines, index, item_match.group(1))
|
|
fingerprint = make_fingerprint(str(path), line_number, "web", item)
|
|
yield Candidate(
|
|
repo_key="web",
|
|
title=format_title("[Task]", item),
|
|
body=body_for_item(
|
|
fingerprint=fingerprint,
|
|
source=str(path),
|
|
line=line_number,
|
|
section="TODO",
|
|
status="open",
|
|
item=item,
|
|
note="Imported from the GovOPlaN website TODO file.",
|
|
),
|
|
labels=labels("web", "type/task", "area/marketing"),
|
|
fingerprint=fingerprint,
|
|
source=str(path),
|
|
line=line_number,
|
|
)
|
|
|
|
|
|
def classify_product_item(item: str, section: str) -> tuple[str, tuple[str, ...]]:
|
|
text = f"{section} {item}".lower()
|
|
repo_key = "core"
|
|
area = "area/devex"
|
|
issue_type = "type/task"
|
|
|
|
if "dsar search/export/delete" in text:
|
|
return "core", labels("core", "type/feature", "area/governance")
|
|
if "fully stream zip uploads" in text:
|
|
return "files", labels("files", "type/task", "area/api")
|
|
if "installer packaging approach" in text:
|
|
return "core", labels("core", "type/task", "area/devex")
|
|
if "profile reselection/revalidation" in text:
|
|
return "campaign", labels("campaign", "type/task", "area/api", "module/mail")
|
|
if "module boundaries" in text:
|
|
return "core", labels("core", issue_type, "area/module-system")
|
|
if "external storage connectors" in text:
|
|
repo_key = "files"
|
|
area = "area/api"
|
|
elif any(phrase in text for phrase in ("report, evidence", "recipient import", "campaign and attachment", "collaboration and advanced governance", "address book, templates")):
|
|
repo_key = "campaign"
|
|
area = "area/webui" if any(word in text for word in ("page", "table", "display", "ui", "wizard", "editor", "flow", "polish")) else "area/api"
|
|
elif "controlled sending" in text:
|
|
repo_key = "campaign"
|
|
area = "area/api"
|
|
elif "mail profiles" in text:
|
|
repo_key = "mail"
|
|
area = "area/api"
|
|
|
|
if any(word in text for word in ("smtp", "imap", "mailbox", "mail profile", "deliverability", "bounce", "openpgp", "s/mime", "pop3", "jmap")):
|
|
repo_key = "mail"
|
|
area = "area/api"
|
|
if any(word in text for word in ("seafile", "nextcloud", "webdav", "smb", "connector", "file rename", "managed storage", "live remote file")):
|
|
repo_key = "files"
|
|
area = "area/api"
|
|
if "external storage connectors" not in text and any(
|
|
word in text
|
|
for word in (
|
|
"campaign",
|
|
"recipient",
|
|
"attachment",
|
|
"zip",
|
|
"evidence",
|
|
"wizard",
|
|
"send",
|
|
"archive",
|
|
"message preview",
|
|
"address book",
|
|
"carddav",
|
|
"mailing lists",
|
|
)
|
|
):
|
|
repo_key = "campaign"
|
|
area = "area/webui" if any(word in text for word in ("page", "table", "display", "ui", "wizard", "editor", "flow", "polish")) else "area/api"
|
|
if repo_key == "core" and any(word in text for word in ("session", "device", "auth", "rbac", "role", "tenant", "governance", "policy", "retention", "dsar", "audit", "encryption", "ldap", "oidc", "saml")):
|
|
repo_key = "core"
|
|
area = "area/auth" if any(word in text for word in ("session", "auth", "ldap", "oidc", "saml")) else "area/governance"
|
|
if repo_key == "core" and any(word in text for word in ("module", "manifest", "module boundaries", "installable", "activatable", "deactivatable", "uninstallable")):
|
|
repo_key = "core"
|
|
area = "area/module-system"
|
|
if repo_key == "core" and any(word in text for word in ("release", "version metadata", "lockfile", "package", "tagging")):
|
|
repo_key = "core"
|
|
area = "area/release"
|
|
if repo_key == "core" and not any(
|
|
phrase in text
|
|
for phrase in (
|
|
"external storage connectors",
|
|
"controlled sending",
|
|
"report, evidence",
|
|
"recipient import",
|
|
"campaign and attachment",
|
|
)
|
|
) and any(word in text for word in ("postgres", "redis", "celery", "worker", "deployment", "installer", "backup", "monitoring", "configuration", "dev profile", ".env")):
|
|
repo_key = "core"
|
|
area = "area/devex"
|
|
if any(word in text for word in ("documentation", "handbook", "docs", "runbook")):
|
|
area = "area/docs"
|
|
if "ui lists often start at index 2" in text:
|
|
issue_type = "type/bug"
|
|
area = "area/webui"
|
|
repo_key = "core"
|
|
if any(word in text for word in ("add ", "implement ", "build ", "provide ", "allow ", "make ", "extend ", "replace ")):
|
|
issue_type = "type/feature"
|
|
if any(word in text for word in ("cleanup", "remove legacy", "fragile", "deferred cleanup")):
|
|
issue_type = "type/debt"
|
|
|
|
return repo_key, labels(repo_key, issue_type, area)
|
|
|
|
|
|
def collect_continued_item(lines: list[str], index: int, initial: str) -> str:
|
|
parts = [initial.strip()]
|
|
next_index = index + 1
|
|
while next_index < len(lines):
|
|
line = lines[next_index]
|
|
if not line.startswith((" ", "\t")):
|
|
break
|
|
stripped = line.strip()
|
|
if not stripped:
|
|
break
|
|
if re.match(r"^[-*]\s+", stripped) or stripped.startswith("#"):
|
|
break
|
|
parts.append(stripped)
|
|
next_index += 1
|
|
return " ".join(parts)
|
|
|
|
|
|
def labels(repo_key: str, *extra: str) -> tuple[str, ...]:
|
|
base = {"status/triage", "source/backlog-import", "codex/ready", MODULE_LABELS[repo_key]}
|
|
base.update(extra)
|
|
if "status/needs-info" in base:
|
|
base.discard("status/triage")
|
|
return tuple(sorted(base))
|
|
|
|
|
|
def type_prefix(label_set: Iterable[str]) -> str:
|
|
labels_set = set(label_set)
|
|
if "type/bug" in labels_set:
|
|
return "[Bug]"
|
|
if "type/feature" in labels_set:
|
|
return "[Feature]"
|
|
if "type/debt" in labels_set:
|
|
return "[Debt]"
|
|
if "type/docs" in labels_set:
|
|
return "[Docs]"
|
|
return "[Task]"
|
|
|
|
|
|
def format_title(prefix: str, item: str) -> str:
|
|
clean = re.sub(r"\s+", " ", item).strip().rstrip(".")
|
|
title = f"{prefix} {clean}"
|
|
if len(title) <= 180:
|
|
return title
|
|
return f"{title[:177].rstrip()}..."
|
|
|
|
|
|
def access_title_item(item: str) -> str:
|
|
lowered = item.lower()
|
|
if "compatibility wrappers" in lowered:
|
|
return "Wire core auth compatibility through access capabilities"
|
|
return item
|
|
|
|
|
|
def body_for_item(
|
|
*,
|
|
fingerprint: str,
|
|
source: str,
|
|
line: int,
|
|
section: str,
|
|
status: str,
|
|
item: str,
|
|
note: str,
|
|
) -> str:
|
|
lines = [
|
|
f"<!-- codex-backlog-fingerprint:{fingerprint} -->",
|
|
"",
|
|
note,
|
|
"",
|
|
f"- Source: `{source}`",
|
|
f"- Line: `{line}`",
|
|
f"- Section: `{section}`",
|
|
]
|
|
if status:
|
|
lines.append(f"- Source status: `{status}`")
|
|
lines.extend(
|
|
[
|
|
"",
|
|
"Imported backlog item:",
|
|
"",
|
|
"```markdown",
|
|
f"- [ ] {item}",
|
|
"```",
|
|
]
|
|
)
|
|
return "\n".join(lines)
|
|
|
|
|
|
def section_title(headings: list[tuple[int, str]]) -> str:
|
|
return " > ".join(title for _, title in headings)
|
|
|
|
|
|
def make_fingerprint(source: str, line: int, repo_key: str, item: str) -> str:
|
|
stable = "\n".join([source, str(line), repo_key, item])
|
|
return hashlib.sha256(stable.encode("utf-8")).hexdigest()[:24]
|
|
|
|
|
|
def group_candidates(candidates: list[Candidate]) -> dict[str, list[Candidate]]:
|
|
grouped: dict[str, list[Candidate]] = {}
|
|
for candidate in candidates:
|
|
grouped.setdefault(candidate.repo_key, []).append(candidate)
|
|
return grouped
|
|
|
|
|
|
def load_repo_states(token: str, repo_keys: list[str]) -> dict[str, RepoState]:
|
|
states: dict[str, RepoState] = {}
|
|
for repo_key in repo_keys:
|
|
root = repo_root(REPO_ROOTS[repo_key])
|
|
target = infer_target(root)
|
|
client = GiteaClient(target, token)
|
|
label_ids = label_ids_by_name(client, target.owner, target.repo)
|
|
issues = client.paginate(repo_path(target.owner, target.repo, "/issues"), query={"state": "all"}, limit=100)
|
|
fingerprints: set[str] = set()
|
|
titles: set[str] = set()
|
|
for issue in issues:
|
|
body = str(issue.get("body") or "")
|
|
if issue.get("state") == "closed" and "Moved to `" in body:
|
|
continue
|
|
titles.add(normalize_title(str(issue.get("title") or "")))
|
|
fingerprints.update(re.findall(r"codex-backlog-fingerprint:([0-9a-f]{24})", body))
|
|
states[repo_key] = RepoState(target=target, client=client, label_ids=label_ids, fingerprints=fingerprints, titles=titles)
|
|
return states
|
|
|
|
|
|
def already_imported(candidate: Candidate, state: RepoState) -> bool:
|
|
return candidate.fingerprint in state.fingerprints or normalize_title(candidate.title) in state.titles
|
|
|
|
|
|
def normalize_title(title: str) -> str:
|
|
return re.sub(r"[^a-z0-9]+", " ", title.lower()).strip()
|
|
|
|
|
|
def validate_labels(candidate: Candidate, state: RepoState) -> None:
|
|
missing = sorted(label for label in candidate.labels if label not in state.label_ids)
|
|
if missing:
|
|
joined = ", ".join(missing)
|
|
raise GiteaError(f"{candidate.repo_key} is missing labels for {candidate.title!r}: {joined}")
|
|
|
|
|
|
def print_preview(candidates: list[Candidate], limit: int = 160) -> None:
|
|
for repo_key, grouped in group_candidates(candidates).items():
|
|
print(f"{repo_key}:")
|
|
for candidate in grouped[:limit]:
|
|
print(f" {candidate.title}")
|
|
if len(grouped) > limit:
|
|
print(f" ... {len(grouped) - limit} more")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|