378 lines
13 KiB
Python
378 lines
13 KiB
Python
#!/usr/bin/env python3
|
|
"""Preview or import inline TODO-style markers as Gitea issues."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import pathlib
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
from typing import Any
|
|
|
|
from gitea_common import GiteaClient, GiteaError, infer_target, label_ids_by_name, load_dotenv, repo_path, repo_root, require_token
|
|
|
|
|
|
MARKER_RE = re.compile(
|
|
r"(?P<prefix>#|//|/\*|\*|--|<!--)?\s*"
|
|
r"\b(?P<marker>TODO|FIXME|XXX|HACK)\b"
|
|
r"\s*(?:\((?P<context>[^)]{1,120})\))?"
|
|
r"\s*(?P<colon>:)?\s*(?P<text>.*)",
|
|
re.IGNORECASE,
|
|
)
|
|
DEFAULT_EXCLUDES = (
|
|
"!**/.git/**",
|
|
"!**/.venv/**",
|
|
"!**/node_modules/**",
|
|
"!**/__pycache__/**",
|
|
"!**/.module-test-build/**",
|
|
"!**/dist/**",
|
|
"!**/build/**",
|
|
"!**/.cache/**",
|
|
"!.gitea/**",
|
|
"!docs/GITEA_ISSUES.md",
|
|
"!scripts/gitea-todo-import.py",
|
|
"!scripts/gitea-sync-labels.py",
|
|
"!scripts/gitea-codex-note.py",
|
|
"!scripts/gitea_common.py",
|
|
)
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--root", type=pathlib.Path, default=pathlib.Path.cwd(), help="repository root or child path")
|
|
parser.add_argument("--remote", default="origin", help="git remote to use for target inference")
|
|
parser.add_argument("--env-file", type=pathlib.Path, help="dotenv file to read before using GITEA_* values")
|
|
parser.add_argument("--apply", action="store_true", help="create missing Gitea issues")
|
|
parser.add_argument("--include-linked", action="store_true", help="include markers that already reference an issue")
|
|
parser.add_argument(
|
|
"--module-label",
|
|
help="project/module label to apply; defaults to a known GovOPlaN mapping or module/core",
|
|
)
|
|
parser.add_argument("--no-area-labels", action="store_true", help="do not infer area/* labels from paths")
|
|
parser.add_argument("--extra-label", action="append", default=[], help="additional label name to apply")
|
|
parser.add_argument("--limit", type=int, default=0, help="maximum number of markers to process")
|
|
parser.add_argument("--json", action="store_true", help="print issue previews as JSON")
|
|
args = parser.parse_args()
|
|
|
|
try:
|
|
root = repo_root(args.root)
|
|
load_dotenv(args.env_file or root / ".env")
|
|
target = infer_target(root, args.remote)
|
|
token = require_token() if args.apply else os.environ.get("GITEA_TOKEN")
|
|
|
|
markers = scan_markers(root, include_linked=args.include_linked)
|
|
if args.limit > 0:
|
|
markers = markers[: args.limit]
|
|
|
|
module_label = args.module_label or infer_module_label(root)
|
|
previews = [
|
|
build_issue_preview(
|
|
target.owner,
|
|
target.repo,
|
|
marker,
|
|
module_label=module_label,
|
|
infer_areas=not args.no_area_labels,
|
|
extra_labels=args.extra_label,
|
|
)
|
|
for marker in markers
|
|
]
|
|
|
|
if args.json:
|
|
print(json.dumps([preview.as_dict() for preview in previews], indent=2, sort_keys=True))
|
|
else:
|
|
print(f"Target: {target.display}")
|
|
print(f"Scanned root: {root}")
|
|
print(f"Found {len(previews)} importable marker(s).")
|
|
for preview in previews:
|
|
print(f"- {preview.title}")
|
|
print(f" {preview.location}")
|
|
print(f" labels: {', '.join(preview.labels)}")
|
|
|
|
if not args.apply:
|
|
if not previews:
|
|
return 0
|
|
print("Dry run only. Re-run with --apply and GITEA_TOKEN to create issues.")
|
|
return 0
|
|
|
|
if not previews:
|
|
print("No issues to create.")
|
|
return 0
|
|
|
|
client = GiteaClient(target, token)
|
|
label_ids = load_label_ids(client, target.owner, target.repo)
|
|
missing_labels = sorted({label for preview in previews for label in preview.labels if label not in label_ids})
|
|
if missing_labels:
|
|
joined = ", ".join(missing_labels)
|
|
raise GiteaError(f"missing labels: {joined}. Run scripts/gitea-sync-labels.py --apply first.")
|
|
|
|
existing_fingerprints = load_existing_fingerprints(client, target.owner, target.repo)
|
|
created = 0
|
|
skipped = 0
|
|
for preview in previews:
|
|
if preview.fingerprint in existing_fingerprints:
|
|
print(f"skip existing {preview.location}")
|
|
skipped += 1
|
|
continue
|
|
issue = client.request_json(
|
|
"POST",
|
|
repo_path(target.owner, target.repo, "/issues"),
|
|
body={
|
|
"title": preview.title,
|
|
"body": preview.body,
|
|
"labels": [label_ids[label] for label in preview.labels],
|
|
},
|
|
)
|
|
print(f"created #{issue.get('number') or issue.get('index')}: {preview.title}")
|
|
created += 1
|
|
|
|
print(f"Created {created} issue(s), skipped {skipped} existing issue(s).")
|
|
return 0
|
|
except GiteaError as exc:
|
|
print(f"error: {exc}", file=sys.stderr)
|
|
return 1
|
|
|
|
|
|
class Marker:
|
|
def __init__(self, path: pathlib.Path, line: int, column: int, marker: str, context: str, text: str, raw: str) -> None:
|
|
self.path = path
|
|
self.line = line
|
|
self.column = column
|
|
self.marker = marker.upper()
|
|
self.context = context
|
|
self.text = text
|
|
self.raw = raw.rstrip("\n")
|
|
|
|
@property
|
|
def location(self) -> str:
|
|
return f"{self.path}:{self.line}"
|
|
|
|
|
|
class IssuePreview:
|
|
def __init__(self, title: str, body: str, labels: list[str], fingerprint: str, location: str) -> None:
|
|
self.title = title
|
|
self.body = body
|
|
self.labels = labels
|
|
self.fingerprint = fingerprint
|
|
self.location = location
|
|
|
|
def as_dict(self) -> dict[str, Any]:
|
|
return {
|
|
"title": self.title,
|
|
"body": self.body,
|
|
"labels": self.labels,
|
|
"fingerprint": self.fingerprint,
|
|
"location": self.location,
|
|
}
|
|
|
|
|
|
def scan_markers(root: pathlib.Path, *, include_linked: bool) -> list[Marker]:
|
|
command = [
|
|
"rg",
|
|
"--json",
|
|
"--hidden",
|
|
"--line-number",
|
|
"--column",
|
|
"-e",
|
|
r"\b(TODO|FIXME|XXX|HACK)\b",
|
|
]
|
|
for pattern in DEFAULT_EXCLUDES:
|
|
command.extend(["--glob", pattern])
|
|
command.append(str(root))
|
|
|
|
result = subprocess.run(
|
|
command,
|
|
check=False,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
text=True,
|
|
)
|
|
if result.returncode not in {0, 1}:
|
|
raise GiteaError(f"rg failed: {result.stderr.strip()}")
|
|
|
|
markers: list[Marker] = []
|
|
for line in result.stdout.splitlines():
|
|
event = json.loads(line)
|
|
if event.get("type") != "match":
|
|
continue
|
|
data = event["data"]
|
|
raw_line = data["lines"]["text"]
|
|
match = MARKER_RE.search(raw_line)
|
|
if not match:
|
|
continue
|
|
if not _looks_like_marker(match):
|
|
continue
|
|
context = (match.group("context") or "").strip()
|
|
if not include_linked and _already_linked(context, raw_line):
|
|
continue
|
|
markers.append(
|
|
Marker(
|
|
path=pathlib.Path(data["path"]["text"]).resolve().relative_to(root),
|
|
line=int(data["line_number"]),
|
|
column=int(data.get("submatches", [{}])[0].get("start", 0)) + 1,
|
|
marker=match.group("marker"),
|
|
context=context,
|
|
text=clean_marker_text(match.group("text") or raw_line),
|
|
raw=raw_line,
|
|
)
|
|
)
|
|
return markers
|
|
|
|
|
|
def build_issue_preview(
|
|
owner: str,
|
|
repo: str,
|
|
marker: Marker,
|
|
*,
|
|
module_label: str,
|
|
infer_areas: bool,
|
|
extra_labels: list[str],
|
|
) -> IssuePreview:
|
|
fingerprint = marker_fingerprint(owner, repo, marker)
|
|
summary = marker.text or f"review {marker.location}"
|
|
title = truncate_title(f"{marker.marker}: {summary} ({marker.location})")
|
|
labels = sorted(set(default_labels(marker, module_label=module_label, infer_areas=infer_areas) + list(extra_labels)))
|
|
body = "\n".join(
|
|
[
|
|
f"<!-- codex-todo-fingerprint:{fingerprint} -->",
|
|
"",
|
|
"Imported from an inline marker.",
|
|
"",
|
|
f"- Repository: `{owner}/{repo}`",
|
|
f"- Location: `{marker.location}`",
|
|
f"- Marker: `{marker.marker}`",
|
|
f"- Context: `{marker.context or 'none'}`",
|
|
"",
|
|
"Current line:",
|
|
"",
|
|
"```text",
|
|
marker.raw,
|
|
"```",
|
|
"",
|
|
"Migration rule:",
|
|
"",
|
|
"- Keep this Gitea issue as the canonical backlog item.",
|
|
"- When touching this code, remove the inline marker or replace it with a short issue reference.",
|
|
]
|
|
)
|
|
return IssuePreview(title=title, body=body, labels=labels, fingerprint=fingerprint, location=marker.location)
|
|
|
|
|
|
def clean_marker_text(text: str) -> str:
|
|
cleaned = text.strip()
|
|
cleaned = re.sub(r"\s*(?:#|//|/\*|\*|--)\s*$", "", cleaned).strip()
|
|
cleaned = re.sub(r"\s*\*/\s*$", "", cleaned).strip()
|
|
return cleaned
|
|
|
|
|
|
def default_labels(marker: Marker, *, module_label: str, infer_areas: bool) -> list[str]:
|
|
labels = ["source/todo-scan", "status/triage", module_label, marker_type_label(marker.marker)]
|
|
area = infer_area_label(marker.path) if infer_areas else ""
|
|
if area:
|
|
labels.append(area)
|
|
return labels
|
|
|
|
|
|
def marker_type_label(marker: str) -> str:
|
|
if marker == "FIXME":
|
|
return "type/bug"
|
|
if marker in {"HACK", "XXX"}:
|
|
return "type/debt"
|
|
return "type/task"
|
|
|
|
|
|
def infer_area_label(path: pathlib.Path) -> str:
|
|
text = path.as_posix()
|
|
if text.startswith("webui/"):
|
|
return "area/webui"
|
|
if text.startswith("alembic/"):
|
|
return "area/migrations"
|
|
if text.startswith("docs/"):
|
|
return "area/docs"
|
|
if text.startswith("scripts/"):
|
|
return "area/devex"
|
|
if "/rbac" in text or "permission" in text:
|
|
return "area/rbac"
|
|
if "/access" in text or "auth" in text or "session" in text:
|
|
return "area/auth"
|
|
if "tenant" in text:
|
|
return "area/tenancy"
|
|
if "governance" in text or "privacy" in text or "audit" in text or "retention" in text:
|
|
return "area/governance"
|
|
if "module" in text:
|
|
return "area/module-system"
|
|
if "migration" in text:
|
|
return "area/migrations"
|
|
if "router" in text or "api" in text:
|
|
return "area/api"
|
|
if "db" in text or "model" in text or "persistence" in text:
|
|
return "area/db"
|
|
return ""
|
|
|
|
|
|
def infer_module_label(root: pathlib.Path) -> str:
|
|
known = {
|
|
"govoplan-core": "module/core",
|
|
"govoplan-access": "module/access",
|
|
"govoplan-admin": "module/admin",
|
|
"govoplan-tenancy": "module/tenancy",
|
|
"govoplan-policy": "module/policy",
|
|
"govoplan-audit": "module/audit",
|
|
"govoplan-mail": "module/mail",
|
|
"govoplan-files": "module/files",
|
|
"govoplan-campaign": "module/campaign",
|
|
"govoplan-web": "module/web",
|
|
}
|
|
if root.name in known:
|
|
return known[root.name]
|
|
if root.name.startswith("govoplan-"):
|
|
suffix = root.name.removeprefix("govoplan-")
|
|
if suffix:
|
|
return f"module/{suffix}"
|
|
return "module/core"
|
|
|
|
|
|
def marker_fingerprint(owner: str, repo: str, marker: Marker) -> str:
|
|
stable = "\n".join([owner, repo, marker.path.as_posix(), marker.marker, marker.context, marker.text])
|
|
return hashlib.sha256(stable.encode("utf-8")).hexdigest()[:24]
|
|
|
|
|
|
def truncate_title(title: str, limit: int = 180) -> str:
|
|
if len(title) <= limit:
|
|
return title
|
|
return f"{title[: limit - 1].rstrip()}..."
|
|
|
|
|
|
def load_label_ids(client: GiteaClient, owner: str, repo: str) -> dict[str, int]:
|
|
return label_ids_by_name(client, owner, repo)
|
|
|
|
|
|
def load_existing_fingerprints(client: GiteaClient, owner: str, repo: str) -> set[str]:
|
|
issues = client.paginate(
|
|
repo_path(owner, repo, "/issues"),
|
|
query={"state": "all", "labels": "source/todo-scan"},
|
|
)
|
|
fingerprints: set[str] = set()
|
|
for issue in issues:
|
|
body = str(issue.get("body") or "")
|
|
for match in re.finditer(r"(?:codex|govoplan)-todo-fingerprint:([0-9a-f]{24})", body):
|
|
fingerprints.add(match.group(1))
|
|
return fingerprints
|
|
|
|
|
|
def _already_linked(context: str, raw_line: str) -> bool:
|
|
linked_text = f"{context} {raw_line}".lower()
|
|
return "gitea#" in linked_text or "issue#" in linked_text or re.search(r"#\d+", linked_text) is not None
|
|
|
|
|
|
def _looks_like_marker(match: re.Match[str]) -> bool:
|
|
text = (match.group("text") or "").strip()
|
|
return bool(match.group("context") or match.group("colon") or (match.group("prefix") and text))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|