feat(devkit): add resumable workspace automation and UI review tooling

Verified with the coordinated workspace changes by devkit full run 2026-09-08T225814-186389-0000-3e3ed7cd (all seven phases passed). This shared UI pass does not mark the individual module reviews complete.
2026-09-09 02:03:17 +02:00
parent 14b19fbead
commit 2ffdb23f69
67 changed files with 17306 additions and 94 deletions
@@ -0,0 +1,386 @@
+#!/usr/bin/env python3
+"""Inventory all module UI review scopes and safely create missing Gitea tracks.
+
+Dry-run by default. Existing issues are never rewritten or closed. The optional
+one-time epic link initialization refuses to replace an edited or completed list.
+"""
+
+from __future__ import annotations
+
+import argparse
+from concurrent.futures import ThreadPoolExecutor
+from contextlib import contextmanager
+import importlib.util
+import json
+from pathlib import Path
+import re
+import socket
+import sys
+from typing import Any, Iterator
+
+from gitea_common import (
+    GiteaClient, GiteaError, RepoTarget, load_dotenv,
+    org_path, repo_path, require_token,
+)
+
+
+META_ROOT = Path(__file__).resolve().parents[2]
+BASE_URL = "https://git.add-ideas.de"
+OWNER = "GovOPlaN"
+EPIC_MARKER = "<!-- govoplan-ui-review:v1:epic -->"
+LIST_START = "<!-- govoplan-ui-review:module-list:start -->"
+LIST_END = "<!-- govoplan-ui-review:module-list:end -->"
+INITIAL_LIST = "The linked inventory is being initialized. All 77 tracks are pending; no checkboxes are complete."
+PRINCIPLES_URL = f"{BASE_URL}/{OWNER}/govoplan-core/src/branch/main/docs/UI_DESIGN_PRINCIPLES.md"
+PROCESS_URL = f"{BASE_URL}/{OWNER}/govoplan/src/branch/main/docs/project/UI_REVIEW_PROGRAM.md"
+PRINCIPLES = (
+    ("UI-01", "Help icons beside the relevant heading/label, not action-button rows"),
+    ("UI-02", "Display-first with scoped edit dialogs; explicit bulk-grid editing exception"),
+    ("UI-03", "Shared page actions, Reload/New ordering, Save/Cancel and destructive separation"),
+    ("UI-04", "Full-width table/card geometry, visible actions, pagination and two-way resizing"),
+    ("UI-05", "Scoped loading/error feedback, useful progress and retained state"),
+    ("UI-06", "Predictable tree selection, expansion, grouping and reordering"),
+    ("UI-07", "Keyboard/focus/accessibility, responsive layouts and understandable German"),
+    ("UI-08", "Authorization, optional-module boundaries and save/cancel/retry data integrity"),
+    ("UI-09", "Revision evidence and retroactive checks for changed design principles"),
+)
+
+
+def source_url(repository: str, path: str) -> str:
+    return f"{BASE_URL}/{OWNER}/{repository}/src/branch/main/{path}"
+
+
+def marker(scope_id: str) -> str:
+    return f"<!-- govoplan-ui-review:v1:module:{scope_id} -->"
+
+
+def normalized_title(value: str) -> str:
+    return " ".join(value.casefold().split())
+
+
+def find_existing(issues: list[dict[str, Any]], scope_id: str, title: str) -> dict[str, Any] | None:
+    matches = [
+        issue for issue in issues if issue.get("pull_request") is None and (
+            marker(scope_id) in (issue.get("body") or "")
+            or normalized_title(issue.get("title") or "") == normalized_title(title)
+        )
+    ]
+    if len(matches) > 1:
+        raise GiteaError(f"Ambiguous review issue matches for {scope_id}; inspect before making changes.")
+    if matches and marker(scope_id) not in (matches[0].get("body") or ""):
+        raise GiteaError(f"Unmanaged exact-title issue for {scope_id}; preserve it and resolve the duplicate manually.")
+    return matches[0] if matches else None
+
+
+def extract_manifests(catalog: dict[str, Any], workspace_root: Path) -> list[dict[str, Any]]:
+    path = META_ROOT / "tools/inventory/platform-interface-inventory.py"
+    spec = importlib.util.spec_from_file_location("ui_review_source_inventory", path)
+    assert spec and spec.loader
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[spec.name] = module
+    spec.loader.exec_module(module)
+    return module._extract_manifests(catalog, workspace_root)
+
+
+def source_groups(root: Path) -> dict[str, list[str]]:
+    groups: dict[str, list[str]] = {
+        "Pages and navigation entrypoints": [],
+        "Dialogs and embedded editing surfaces": [],
+        "Settings and administrator surfaces": [],
+        "Widgets, public/operator and contributed surfaces": [],
+        "Shared components and other UI entrypoints": [],
+    }
+    source_root = root / "webui/src"
+    for path in sorted(source_root.rglob("*.tsx")):
+        relative = path.relative_to(root).as_posix()
+        lower = relative.casefold()
+        name = path.stem.casefold()
+        if "page" in name or "navigation" in name or name in {"app", "routes", "index"}:
+            groups["Pages and navigation entrypoints"].append(relative)
+        elif re.search(r"dialog|modal|drawer|chooser|overlay", name):
+            groups["Dialogs and embedded editing surfaces"].append(relative)
+        elif re.search(r"settings|configur|admin", lower):
+            groups["Settings and administrator surfaces"].append(relative)
+        elif re.search(r"widget|public|operator|contribution", lower):
+            groups["Widgets, public/operator and contributed surfaces"].append(relative)
+        else:
+            groups["Shared components and other UI entrypoints"].append(relative)
+    return groups
+
+
+def build_scopes(
+    catalog: dict[str, Any], workspace_root: Path, manifests: list[dict[str, Any]],
+) -> list[dict[str, Any]]:
+    selected = [
+        repo for repo in catalog["repositories"]
+        if repo["category"] in {"module", "connector"} or repo["name"] == "govoplan-core"
+    ]
+    by_repository: dict[str, dict[str, Any]] = {}
+    ids: set[str] = set()
+    for manifest in manifests:
+        if manifest["repository"] in by_repository or manifest["id"] in ids:
+            raise GiteaError("Duplicate source manifest repository or module ID.")
+        by_repository[manifest["repository"]] = manifest
+        ids.add(manifest["id"])
+    selected_names = {repo["name"] for repo in selected}
+    if set(by_repository) - selected_names:
+        raise GiteaError("Source manifests contain repositories absent from the module review catalog.")
+    scopes: list[dict[str, Any]] = []
+    for repo in selected:
+        root = workspace_root / repo["path"]
+        if not root.is_dir():
+            raise GiteaError(f"Missing source checkout for {repo['name']}; cannot infer review scope safely.")
+        manifest = by_repository.get(repo["name"])
+        paths = sorted(path.relative_to(root).as_posix() for path in root.glob("src/*/backend/manifest.py"))
+        if repo["name"] == "govoplan-core":
+            scope_id, name, kind = "core", "Core / shared shell", "core"
+        elif manifest:
+            scope_id, name, kind = manifest["id"], manifest["name"], "manifest"
+        else:
+            if paths or (root / "pyproject.toml").exists() or (root / "webui/package.json").exists():
+                raise GiteaError(f"{repo['name']} has implementation but no extracted manifest; inspect instead of calling it a placeholder.")
+            scope_id = "catalog:" + repo["name"]
+            name = repo["name"].removeprefix("govoplan-").replace("-", " ").title()
+            kind = "placeholder"
+        groups = source_groups(root)
+        scopes.append({
+            "scope_id": scope_id, "name": name, "repository": repo["name"],
+            "kind": kind, "manifest_paths": paths,
+            "frontend": manifest.get("frontend") if manifest else None,
+            "source_groups": groups,
+            "ui_source_count": sum(len(paths) for paths in groups.values()),
+        })
+    if len({scope["scope_id"] for scope in scopes}) != len(scopes):
+        raise GiteaError("Duplicate review scope IDs.")
+    return sorted(scopes, key=lambda scope: (scope["kind"] == "placeholder", scope["scope_id"] != "core", scope["name"].casefold()))
+
+
+def issue_title(scope: dict[str, Any]) -> str:
+    suffix = "readiness and future UI review" if scope["kind"] == "placeholder" else "visual and interaction conformance"
+    return f"[UI review] {scope['name']}: {suffix}"
+
+
+def source_seed(scope: dict[str, Any]) -> str:
+    repo = scope["repository"]
+    lines = [
+        "This is a **source-derived starting inventory, not a completed runtime audit**. Verify nested routes, embedded dialogs and contributions in the installed module context; add missing surfaces to this issue.",
+        "", f"Repository: [{repo}]({BASE_URL}/{OWNER}/{repo}).",
+    ]
+    if scope["kind"] == "placeholder":
+        lines += [
+            "", f"Catalog entry `{repo}` is currently README-only; there is no runtime module ID, manifest or standalone WebUI to claim as reviewed. Source: [README]({source_url(repo, 'README.md')}).",
+            "", "- [ ] Confirm the catalog/readiness scope and record prerequisites for the first implementation.",
+            "- [ ] Keep the future interface review pending until actual configuration, public/operator or UI surfaces exist; do not manufacture N/A evidence to close this track.",
+        ]
+        return "\n".join(lines)
+    for path in scope["manifest_paths"]:
+        lines.append(f"Manifest source: [{path}]({source_url(repo, path)}).")
+    if scope["kind"] == "core":
+        lines += [
+            "", "Core/shared shell additionally owns navigation/rail/breadcrumbs, module routing, page/action archetypes, cards/tables/forms/dialogs, loading and error surfaces, help affordances, authentication and user settings. Review optional-module and permission contexts, not just standalone primitives.",
+        ]
+    frontend = scope["frontend"]
+    if frontend:
+        for key, label in (
+            ("routes", "Declared routes"), ("public_routes", "Declared public routes"),
+            ("settings_routes", "Declared settings routes"), ("nav_items", "Declared navigation"),
+            ("view_surfaces", "Declared view, settings and contributed surfaces"),
+        ):
+            entries = frontend.get(key) or []
+            lines += ["", f"**{label} ({len(entries)}):**"]
+            if not entries:
+                lines.append("None declared in this manifest; verify indirect/contributed surfaces before marking anything not applicable.")
+            for entry in entries:
+                identity = entry.get("path") or entry.get("id") or entry.get("component") or "unnamed declaration"
+                detail = entry.get("component") or entry.get("label") or entry.get("kind") or ""
+                lines.append(f"- `{identity}`" + (f" — {detail}" if detail else ""))
+    elif scope["kind"] != "core":
+        lines += [
+            "", "No standalone frontend is declared. **The review is still pending:** inspect owned configuration/admin workflows, manifest documentation, errors and any interfaces contributed through host modules, public routes or operator tools. Record concrete evidence before claiming a principle does not apply.",
+        ]
+    lines += ["", f"<details><summary>Source entrypoint seed ({scope['ui_source_count']} TSX files; classification is heuristic)</summary>", ""]
+    for label, paths in scope["source_groups"].items():
+        if not paths:
+            continue
+        lines += [f"**{label} ({len(paths)}):**", ""]
+        for path in paths[:40]:
+            lines.append(f"- [{path}]({source_url(repo, path)})")
+        if len(paths) > 40:
+            lines.append(f"- {len(paths) - 40} further files: inspect [the source tree]({source_url(repo, 'webui/src')}); expand the issue inventory during review.")
+        lines.append("")
+    lines += ["</details>"]
+    return "\n".join(lines)
+
+
+def issue_body(scope: dict[str, Any], epic_url: str) -> str:
+    lines = [
+        marker(scope["scope_id"]), "## Status and objective", "",
+        f"**Pending / not reviewed.** This module track belongs to the [product-wide UI review epic]({epic_url}).",
+        f"Apply the [Core design principles]({PRINCIPLES_URL}) using the [shared review process]({PROCESS_URL}). The current heading-help icon pass and any existing isolated fixes are preparation, not evidence that this whole module is complete. Source documents are being prepared in the current working tree; this issue does not claim they are released.",
+        "", "Prioritize usability defects, consistent interaction and shared-component fixes before broader features. Preserve permissions, security, optional-module boundaries and data integrity.",
+        "", "## Source inventory to verify", "", source_seed(scope),
+        "", "## Review and implementation TODO", "",
+        "- [ ] Confirm every actual page, nested route, dialog, field/form, table/tree, settings level, public/operator surface, widget and cross-module contribution; document role/module prerequisites.",
+        "- [ ] Exercise EN/DE, keyboard/focus, narrow and wide layouts, long content, empty/loading/error states and realistic datasets.",
+        "- [ ] Check UI-01 heading/label help placement and UI-02 display-first/scoped editing; document any justified large-grid bulk-edit exception with explicit mode, Save/Cancel and dirty-navigation guard.",
+        "- [ ] Verify consistent top-right actions (Reload left of New), clean/dirty Save/Cancel behavior, destructive separation and safe navigation/reload.",
+        "- [ ] Verify full-width cards/tables, visible last-column actions, pagination, initial sizing, two-way pointer/keyboard resizing and preference reload with fixed columns and horizontal overflow.",
+        "- [ ] Check scoped progress/error feedback and predictable tree selection versus expansion; no unnecessary global blocking or repeated background reload.",
+        "- [ ] Record findings and implement shared-contract corrections plus all affected consumers, not local CSS/action-row exceptions without justification.",
+        "- [ ] Verify save/cancel/retry and partial-failure behavior without unintended writes, sends, deletes or loss of persisted/unsaved data.",
+        "- [ ] Update owning manifest-driven EN/DE documentation; record targeted automated checks and manual evidence against actual module surfaces.",
+        "- [ ] Complete the principle matrix, list unresolved decisions/manual checks and link follow-ups before proposing closure.",
+    ]
+    if scope["scope_id"] == "campaigns":
+        lines += [
+            "", "### Campaign-specific starting direction", "",
+            "- [ ] Present a compact read-only campaign settings dashboard/overview and use explicit scoped edit dialogs for settings instead of a permanently editable form wall.",
+            "- [ ] Keep large recipient/attachment tables practical through an explicit bulk-edit mode where appropriate, with Save/Cancel, dirty-state protection and reload persistence; this is the documented UI-02 exception, not silent autosave.",
+            "- [ ] Review the complete compose → attachments → validation/review → delivery/report/operator workflow, including mail-profile migration, ZIP policy, multiple recipients and SMTP/IMAP progress, without sending live messages just to collect UI evidence.",
+        ]
+    lines += [
+        "", "## Principle applicability / application / evidence / exceptions", "",
+        "Reviewed Core principle revision: **not yet recorded**. No exceptions approved.", "",
+        "| Principle | Applicable surfaces / justified N/A | Applied / remaining work | Evidence | Exception / owner / follow-up |",
+        "| --- | --- | --- | --- | --- |",
+    ]
+    for identity, description in PRINCIPLES:
+        lines.append(f"| {identity}: {description} | Pending inventory | Pending review | Not yet recorded | None approved |")
+    lines += [
+        "", "When a principle changes after this review, re-check applicability and record current evidence. Reopen this issue or link an owned follow-up for outstanding work; notify the central epic. A past review must not silently remain green against an obsolete rule.",
+        "", "## Findings / TODO / done ledger", "",
+        "| Finding / surface / reproduction | Principle and expected behavior | TODO / implementation or follow-up | Verified done evidence |",
+        "| --- | --- | --- | --- |",
+        "| Full review not started | UI-01–UI-09 | Inventory and review pending | None; no completed review claimed |",
+        "", "## Manual checks, decisions and closure evidence", "",
+        "- Manual work pending: safe actual-module walkthrough in both languages and realistic viewports, keyboard/focus, permissions/optional-module contexts, loading/error/empty states, edits/Save/Cancel/navigation/reload, tables/trees and progress.",
+        "- Decisions: none invented by this bootstrap. Record any product or policy choice with context and a recommendation; isolate independent implementation work from blocked decisions.",
+        "- Automated evidence: not yet recorded for this complete module review. Existing targeted fixes/tests may be linked as partial evidence only.",
+        "- Closure gate: verified inventory, complete current principle matrix, resolved required findings, owning EN/DE documentation and automated/manual evidence. Unimplemented placeholders remain pending until real surfaces can be reviewed or an explicit catalog/product decision changes scope.",
+    ]
+    return "\n".join(lines) + "\n"
+
+
+def result_record(scope: dict[str, Any], issue: dict[str, Any] | None, action: str) -> dict[str, Any]:
+    return {
+        "scope_id": scope["scope_id"], "name": scope["name"], "repository": scope["repository"],
+        "kind": scope["kind"], "ui_source_count": scope["ui_source_count"],
+        "number": issue.get("number") if issue else None,
+        "url": issue.get("html_url") if issue else None,
+        "state_at_verification": issue.get("state") if issue else None,
+        "operation": action,
+    }
+
+
+def render_links(records: list[dict[str, Any]]) -> str:
+    lines = ["### Implemented scopes — pending review", ""]
+    for placeholder in (False, True):
+        if placeholder:
+            lines += ["", "### Catalogued placeholders — pending readiness / future UI review", ""]
+        for item in records:
+            if (item["kind"] == "placeholder") == placeholder:
+                if not item["url"]:
+                    raise GiteaError("Cannot initialize an incomplete issue link inventory.")
+                lines.append(f"- [ ] [{item['name']}]({item['url']}) — `{item['scope_id']}` / `{item['repository']}`; pending.")
+    return "\n".join(lines)
+
+
+def initialized_epic_body(body: str, records: list[dict[str, Any]]) -> str:
+    if body.count(LIST_START) != 1 or body.count(LIST_END) != 1 or EPIC_MARKER not in body:
+        raise GiteaError("Epic managed-list markers are absent or ambiguous; preserve its body.")
+    before, tail = body.split(LIST_START, 1)
+    current, after = tail.split(LIST_END, 1)
+    if current.strip() != INITIAL_LIST:
+        if all(item["url"] and f"]({item['url']})" in current for item in records):
+            return body  # Never reset human checkboxes, evidence or subsequent edits.
+        raise GiteaError("Epic list was already edited; update missing links manually without replacing progress.")
+    return before + LIST_START + "\n" + render_links(records) + "\n" + LIST_END + after
+
+
+@contextmanager
+def ipv4_for_target(enabled: bool) -> Iterator[None]:
+    original = socket.getaddrinfo
+    def scoped(host: Any, port: Any, family: int = 0, type: int = 0, proto: int = 0, flags: int = 0) -> Any:
+        return original(host, port, socket.AF_INET if host == "git.add-ideas.de" else family, type, proto, flags)
+    if enabled:
+        socket.getaddrinfo = scoped
+    try:
+        yield
+    finally:
+        socket.getaddrinfo = original
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--env-file", type=Path)
+    parser.add_argument("--epic", type=int, required=True, help="Existing managed UI-review epic number in GovOPlaN/govoplan")
+    parser.add_argument("--apply", action="store_true", help="Create missing review issues; existing issues remain untouched")
+    parser.add_argument("--initialize-links", action="store_true", help="One-time initialization of the untouched epic module-list placeholder")
+    parser.add_argument("--ipv4", action="store_true", help="Prefer IPv4 only for git.add-ideas.de; keep HTTPS verification")
+    args = parser.parse_args()
+    if args.epic <= 0 or (args.initialize_links and not args.apply):
+        parser.error("A positive --epic is required; --initialize-links requires --apply.")
+    try:
+        catalog = json.loads((META_ROOT / "repositories.json").read_text(encoding="utf-8"))
+        workspace_root = Path(catalog["default_parent"])
+        scopes = build_scopes(catalog, workspace_root, extract_manifests(catalog, workspace_root))
+        load_dotenv(args.env_file)
+        token = require_token()
+        target = RepoTarget(BASE_URL, OWNER, "govoplan")
+        with ipv4_for_target(args.ipv4), GiteaClient(target, token) as central:
+            epic = central.request_json("GET", repo_path(OWNER, "govoplan", f"/issues/{args.epic}"))
+            if EPIC_MARKER not in (epic.get("body") or "") or epic.get("state") != "open":
+                raise GiteaError("Expected an open, managed UI-review epic; no child issues created.")
+            epic_url = f"{BASE_URL}/{OWNER}/govoplan/issues/{args.epic}"
+            org_labels = {item["name"]: item["id"] for item in central.paginate(org_path(OWNER, "/labels"))}
+
+            def reconcile(scope: dict[str, Any]) -> dict[str, Any]:
+                repo = scope["repository"]
+                with GiteaClient(RepoTarget(BASE_URL, OWNER, repo), token) as client:
+                    issues = client.paginate(repo_path(OWNER, repo, "/issues"), query={"state": "all", "type": "issues"})
+                    existing = find_existing(issues, scope["scope_id"], issue_title(scope))
+                    if existing:
+                        return result_record(scope, existing, "existing")
+                    if not args.apply:
+                        return result_record(scope, None, "would-create")
+                    labels = dict(org_labels)
+                    labels.update({item["name"]: item["id"] for item in client.paginate(repo_path(OWNER, repo, "/labels"))})
+                    desired = ["type/task", "area/webui", "area/docs", "priority/p2", f"module/{repo.removeprefix('govoplan-')}"]
+                    desired += ["status/triage"] if scope["kind"] == "placeholder" else ["status/ready", "codex/ready"]
+                    issue = client.request_json("POST", repo_path(OWNER, repo, "/issues"), body={
+                        "title": issue_title(scope), "body": issue_body(scope, epic_url),
+                        "labels": [labels[name] for name in desired if name in labels],
+                    })
+                    verified = client.request_json("GET", repo_path(OWNER, repo, f"/issues/{issue['number']}"))
+                    if marker(scope["scope_id"]) not in (verified.get("body") or "") or verified.get("state") != "open":
+                        raise GiteaError(f"New review issue verification failed for {scope['scope_id']}.")
+                    print(f"Created {repo}#{verified['number']} (pending)", file=sys.stderr, flush=True)
+                    return result_record(scope, verified, "created")
+
+            with ThreadPoolExecutor(max_workers=4) as executor:
+                records = list(executor.map(reconcile, scopes))
+            if args.initialize_links:
+                fresh = central.request_json("GET", repo_path(OWNER, "govoplan", f"/issues/{args.epic}"))
+                body = initialized_epic_body(fresh["body"], records)
+                if body != fresh["body"]:
+                    central.request_json("PATCH", repo_path(OWNER, "govoplan", f"/issues/{args.epic}"), body={"body": body})
+            summary = {
+                "schema_version": 1, "snapshot_purpose": "Issue discovery links; live Gitea issues own review state and evidence.",
+                "epic": {"repository": "govoplan", "number": args.epic, "url": epic_url},
+                "scope_count": len(scopes),
+                "manifest_modules": sum(scope["kind"] == "manifest" for scope in scopes),
+                "implemented_scopes": sum(scope["kind"] != "placeholder" for scope in scopes),
+                "catalogued_placeholders": sum(scope["kind"] == "placeholder" for scope in scopes),
+                "created": sum(item["operation"] == "created" for item in records),
+                "missing": sum(item["operation"] == "would-create" for item in records),
+                "issues": records,
+            }
+            print(json.dumps(summary, ensure_ascii=False, indent=2))
+        return 0
+    except (GiteaError, OSError, ValueError) as exc:
+        print(f"UI review program: {exc}", file=sys.stderr)
+        return 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())