feat(devkit): add resumable workspace automation and UI review tooling
Dependency Audit / dependency-audit (push) Successful in 1m45s
Deployment Installer / deployment-installer (push) Successful in 6s
Security Audit / security-audit (push) Successful in 11m30s

Verified with the coordinated workspace changes by devkit full run
2026-09-08T225814-186389-0000-3e3ed7cd (all seven phases passed).
This shared UI pass does not mark the individual module reviews complete.
This commit is contained in:
2026-09-09 02:03:17 +02:00
parent 14b19fbead
commit 2ffdb23f69
67 changed files with 17306 additions and 94 deletions
+270 -76
View File
@@ -1,13 +1,151 @@
#!/usr/bin/env bash
set -euo pipefail
# The JSON catalog is the only phase order/metadata authority. Check commands
# remain in the marked functions below; devkit invokes this script, not copies.
META_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
ROOT="${GOVOPLAN_CORE_ROOT:-$META_ROOT/../govoplan-core}"
FOCUSED_MODE=run
FOCUSED_PHASE=""
FOCUSED_JSON=0
focused_usage() {
echo "Usage: check-focused.sh [--phase ID] | --list-phases [--json]"
echo "Without --phase, run every canonical phase in catalog order (fail fast)."
}
while [[ $# -gt 0 ]]; do
case "$1" in
--phase)
if [[ "$FOCUSED_MODE" != run || -n "$FOCUSED_PHASE" || $# -lt 2 || -z "$2" || "$2" == --* ]]; then
echo "check-focused: --phase requires one ID and cannot be combined with listing." >&2
exit 2
fi
FOCUSED_PHASE="$2"
shift 2
;;
--list-phases)
if [[ "$FOCUSED_MODE" != run || -n "$FOCUSED_PHASE" ]]; then
echo "check-focused: duplicate or incompatible phase selection." >&2
exit 2
fi
FOCUSED_MODE=list
shift
;;
--json)
if [[ "$FOCUSED_JSON" == 1 ]]; then
echo "check-focused: duplicate --json." >&2
exit 2
fi
FOCUSED_JSON=1
shift
;;
--help|-h)
focused_usage
exit 0
;;
*)
echo "check-focused: unknown argument: $1" >&2
focused_usage >&2
exit 2
;;
esac
done
if [[ "$FOCUSED_JSON" == 1 && "$FOCUSED_MODE" != list ]]; then
echo "check-focused: --json is supported only with --list-phases." >&2
exit 2
fi
# Metadata-only operations need standard Python, not an installed product venv,
# Node, npm, a Core checkout, or any temporary/state directory.
FOCUSED_METADATA_PYTHON="$(command -v python3)" || {
echo "check-focused: Python 3 is required to read phase metadata." >&2
exit 127
}
FOCUSED_SELECTION="$(
PYTHONDONTWRITEBYTECODE=1 "$FOCUSED_METADATA_PYTHON" -I -S - \
"$META_ROOT/tools/checks/focused-phases.json" "$FOCUSED_MODE" "$FOCUSED_PHASE" "$FOCUSED_JSON" <<'PY'
import json
import os
from pathlib import Path
import re
import stat
import sys
try:
path = Path(sys.argv[1])
descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0))
with os.fdopen(descriptor, "rb") as handle:
status = os.fstat(handle.fileno())
if not stat.S_ISREG(status.st_mode) or status.st_size > 1024 * 1024:
raise ValueError("phase metadata must be a bounded regular file")
encoded = handle.read(1024 * 1024 + 1)
if len(encoded) > 1024 * 1024:
raise ValueError("phase metadata exceeds its size bound")
def unique(pairs):
result = {}
for key, value in pairs:
if key in result:
raise ValueError("duplicate phase metadata key")
result[key] = value
return result
catalog = json.loads(encoded, object_pairs_hook=unique)
if not isinstance(catalog, dict) or set(catalog) != {"schema_version", "phases"} or type(catalog["schema_version"]) is not int or catalog["schema_version"] != 1:
raise ValueError("unsupported phase metadata schema")
phases = catalog["phases"]
if not isinstance(phases, list) or not 1 <= len(phases) <= 32:
raise ValueError("phase catalog requires 132 definitions")
seen = set()
fields = {"id", "title", "cwd", "order_after", "depends_on", "resources", "outputs", "notes"}
for phase in phases:
if not isinstance(phase, dict) or set(phase) != fields:
raise ValueError("invalid phase metadata fields")
identity = phase["id"]
if not isinstance(identity, str) or not re.fullmatch(r"[a-z][a-z0-9-]{0,63}", identity) or identity in seen:
raise ValueError("invalid or duplicate phase ID")
if not isinstance(phase["title"], str) or not phase["title"].strip() or len(phase["title"]) > 256 or any(ord(char) < 32 for char in phase["title"]):
raise ValueError("invalid phase title")
if phase["cwd"] not in {"core", "meta", "core-webui", "access-webui"}:
raise ValueError("unsupported phase working directory")
for field in ("order_after", "depends_on", "resources", "outputs", "notes"):
values = phase[field]
if not isinstance(values, list) or len(values) > 64 or any(not isinstance(value, str) or not value or len(value) > 4096 or "\0" in value for value in values):
raise ValueError("invalid phase list: " + field)
if len(set(values)) != len(values):
raise ValueError("duplicate phase list value: " + field)
if (set(phase["order_after"]) | set(phase["depends_on"])) - seen:
raise ValueError("phase prerequisites must precede their consumer")
seen.add(identity)
mode, selected, as_json = sys.argv[2:]
if selected and selected not in seen:
raise ValueError("unknown phase: " + selected)
if mode == "list":
print(json.dumps(catalog, indent=2) if as_json == "1" else "\n".join(phase["id"] + "\t" + phase["title"] for phase in phases))
else:
print("\n".join(phase["id"] for phase in phases if not selected or phase["id"] == selected))
except (OSError, ValueError, TypeError, RecursionError) as exc:
print("check-focused: " + str(exc), file=sys.stderr)
raise SystemExit(2)
PY
)"
if [[ "$FOCUSED_MODE" == list ]]; then
printf '%s\n' "$FOCUSED_SELECTION"
exit 0
fi
focused_setup() {
WORKSPACE_ROOT="${GOVOPLAN_WORKSPACE_ROOT:-$(dirname "$META_ROOT")}"
export GOVOPLAN_WORKSPACE_ROOT="$WORKSPACE_ROOT"
ROOT="${GOVOPLAN_CORE_ROOT:-$WORKSPACE_ROOT/govoplan-core}"
ROOT="$(cd "$ROOT" && pwd)"
VENV_ROOT="${GOVOPLAN_VENV_ROOT:-$META_ROOT/.venv}"
PYTHON="${PYTHON:-$VENV_ROOT/bin/python}"
NODE="/home/zemion/.nvm/versions/node/v22.22.3/bin"
NPM="$NODE/npm"
NODE="$(command -v "${NODE:-node}")" || { echo "check-focused: Node is unavailable; run ./devkit doctor." >&2; exit 127; }
NPM="$(command -v "${NPM:-npm}")" || { echo "check-focused: npm is unavailable; run ./devkit doctor." >&2; exit 127; }
NODE_BIN="$(dirname "$NODE")"
WEBUI_BIN="$ROOT/webui/node_modules/.bin"
NPM_USERCONFIG="$(mktemp "${TMPDIR:-/tmp}/govoplan-npmrc.XXXXXXXX")"
@@ -19,7 +157,7 @@ trap 'rm -f "$NPM_USERCONFIG"' EXIT
exit 127
}
export PATH="$WEBUI_BIN:$NODE:$PATH"
export PATH="$WEBUI_BIN:$NODE_BIN:$PATH"
export NPM_CONFIG_USERCONFIG="$NPM_USERCONFIG"
export GOVOPLAN_NPM_USERCONFIG="$NPM_USERCONFIG"
unset npm_config_tmp NPM_CONFIG_TMP
@@ -27,28 +165,41 @@ unset npm_config_tmp NPM_CONFIG_TMP
# Validate the current sibling checkouts even when a newly added module has not
# yet been installed into an existing development virtualenv.
SOURCE_PYTHONPATH=""
for source_dir in "$META_ROOT"/../govoplan*/src; do
for source_dir in "$WORKSPACE_ROOT"/govoplan*/src; do
[ -d "$source_dir" ] || continue
SOURCE_PYTHONPATH="${SOURCE_PYTHONPATH:+$SOURCE_PYTHONPATH:}$source_dir"
done
export PYTHONPATH="${SOURCE_PYTHONPATH}${PYTHONPATH:+:$PYTHONPATH}"
}
focused_phase_preflight() {
# devkit-phase: preflight begin
cd "$ROOT"
GOVOPLAN_CORE_ROOT="$ROOT" PYTHON="$PYTHON" CHECK_TESTCLIENT_DEPRECATIONS=1 bash "$META_ROOT/tools/checks/check-dependency-hygiene.sh"
"$PYTHON" "$META_ROOT/tools/checks/check-contracts.py" --no-impact
PYTHONDONTWRITEBYTECODE=1 "$PYTHON" "$META_ROOT/tools/checks/check-manifest-shapes.py" --require-architecture
PYTHONDONTWRITEBYTECODE=1 "$PYTHON" "$META_ROOT/tools/checks/check-dsar-coverage.py"
"$NODE/node" "$META_ROOT/tests/test-jsx-value-imports.mjs"
"$NODE/node" "$META_ROOT/tools/checks/check-jsx-value-imports.mjs"
"$NODE/node" "$META_ROOT/../govoplan-files/webui/scripts/test-archive-client.mjs"
"$NODE" "$META_ROOT/tests/test-jsx-value-imports.mjs"
"$NODE" "$META_ROOT/tools/checks/check-jsx-value-imports.mjs"
"$NODE" "$META_ROOT/tests/test-heading-help.mjs"
"$NODE" "$META_ROOT/tools/checks/check-heading-help.mjs"
"$NODE" --test "$META_ROOT/tests/test-devkit-display-labels.mjs"
"$NODE" --test "$ROOT/webui/tests/component-test-runner.test.mjs"
"$NODE" "$WORKSPACE_ROOT/govoplan-files/webui/scripts/test-archive-client.mjs"
# devkit-phase: preflight end
}
focused_phase_tooling() {
# devkit-phase: tooling begin
cd "$META_ROOT"
"$PYTHON" tools/inventory/platform-interface-inventory.py --strict-declarations --strict-endpoints
"$PYTHON" tools/inventory/platform-interface-inventory.py --workspace-root "$WORKSPACE_ROOT" --strict-declarations --strict-endpoints
"$PYTHON" tools/repo/sync-module-package-workflows.py --check
"$PYTHON" tools/release/generate-developer-meta-package.py --check
"$PYTHON" tools/checks/check-webui-package-facades.py
"$PYTHON" -m unittest tests.test_webui_package_facades
"$PYTHON" -m pytest -q tests/test_ui_review_program.py "$META_ROOT"/tests/test_devkit_*.py
"$PYTHON" -m pytest -q tests/test_focused_phases.py
"$PYTHON" -m unittest tests.test_module_package_workflows tests.test_package_registry_release
"$PYTHON" -m unittest tests.test_deployment_installer tests.test_webui_release_dependency_retries
"$PYTHON" -m pytest -q tests/test_release_meta_source_tag.py tests/test_release_source_tag_batch.py tests/test_release_meta_preparation.py
@@ -59,14 +210,20 @@ cd "$META_ROOT"
"$PYTHON" -m unittest tests.test_configuration_package_artifacts
"$PYTHON" -m unittest tests.test_institutional_governance_journey
"$PYTHON" -m unittest tests.test_institutional_service_journey
# devkit-phase: tooling end
}
focused_phase_backend() {
# devkit-phase: backend begin
cd "$ROOT"
"$PYTHON" - <<'PY'
import ast
import pathlib
import os
import sys
repos_root = pathlib.Path("/mnt/DATA/git")
repos_root = pathlib.Path(os.environ["GOVOPLAN_WORKSPACE_ROOT"])
roots = [
repo / "src"
for repo in sorted(repos_root.glob("govoplan*"))
@@ -107,107 +264,127 @@ PY
"$PYTHON" -m unittest tests.test_ownership_history_migration tests.test_ownership tests.test_ownership_api
"$PYTHON" -m unittest tests.test_navigation_preferences tests.test_api_smoke.ApiSmokeTests.test_navigation_separator_layout_survives_system_tenant_and_personal_saves
"$PYTHON" -m pytest -q \
/mnt/DATA/git/govoplan-files/tests/test_managed_archives.py \
/mnt/DATA/git/govoplan-files/tests/test_archive_work.py \
/mnt/DATA/git/govoplan-files/tests/test_archive_staging.py \
/mnt/DATA/git/govoplan-files/tests/test_upload_response_batching.py \
/mnt/DATA/git/govoplan-files/tests/test_archive_performance.py
"${WORKSPACE_ROOT}/govoplan-files/tests/test_managed_archives.py" \
"${WORKSPACE_ROOT}/govoplan-files/tests/test_archive_work.py" \
"${WORKSPACE_ROOT}/govoplan-files/tests/test_archive_staging.py" \
"${WORKSPACE_ROOT}/govoplan-files/tests/test_upload_response_batching.py" \
"${WORKSPACE_ROOT}/govoplan-files/tests/test_archive_performance.py"
"$PYTHON" -m pytest -q \
/mnt/DATA/git/govoplan-files/tests/test_archive_workers.py \
/mnt/DATA/git/govoplan-files/tests/test_archive_inspection_bounds.py \
/mnt/DATA/git/govoplan-files/tests/test_archives.py
"$PYTHON" -m pytest -q /mnt/DATA/git/govoplan-access/tests/test_external_function_mapping_migration.py
"$PYTHON" -m pytest -q /mnt/DATA/git/govoplan-access/tests
"$PYTHON" -m pytest -q /mnt/DATA/git/govoplan-templates/tests
"$PYTHON" -m unittest discover -s /mnt/DATA/git/govoplan-connectors/tests
"$PYTHON" -m unittest discover -s /mnt/DATA/git/govoplan-datasources/tests
"$PYTHON" -m unittest discover -s /mnt/DATA/git/govoplan-dataflow/tests
"$PYTHON" -m unittest discover -s /mnt/DATA/git/govoplan-workflow-engine/tests
"$PYTHON" -m unittest discover -s /mnt/DATA/git/govoplan-workflow/tests
"$PYTHON" -m unittest discover -s /mnt/DATA/git/govoplan-views/tests
"$PYTHON" -m unittest discover -s /mnt/DATA/git/govoplan-quick-access/tests
"$PYTHON" -m unittest discover -s /mnt/DATA/git/govoplan-dashboard/tests
"$PYTHON" -m unittest discover -s /mnt/DATA/git/govoplan-postbox/tests
"$PYTHON" -m unittest discover -s /mnt/DATA/git/govoplan-portal/tests
"$PYTHON" -m unittest discover -s /mnt/DATA/git/govoplan-payments/tests
"$PYTHON" -m unittest discover -s /mnt/DATA/git/govoplan-forms/tests
"$PYTHON" -m unittest discover -s /mnt/DATA/git/govoplan-forms-runtime/tests
"$PYTHON" -m unittest discover -s /mnt/DATA/git/govoplan-cases/tests
"$PYTHON" -m unittest discover -s /mnt/DATA/git/govoplan-committee/tests
"$PYTHON" -m unittest discover -s /mnt/DATA/git/govoplan-voting/tests
"$PYTHON" -m unittest discover -s /mnt/DATA/git/govoplan-approvals/tests
"$PYTHON" -m unittest discover -s /mnt/DATA/git/govoplan-identity-trust/tests
"$PYTHON" -m unittest discover -s /mnt/DATA/git/govoplan-encryption/tests
"$PYTHON" -m pytest -q /mnt/DATA/git/govoplan-wiki/tests
"${WORKSPACE_ROOT}/govoplan-files/tests/test_archive_workers.py" \
"${WORKSPACE_ROOT}/govoplan-files/tests/test_archive_inspection_bounds.py" \
"${WORKSPACE_ROOT}/govoplan-files/tests/test_archives.py"
"$PYTHON" -m pytest -q "${WORKSPACE_ROOT}/govoplan-access/tests/test_external_function_mapping_migration.py"
"$PYTHON" -m pytest -q "${WORKSPACE_ROOT}/govoplan-access/tests"
"$PYTHON" -m pytest -q "${WORKSPACE_ROOT}/govoplan-templates/tests"
"$PYTHON" -m unittest discover -s "${WORKSPACE_ROOT}/govoplan-connectors/tests"
"$PYTHON" -m unittest discover -s "${WORKSPACE_ROOT}/govoplan-datasources/tests"
"$PYTHON" -m unittest discover -s "${WORKSPACE_ROOT}/govoplan-dataflow/tests"
"$PYTHON" -m unittest discover -s "${WORKSPACE_ROOT}/govoplan-workflow-engine/tests"
"$PYTHON" -m unittest discover -s "${WORKSPACE_ROOT}/govoplan-workflow/tests"
"$PYTHON" -m unittest discover -s "${WORKSPACE_ROOT}/govoplan-views/tests"
"$PYTHON" -m unittest discover -s "${WORKSPACE_ROOT}/govoplan-quick-access/tests"
"$PYTHON" -m unittest discover -s "${WORKSPACE_ROOT}/govoplan-dashboard/tests"
"$PYTHON" -m unittest discover -s "${WORKSPACE_ROOT}/govoplan-postbox/tests"
"$PYTHON" -m unittest discover -s "${WORKSPACE_ROOT}/govoplan-portal/tests"
"$PYTHON" -m unittest discover -s "${WORKSPACE_ROOT}/govoplan-payments/tests"
"$PYTHON" -m unittest discover -s "${WORKSPACE_ROOT}/govoplan-forms/tests"
"$PYTHON" -m unittest discover -s "${WORKSPACE_ROOT}/govoplan-forms-runtime/tests"
"$PYTHON" -m unittest discover -s "${WORKSPACE_ROOT}/govoplan-cases/tests"
"$PYTHON" -m unittest discover -s "${WORKSPACE_ROOT}/govoplan-committee/tests"
"$PYTHON" -m unittest discover -s "${WORKSPACE_ROOT}/govoplan-voting/tests"
"$PYTHON" -m unittest discover -s "${WORKSPACE_ROOT}/govoplan-approvals/tests"
"$PYTHON" -m unittest discover -s "${WORKSPACE_ROOT}/govoplan-identity-trust/tests"
"$PYTHON" -m unittest discover -s "${WORKSPACE_ROOT}/govoplan-encryption/tests"
"$PYTHON" -m pytest -q "${WORKSPACE_ROOT}/govoplan-wiki/tests"
"$PYTHON" -m pytest -q \
/mnt/DATA/git/govoplan-campaign/tests/test_approval_gate.py \
/mnt/DATA/git/govoplan-campaign/tests/test_editor_state_security.py \
/mnt/DATA/git/govoplan-campaign/tests/test_mail_profile_boundary.py \
/mnt/DATA/git/govoplan-campaign/tests/test_independent_configuration_repairs.py \
/mnt/DATA/git/govoplan-campaign/tests/test_incremental_review_persistence.py \
/mnt/DATA/git/govoplan-campaign/tests/test_reviewed_build_mock.py \
/mnt/DATA/git/govoplan-campaign/tests/test_delivery_policy_settings.py \
/mnt/DATA/git/govoplan-campaign/tests/test_synchronous_delivery_policy.py \
/mnt/DATA/git/govoplan-campaign/tests/test_workerless_recovery.py \
/mnt/DATA/git/govoplan-campaign/tests/test_imap_batch_integration.py \
/mnt/DATA/git/govoplan-campaign/tests/test_testbed_claim_recovery.py \
/mnt/DATA/git/govoplan-campaign/tests/test_campaign_optimistic_concurrency.py \
/mnt/DATA/git/govoplan-campaign/tests/test_archive_encryption_governance.py \
/mnt/DATA/git/govoplan-policy/tests/test_campaign_archive_encryption.py \
/mnt/DATA/git/govoplan-policy/tests/test_archive_encryption_api.py
"${WORKSPACE_ROOT}/govoplan-campaign/tests/test_approval_gate.py" \
"${WORKSPACE_ROOT}/govoplan-campaign/tests/test_editor_state_security.py" \
"${WORKSPACE_ROOT}/govoplan-campaign/tests/test_mail_profile_boundary.py" \
"${WORKSPACE_ROOT}/govoplan-campaign/tests/test_independent_configuration_repairs.py" \
"${WORKSPACE_ROOT}/govoplan-campaign/tests/test_incremental_review_persistence.py" \
"${WORKSPACE_ROOT}/govoplan-campaign/tests/test_reviewed_build_mock.py" \
"${WORKSPACE_ROOT}/govoplan-campaign/tests/test_delivery_policy_settings.py" \
"${WORKSPACE_ROOT}/govoplan-campaign/tests/test_synchronous_delivery_policy.py" \
"${WORKSPACE_ROOT}/govoplan-campaign/tests/test_workerless_recovery.py" \
"${WORKSPACE_ROOT}/govoplan-campaign/tests/test_imap_batch_integration.py" \
"${WORKSPACE_ROOT}/govoplan-campaign/tests/test_testbed_claim_recovery.py" \
"${WORKSPACE_ROOT}/govoplan-campaign/tests/test_campaign_optimistic_concurrency.py" \
"${WORKSPACE_ROOT}/govoplan-campaign/tests/test_archive_encryption_governance.py" \
"${WORKSPACE_ROOT}/govoplan-policy/tests/test_campaign_archive_encryption.py" \
"${WORKSPACE_ROOT}/govoplan-policy/tests/test_archive_encryption_api.py"
"$PYTHON" "$META_ROOT/tools/checks/check-datasource-composition.py"
"$PYTHON" "$META_ROOT/tools/checks/check-sanctions-screening-composition.py"
"$PYTHON" -m pytest -q /mnt/DATA/git/govoplan-mail/tests/test_campaign_protocol_authorization.py /mnt/DATA/git/govoplan-mail/tests/test_campaign_imap_batch.py
"$PYTHON" -m unittest discover -s /mnt/DATA/git/govoplan-mail/tests
"$PYTHON" -m pytest -q "${WORKSPACE_ROOT}/govoplan-mail/tests/test_campaign_protocol_authorization.py" "${WORKSPACE_ROOT}/govoplan-mail/tests/test_campaign_imap_batch.py"
"$PYTHON" -m unittest discover -s "${WORKSPACE_ROOT}/govoplan-mail/tests"
"$PYTHON" -m unittest tests.test_api_smoke.ApiSmokeTests.test_mailbox_message_listing_reports_total_count
"$PYTHON" -m unittest \
tests.test_api_smoke.ApiSmokeTests.test_managed_attachment_patterns_preview_build_and_mock_send \
tests.test_api_smoke.ApiSmokeTests.test_reports_and_job_review_are_scoped_to_the_selected_version \
tests.test_api_smoke.ApiSmokeTests.test_worker_loss_becomes_unknown_and_requires_reconciliation_before_retry
# devkit-phase: backend end
}
focused_phase_core_ui() {
# devkit-phase: core-ui begin
cd "$ROOT/webui"
"$NPM" run test:api-client-cache
"$NPM" run test:auth-action-state
"$NPM" run test:dependency-security
"$NPM" run test:layout-primitives
"$NPM" run test:mail-components
"$NPM" run test:components -- layout-primitives page-layout data-grid-actions mail-components
"$NODE" --test tests/breadcrumb-bar.test.mjs
"$NPM" run test:module-capabilities
"$NPM" run test:module-permutations
"$NPM" run test:conformance
# devkit-phase: core-ui end
}
cd /mnt/DATA/git/govoplan-access/webui
focused_phase_module_builds() {
# devkit-phase: module-builds begin
cd "$ROOT/webui"
"$NPM" run test:module-permutations
# devkit-phase: module-builds end
}
focused_phase_browser() {
# devkit-phase: browser begin
cd "$ROOT/webui"
"$NPM" run test:conformance
# devkit-phase: browser end
}
focused_phase_module_ui() {
# devkit-phase: module-ui begin
cd "${WORKSPACE_ROOT}/govoplan-access/webui"
"$NPM" run test:passwords
"$WEBUI_BIN/tsc" -p /mnt/DATA/git/govoplan-payments/webui/tsconfig.json
"$WEBUI_BIN/tsc" -p "${WORKSPACE_ROOT}/govoplan-payments/webui/tsconfig.json"
cd /mnt/DATA/git/govoplan-payments/webui
cd "${WORKSPACE_ROOT}/govoplan-payments/webui"
"$NPM" run test:interface-pattern
cd /mnt/DATA/git/govoplan-dataflow/webui
cd "${WORKSPACE_ROOT}/govoplan-dataflow/webui"
"$NPM" run test:structure
cd /mnt/DATA/git/govoplan-datasources/webui
cd "${WORKSPACE_ROOT}/govoplan-datasources/webui"
"$NPM" run typecheck
cd /mnt/DATA/git/govoplan-workflow/webui
cd "${WORKSPACE_ROOT}/govoplan-workflow/webui"
"$NPM" run typecheck
cd /mnt/DATA/git/govoplan-dashboard/webui
cd "${WORKSPACE_ROOT}/govoplan-dashboard/webui"
"$NPM" run test:dashboard-layout
cd /mnt/DATA/git/govoplan-approvals/webui
cd "${WORKSPACE_ROOT}/govoplan-approvals/webui"
"$NPM" run test:workspace-layout
cd /mnt/DATA/git/govoplan-postbox/webui
cd "${WORKSPACE_ROOT}/govoplan-postbox/webui"
"$NPM" run test:ui-structure
cd /mnt/DATA/git/govoplan-mail/webui
cd "${WORKSPACE_ROOT}/govoplan-mail/webui"
"$NPM" run test:mail-ui
cd /mnt/DATA/git/govoplan-files/webui
cd "${WORKSPACE_ROOT}/govoplan-files/webui"
"$NPM" run test:managed-archive
cd /mnt/DATA/git/govoplan-campaign/webui
cd "${WORKSPACE_ROOT}/govoplan-campaign/webui"
"$NPM" run test:policy-ui
"$NPM" run test:template-preview
"$NPM" run test:review-workflow
@@ -215,8 +392,25 @@ cd /mnt/DATA/git/govoplan-campaign/webui
"$NPM" run test:campaign-collaboration
"$NPM" run test:campaign-work
cd /mnt/DATA/git/govoplan-policy/webui
cd "${WORKSPACE_ROOT}/govoplan-policy/webui"
"$NPM" run test:archive-encryption
cd /mnt/DATA/git/govoplan-wiki/webui
cd "${WORKSPACE_ROOT}/govoplan-wiki/webui"
"$NPM" run test:interface-pattern
# devkit-phase: module-ui end
}
# Validate every selected implementation before setup can create a temp npmrc.
while IFS= read -r phase_id; do
if ! declare -F "focused_phase_${phase_id//-/_}" >/dev/null; then
echo "check-focused: missing phase implementation: $phase_id" >&2
exit 2
fi
done <<< "$FOCUSED_SELECTION"
while IFS= read -r phase_id; do
(
focused_setup
"focused_phase_${phase_id//-/_}"
)
done <<< "$FOCUSED_SELECTION"
+161
View File
@@ -0,0 +1,161 @@
#!/usr/bin/env node
/** UI-01: contextual documentation belongs beside text, never in action slots. */
import { existsSync, readFileSync, readdirSync } from "node:fs";
import { createRequire } from "node:module";
import { relative, resolve } from "node:path";
import { fileURLToPath } from "node:url";
const workspaceRoot = resolve(import.meta.dirname, "../../..");
const require = createRequire(resolve(workspaceRoot, "govoplan-core/webui/package.json"));
const ts = require("typescript");
const titleOwners = new Set(["PageLayout", "PageHeader", "PageTitle", "AdminPageLayout", "Card", "Dialog", "PageActionBar", "WorkspaceActionBar"]);
const interactiveOwners = new Set(["a", "button", "Button", "IconButton"]);
// Existing domain dialog adapter; its owning structural tests must retain
// forwarding to Core Dialog.titleHelp (not a module-local heading definition).
const domainTitleOwners = new Map([["FileDialog", "/govoplan-files/webui/src/"]]);
export function findDetachedDocumentation(sources) {
const files = new Map(sources.map(({ path, source }) => [resolve(path),
ts.createSourceFile(resolve(path), source, ts.ScriptTarget.Latest, true, ts.ScriptKind.TSX)]));
const options = { noEmit: true, noResolve: true, noLib: true, types: [], jsx: ts.JsxEmit.Preserve };
const host = ts.createCompilerHost(options);
host.getSourceFile = (path) => files.get(resolve(path));
const checker = ts.createProgram([...files.keys()], options, host).getTypeChecker();
const findings = [];
let links = 0;
for (const [path, source] of files) {
const identifiers = [];
const helpNodes = [];
function importedName(node) {
if (ts.isPropertyAccessExpression(node) && ts.isIdentifier(node.expression)) {
const namespace = checker.getSymbolAtLocation(node.expression)?.declarations?.find(ts.isNamespaceImport);
if (namespace) return node.name.text;
}
const declarations = checker.getSymbolAtLocation(node)?.declarations ?? [];
const declaration = declarations.find(ts.isImportSpecifier);
if (declaration) return (declaration.propertyName ?? declaration.name).text;
const defaultImport = declarations.find(ts.isImportClause);
if (defaultImport && ts.isStringLiteral(defaultImport.parent.moduleSpecifier)) {
const component = defaultImport.parent.moduleSpecifier.text.split("/").at(-1).replace(/\.[cm]?[jt]sx?$/, "");
if (component === "DocumentationHelpLink" || titleOwners.has(component) || component === "TextWithHelp" || interactiveOwners.has(component)) return component;
}
return node.getText(source);
}
function tag(node) {
const opening = ts.isJsxElement(node) ? node.openingElement : ts.isJsxSelfClosingElement(node) ? node : null;
return opening ? importedName(opening.tagName) : null;
}
function collect(node) {
if (ts.isIdentifier(node)) identifiers.push(node);
if ((ts.isJsxSelfClosingElement(node) || ts.isJsxElement(node)) && tag(node) === "DocumentationHelpLink") helpNodes.push(node);
ts.forEachChild(node, collect);
}
collect(source);
function staticallyHidden(opening) {
const hidden = opening.attributes.properties.find((attribute) => ts.isJsxAttribute(attribute) && attribute.name.getText(source) === "hidden");
if (!hidden) return false;
if (!hidden.initializer || ts.isStringLiteral(hidden.initializer)) return true;
return ts.isJsxExpression(hidden.initializer) && hidden.initializer.expression?.kind === ts.SyntaxKind.TrueKeyword;
}
// Reject text that is definitely absent while preserving dynamic translated
// titles and components whose rendered text cannot be established statically.
function emptyText(node, seen = new Set()) {
if (!node) return true;
if (ts.isJsxText(node)) return !node.getText(source).trim();
if (ts.isJsxExpression(node)) return emptyText(node.expression, seen);
if (ts.isStringLiteralLike(node)) return !node.text.trim();
if ([ts.SyntaxKind.NullKeyword, ts.SyntaxKind.FalseKeyword, ts.SyntaxKind.TrueKeyword].includes(node.kind) || ts.isVoidExpression(node)) return true;
if (ts.isParenthesizedExpression(node) || ts.isAsExpression(node) || ts.isSatisfiesExpression(node) || ts.isNonNullExpression(node)) return emptyText(node.expression, seen);
if (ts.isIdentifier(node)) {
const symbol = checker.getSymbolAtLocation(node);
const declaration = symbol?.declarations?.find(ts.isVariableDeclaration);
const immutable = declaration && ts.isVariableDeclarationList(declaration.parent) && Boolean(declaration.parent.flags & ts.NodeFlags.Const);
if (immutable && declaration.initializer && !seen.has(symbol)) return emptyText(declaration.initializer, new Set(seen).add(symbol));
return node.text === "undefined" && !symbol?.declarations?.length;
}
if (ts.isConditionalExpression(node)) return emptyText(node.whenTrue, seen) && emptyText(node.whenFalse, seen);
if (ts.isJsxFragment(node)) return node.children.every((child) => emptyText(child, seen));
if (ts.isArrayLiteralExpression(node)) return node.elements.every((child) => emptyText(child, seen));
if (ts.isJsxElement(node) || ts.isJsxSelfClosingElement(node)) {
const opening = ts.isJsxElement(node) ? node.openingElement : node;
if (staticallyHidden(opening)) return true;
if (/^[a-z]/.test(opening.tagName.getText(source))) return !ts.isJsxElement(node) || node.children.every((child) => emptyText(child, seen));
}
return false;
}
function isAnchored(node, seen = new Set()) {
// Check the complete rendered ancestry before returning at a recognized
// slot: the entire heading/text contract may itself be inside a button.
for (let parent = node.parent; parent; parent = parent.parent) {
if (interactiveOwners.has(tag(parent))) return false;
const opening = ts.isJsxElement(parent) ? parent.openingElement : ts.isJsxSelfClosingElement(parent) ? parent : null;
if (opening && staticallyHidden(opening)) return false;
}
for (let parent = node.parent; parent; parent = parent.parent) {
if (ts.isJsxAttribute(parent)) {
const owner = parent.parent.parent;
const name = importedName(owner.tagName);
const slot = parent.name.getText(source);
if (slot === "titleHelp" && (titleOwners.has(name) || (domainTitleOwners.has(name) && path.includes(domainTitleOwners.get(name))))) {
if (name === "PageTitle") {
const element = ts.isJsxOpeningElement(owner) ? owner.parent : null;
return Boolean(element?.children.some((child) => !emptyText(child)));
}
const title = owner.attributes.properties.find((attribute) => ts.isJsxAttribute(attribute) && attribute.name.getText(source) === "title");
return Boolean(title && !emptyText(title.initializer));
}
if (slot === "help" && name === "TextWithHelp") {
const element = ts.isJsxOpeningElement(owner) ? owner.parent : null;
return Boolean(element?.children.some((child) => !emptyText(child)));
}
return false;
}
if (ts.isVariableDeclaration(parent) && ts.isIdentifier(parent.name)) {
const symbol = checker.getSymbolAtLocation(parent.name);
if (!symbol || seen.has(symbol)) return false;
const next = new Set(seen).add(symbol);
const references = identifiers.filter((identifier) => identifier !== parent.name && checker.getSymbolAtLocation(identifier) === symbol);
return references.length > 0 && references.every((reference) => isAnchored(reference, next));
}
}
return false;
}
for (const node of helpNodes) {
links += 1;
// FieldLabel is the central label+book implementation; its browser/component
// contract verifies sibling text and prevents nested interactive controls.
if (path.endsWith("/govoplan-core/webui/src/components/help/FieldLabel.tsx")) continue;
if (!isAnchored(node)) {
const position = source.getLineAndCharacterOfPosition(node.getStart(source));
findings.push({ path, line: position.line + 1, column: position.character + 1 });
}
}
}
return { findings, links };
}
function sourceFiles(directory) {
return readdirSync(directory, { withFileTypes: true }).flatMap((entry) => {
const path = resolve(directory, entry.name);
return entry.isDirectory() ? sourceFiles(path) : entry.name.endsWith(".tsx") ? [path] : [];
});
}
export function checkWorkspace(root = workspaceRoot) {
const modules = readdirSync(root, { withFileTypes: true })
.filter((entry) => entry.isDirectory() && entry.name.startsWith("govoplan"))
.map((entry) => resolve(root, entry.name, "webui/src")).filter(existsSync);
const paths = modules.flatMap(sourceFiles);
const { findings, links } = findDetachedDocumentation(paths.map((path) => ({ path, source: readFileSync(path, "utf8") })));
for (const finding of findings) {
console.error(`${relative(root, finding.path)}:${finding.line}:${finding.column}: UI-01 documentation must use a heading's titleHelp or TextWithHelp beside visible text, not an action slot or detached row.`);
}
if (!findings.length) console.log(`Heading-help contract passed: ${links} documentation links in ${paths.length} TSX files across ${modules.length} WebUI modules.`);
return findings.length ? 1 : 0;
}
if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) process.exitCode = checkWorkspace();
@@ -4,6 +4,7 @@ set -euo pipefail
META_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
ROOT="${GOVOPLAN_CORE_ROOT:-$META_ROOT/../govoplan-core}"
ROOT="$(cd "$ROOT" && pwd)"
WORKSPACE_ROOT="${GOVOPLAN_WORKSPACE_ROOT:-$(dirname "$ROOT")}"
VENV_ROOT="${GOVOPLAN_VENV_ROOT:-$META_ROOT/.venv}"
PYTHON="${PYTHON:-$VENV_ROOT/bin/python}"
NPM="${NPM:-/home/zemion/.nvm/versions/node/v22.22.3/bin/npm}"
@@ -184,6 +185,7 @@ run_step "Validate installed module manifests and registry"
run_step "Validate platform interface and endpoint declarations"
"$PYTHON" "$META_ROOT/tools/inventory/platform-interface-inventory.py" \
--workspace-root "$WORKSPACE_ROOT" \
--strict-declarations \
--strict-endpoints
+142
View File
@@ -0,0 +1,142 @@
{
"schema_version": 1,
"phases": [
{
"id": "preflight",
"title": "Dependency and shared contract preflight",
"cwd": "core",
"order_after": [],
"depends_on": [],
"resources": [
"backend:test-state"
],
"outputs": [],
"notes": [
"Checks dependency hygiene, cross-module contracts, manifests and static WebUI conventions."
]
},
{
"id": "tooling",
"title": "Workspace inventories and development/release tooling tests",
"cwd": "meta",
"order_after": [
"preflight"
],
"depends_on": [],
"resources": [
"backend:test-state",
"artifact:platform-inventory"
],
"outputs": [
"{meta}/audit-reports/platform-inventory/platform-interface-inventory.json",
"{meta}/audit-reports/platform-inventory/platform-interface-inventory.md"
],
"notes": [
"Inventory reports are regenerated here; no later phase consumes them. Reused check evidence does not promise these reports still exist."
]
},
{
"id": "backend",
"title": "Workspace syntax, backend modules and integration checks",
"cwd": "core",
"order_after": [
"tooling"
],
"depends_on": [],
"resources": [
"backend:test-state"
],
"outputs": [],
"notes": [
"Tests may create their own temporary fixtures or test caches; no generated artifact is required by a later phase."
]
},
{
"id": "core-ui",
"title": "Core source tests and selected component contracts",
"cwd": "core-webui",
"order_after": [
"backend"
],
"depends_on": [],
"resources": [
"webui:govoplan-core"
],
"outputs": [
"{core}/webui/.module-test-build"
],
"notes": [
"The selected component batch creates and removes a private compilation directory. Module-capability tests rebuild .module-test-build themselves; later phases do not consume it."
]
},
{
"id": "module-builds",
"title": "Optional-module build permutations and bundle checks",
"cwd": "core-webui",
"order_after": [
"core-ui"
],
"depends_on": [],
"resources": [
"webui:govoplan-core"
],
"outputs": [
"{core}/webui/dist",
"{core}/webui/dist/module-permutation-bundle-metrics.json"
],
"notes": [
"Each permutation builds and reads its own fresh bundle metrics. Final dist and aggregate metrics persist, but browser/module-ui phases do not consume them. Reused check evidence is not artifact or deployment attestation."
]
},
{
"id": "browser",
"title": "Core browser conformance",
"cwd": "core-webui",
"order_after": [
"module-builds"
],
"depends_on": [],
"resources": [
"webui:govoplan-core",
"browser:chromium",
"port:4174"
],
"outputs": [
"{core}/webui/test-results",
"{core}/webui/node_modules/.vite/govoplan-conformance"
],
"notes": [
"Playwright starts its own Vite server from conformance sources with an isolated cache. It does not serve or require module-builds dist. The server is owned by the phase and stopped on completion."
]
},
{
"id": "module-ui",
"title": "Owning-module UI and type checks",
"cwd": "access-webui",
"order_after": [
"browser"
],
"depends_on": [],
"resources": [
"webui:govoplan-core",
"webui:govoplan-access",
"webui:govoplan-payments",
"webui:govoplan-dataflow",
"webui:govoplan-datasources",
"webui:govoplan-workflow",
"webui:govoplan-dashboard",
"webui:govoplan-approvals",
"webui:govoplan-postbox",
"webui:govoplan-mail",
"webui:govoplan-files",
"webui:govoplan-campaign",
"webui:govoplan-policy",
"webui:govoplan-wiki"
],
"outputs": [],
"notes": [
"The original 19-command tail retains each owning package's cwd. Its scripts prepare any private test output they need; no earlier phase artifact is a prerequisite."
]
}
]
}
+212
View File
@@ -0,0 +1,212 @@
#!/usr/bin/env node
// Complement the existing i18n-marker inventory with known plain-text display
// slots. Parse source only: never execute module registration/catalog code.
import { existsSync, readFileSync, readdirSync, realpathSync } from "node:fs";
import { createRequire } from "node:module";
import { dirname, join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
const displayProps = new Map([
...["PageLayout", "PageHeader", "PageTitle", "AdminPageLayout", "Card", "Dialog", "PageActionBar", "WorkspaceActionBar"].map((name) => [name, new Set(["title", "subtitle"])]),
["FieldLabel", new Set(["label"])], ["MetricCard", new Set(["label"])],
]);
const childOwners = new Set(["PageTitle", "TextWithHelp", "h1", "h2", "h3", "h4", "h5", "h6"]);
const unknown = Symbol("dynamic");
export function createReader(ts, allowedRoot) {
const sources = new Map();
function source(file) {
file = resolve(file);
if (file !== allowedRoot && !file.startsWith(`${resolve(allowedRoot)}/`)) return null;
if (!existsSync(file)) return null;
if (!realpathSync(file).startsWith(`${resolve(allowedRoot)}/`)) return null;
if (!sources.has(file)) sources.set(file, ts.createSourceFile(file, readFileSync(file, "utf8"), ts.ScriptTarget.Latest, true, file.endsWith(".tsx") ? ts.ScriptKind.TSX : ts.ScriptKind.TS));
return sources.get(file);
}
function unwrap(node) {
while (node && (ts.isAsExpression(node) || ts.isSatisfiesExpression(node) || ts.isParenthesizedExpression(node))) node = node.expression;
return node;
}
function imported(sf, identifier) {
for (const item of sf.statements) {
if (!ts.isImportDeclaration(item) || !item.importClause || !ts.isStringLiteral(item.moduleSpecifier)) continue;
const names = item.importClause.namedBindings;
if (names && ts.isNamedImports(names)) {
const match = names.elements.find((entry) => entry.name.text === identifier);
if (match) return { path: item.moduleSpecifier.text, name: match.propertyName?.text ?? match.name.text };
}
if (item.importClause.name?.text === identifier) return { path: item.moduleSpecifier.text, name: "default" };
}
return null;
}
function targetFile(sf, specifier) {
if (!specifier.startsWith(".")) return null;
const base = resolve(dirname(sf.fileName), specifier);
return [base, `${base}.ts`, `${base}.tsx`, join(base, "index.ts")].find((file) => existsSync(file) && /\.tsx?$/.test(file)) ?? null;
}
function declaration(sf, name) {
for (const statement of sf.statements) {
if (ts.isVariableStatement(statement) && statement.declarationList.flags & ts.NodeFlags.Const) {
const item = statement.declarationList.declarations.find((node) => ts.isIdentifier(node.name) && node.name.text === name);
if (item) return item.initializer;
}
if (name === "default" && ts.isExportAssignment(statement)) return statement.expression;
}
return null;
}
function value(node, sf, seen = new Set()) {
node = unwrap(node);
if (!node) return unknown;
if (ts.isStringLiteralLike(node)) return node.text;
if (ts.isIdentifier(node)) {
const key = `${sf.fileName}:${node.text}`;
if (seen.has(key)) return unknown;
const visited = new Set([...seen, key]);
const local = declaration(sf, node.text);
if (local) return value(local, sf, visited);
const external = imported(sf, node.text);
const target = external && targetFile(sf, external.path);
const loaded = target && source(target);
return loaded ? value(declaration(loaded, external.name), loaded, visited) : unknown;
}
if (ts.isObjectLiteralExpression(node)) {
const result = {};
for (const property of node.properties) {
if (ts.isSpreadAssignment(property)) {
const spread = value(property.expression, sf, seen);
if (spread !== unknown && spread && typeof spread === "object") Object.assign(result, spread);
else result.__dynamicSpread = true;
} else if (ts.isPropertyAssignment(property)) {
result[property.name.text ?? property.name.getText(sf)] = value(property.initializer, sf, seen);
} else if (ts.isShorthandPropertyAssignment(property)) result[property.name.text] = value(property.name, sf, seen);
}
return result;
}
return unknown;
}
return { source, value, declaration, imported, unwrap };
}
function componentName(ts, reader, sf, node) {
const written = node.getText(sf);
if (/^h[1-6]$/.test(written)) return written;
if (ts.isIdentifier(node)) {
const imported = reader.imported(sf, written);
if (imported?.name === "default") return imported.path.split("/").at(-1).replace(/\.[tj]sx?$/, "");
return imported?.name ?? written;
}
if (ts.isPropertyAccessExpression(node)) {
const owner = node.expression.getText(sf);
const namespace = sf.statements.find((item) => ts.isImportDeclaration(item) && item.importClause?.namedBindings && ts.isNamespaceImport(item.importClause.namedBindings) && item.importClause.namedBindings.name.text === owner);
if (namespace) return node.name.text;
}
return written;
}
function sourceFiles(directory) {
if (!existsSync(directory)) return [];
return readdirSync(directory, { withFileTypes: true }).flatMap((entry) => {
if (entry.name.startsWith(".") || entry.name === "node_modules") return [];
const file = join(directory, entry.name);
if (entry.isDirectory()) return sourceFiles(file);
return entry.isFile() && /\.[tj]sx?$/.test(file) ? [file] : [];
});
}
export function auditRepository(ts, repositoryRoot, coreRoot) {
const reader = createReader(ts, repositoryRoot);
const coreReader = createReader(ts, coreRoot);
const coreFile = coreReader.source(join(coreRoot, "webui/src/i18n/generatedTranslations.ts"));
const coreCatalog = coreFile ? coreReader.value(coreReader.declaration(coreFile, "generatedTranslations"), coreFile) : {};
const moduleFile = reader.source(join(repositoryRoot, "webui/src/module.ts"));
let moduleCatalog = {};
let registration = repositoryRoot === coreRoot ? "core-default" : "absent";
if (moduleFile) {
for (const statement of moduleFile.statements) {
if (!ts.isVariableStatement(statement) || !statement.modifiers?.some((item) => item.kind === ts.SyntaxKind.ExportKeyword)) continue;
for (const node of statement.declarationList.declarations) {
if (!node.initializer || !(/PlatformWebModule/.test(node.type?.getText(moduleFile) ?? "") || /Module$/.test(node.name.getText(moduleFile)))) continue;
const evaluated = reader.value(node.initializer, moduleFile);
if (evaluated && typeof evaluated === "object" && Object.hasOwn(evaluated, "translations")) {
moduleCatalog = evaluated.translations;
registration = moduleCatalog !== unknown && moduleCatalog && typeof moduleCatalog === "object" &&
!moduleCatalog.__dynamicSpread && !moduleCatalog.en?.__dynamicSpread && !moduleCatalog.de?.__dynamicSpread ? "registered" : "dynamic";
}
}
}
}
const findings = [], review = [], labels = [];
const add = (text, node, sf, slot) => {
const position = sf.getLineAndCharacterOfPosition(node.getStart(sf));
const location = { file: sf.fileName, line: position.line + 1, slot };
if (typeof text !== "string") {
review.push({ ...location, code: "dynamic-display-slot", message: "Runtime data or computed text: verify with the owning module/locale context." });
return;
}
text = text.replace(/\s+/g, " ").trim();
if (!text || text.startsWith("i18n:") || !/[\p{L}]/u.test(text)) return;
const missing = ["en", "de"].filter((locale) => {
const translated = moduleCatalog?.[locale]?.[text] ?? coreCatalog?.[locale]?.[text];
return typeof translated !== "string" || !translated.trim();
});
labels.push({ ...location, text, missing_locales: missing });
if (missing.length) {
const item = { ...location, code: "plain-label-missing-translation", text, missing_locales: missing, registration };
if (registration === "dynamic") review.push({ ...item, code: "dynamic-catalog-review" });
else findings.push(item);
}
};
for (const file of sourceFiles(join(repositoryRoot, "webui/src"))) {
if (file.includes("/i18n/")) continue;
const sf = reader.source(file);
function textChild(node, owner) {
if (ts.isJsxText(node)) add(node.text, node, sf, `${owner}.children`);
else if (ts.isJsxExpression(node)) { if (node.expression) add(reader.value(node.expression, sf), node, sf, `${owner}.children`); }
else if (ts.isJsxElement(node) || ts.isJsxFragment(node)) for (const child of node.children) textChild(child, owner);
}
function visit(node) {
if (ts.isJsxOpeningElement(node) || ts.isJsxSelfClosingElement(node)) {
const owner = componentName(ts, reader, sf, node.tagName);
for (const property of node.attributes.properties) {
if (!ts.isJsxAttribute(property) || !displayProps.get(owner)?.has(property.name.text) || !property.initializer) continue;
const expression = ts.isJsxExpression(property.initializer) ? property.initializer.expression : property.initializer;
add(reader.value(expression, sf), property, sf, `${owner}.${property.name.text}`);
}
}
if (ts.isJsxElement(node)) {
const owner = componentName(ts, reader, sf, node.openingElement.tagName);
if (childOwners.has(owner)) for (const child of node.children) textChild(child, owner);
}
ts.forEachChild(node, visit);
}
visit(sf);
}
const hasCatalog = sourceFiles(join(repositoryRoot, "webui/src/i18n")).some((file) => /Translations\.ts$/.test(file));
if (hasCatalog && registration === "absent") findings.push({ file: moduleFile?.fileName ?? join(repositoryRoot, "webui/src/module.ts"), line: 1, code: "catalog-not-registered", message: "Module-owned catalog exists but no static module translations registration was found." });
return { repository: repositoryRoot, registration, labels, findings, review };
}
if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
const args = process.argv.slice(2);
let workspace = resolve(dirname(fileURLToPath(import.meta.url)), "../../../..");
const repos = [];
for (let index = 0; index < args.length; index++) {
if (args[index] === "--workspace-root") workspace = resolve(args[++index]);
else if (args[index] === "--repo") repos.push(args[++index]);
else throw new Error(`Unknown argument: ${args[index]}`);
}
const core = join(workspace, "govoplan-core");
const require = createRequire(join(core, "webui/package.json"));
const ts = require("typescript");
const catalog = JSON.parse(readFileSync(join(workspace, "govoplan/repositories.json"), "utf8"));
const selected = catalog.repositories.filter((repo) => !repos.length || repos.includes(repo.name));
if (repos.some((name) => !selected.some((repo) => repo.name === name))) throw new Error("Unknown repository selection");
for (const repo of selected) {
if (typeof repo.path !== "string" || !resolve(workspace, repo.path).startsWith(`${workspace}/`)) throw new Error("Repository path escapes the workspace");
}
const results = selected.filter((repo) => existsSync(join(workspace, repo.path, "webui/src"))).map((repo) => auditRepository(ts, join(workspace, repo.path), core));
const findings = results.reduce((total, item) => total + item.findings.length, 0);
process.stdout.write(JSON.stringify({ schema_version: 1, results, finding_count: findings,
limitations: ["Known display slots only; this is not a complete UI or linguistic review.", "Runtime data, computed labels and dynamic registrations require manual review.", "Core defaults plus each owning module are checked; optional sibling catalogs cannot mask missing registration.", "Explicit i18n markers are checked by the existing platform interface inventory."] }) + "\n");
process.exitCode = findings ? 1 : 0;
}
+6
View File
@@ -0,0 +1,6 @@
#!/usr/bin/env python3
"""Command-line entry point; use the repository-root ./devkit launcher."""
from govoplan_devkit.cli import main
if __name__ == "__main__":
raise SystemExit(main())
+32
View File
@@ -0,0 +1,32 @@
{
"schema_version": 1,
"name": "Example Python project",
"repositories": [{"name": "app", "path": "."}],
"checks": [
{
"id": "whitespace",
"title": "Git whitespace check",
"argv": ["git", "diff", "--check"],
"cwd": ".",
"repos": ["app"],
"inputs": {"repos": ["app"]},
"timeout_seconds": 30
},
{
"id": "unit-tests",
"title": "Python unit tests",
"argv": ["{python}", "-m", "unittest", "discover", "-s", "tests"],
"cwd": ".",
"repos": ["app"],
"inputs": {"repos": ["app"]},
"after": ["whitespace"],
"resources": ["test-database"],
"timeout_seconds": 300
}
],
"profiles": {
"quick": ["whitespace", "unit-tests"],
"backend": ["unit-tests"],
"full": ["whitespace", "unit-tests"]
}
}
+3
View File
@@ -0,0 +1,3 @@
"""Small, deterministic development commands for people, CI and coding agents."""
__version__ = "1.0.0"
+566
View File
@@ -0,0 +1,566 @@
"""Test planning over existing checks; planning never executes a test or server."""
from __future__ import annotations
from pathlib import Path
from copy import deepcopy
from itertools import islice
import re
from .common import read_json
from .package_tests import declared_tests, discovered_sources, source_stage_id
PROFILES = ("quick", "ui", "backend", "full")
def stage(
identity: str,
title: str,
argv: list[str],
cwd: Path,
*,
reason: str,
deps: list[str] | None = None,
after: list[str] | None = None,
resources: list[str] | None = None,
timeout_seconds: int = 300,
) -> dict:
return {
"id": identity,
"title": title,
"argv": argv,
"cwd": str(cwd),
"deps": deps or [],
"after": after or [],
"resources": resources or [],
"timeout_seconds": timeout_seconds,
"reason": reason,
}
def _custom_stages(
project, workspace_root: Path, profile: str, selected, filtered: bool
) -> list[dict]:
# load_project validates every nested declaration before selection.
records = project.config.get("checks", [])
checks = {item["id"]: item for item in records}
profiles = project.config.get("profiles", {})
if profile not in profiles:
raise ValueError(
f"Project {project.name!r} does not declare profile {profile!r}"
)
selected_names = {repo.name for repo in selected}
wanted = set()
def include(identity: str, visiting: frozenset[str] = frozenset()) -> None:
if identity not in checks:
raise ValueError(f"Unknown check dependency: {identity}")
if identity in visiting:
raise ValueError(f"Cyclic check dependency: {identity}")
if identity in wanted:
return
for dependency in [
*checks[identity].get("deps", []),
*checks[identity].get("after", []),
]:
include(dependency, visiting | {identity})
wanted.add(identity)
for identity in profiles[profile]:
if identity not in checks:
raise ValueError(f"Unknown profile check: {identity}")
owned = set(checks[identity].get("repos", []))
if not filtered or not owned or selected_names & owned:
include(identity)
result = []
# A dependency may precede/follow its consumer in the configuration: the
# execution engine owns scheduling, not JSON declaration order.
for identity, item in checks.items():
if identity in wanted:
result.append(
stage(
identity,
item.get("title", identity),
list(item["argv"]),
workspace_root / item.get("cwd", "."),
deps=list(item.get("deps", [])),
after=list(item.get("after", [])),
resources=list(item.get("resources", [])),
timeout_seconds=item.get("timeout_seconds", 300),
reason=f"Project profile {profile}",
)
)
for field in ("inputs", "reuse"):
if field in item:
result[-1][field] = deepcopy(item[field])
return result
def focused_phases(meta: Path) -> list[dict]:
"""Read the same bounded, ordered phase metadata as the standalone gate."""
value = read_json(meta / "tools/checks/focused-phases.json", max_bytes=1024 * 1024)
if (
not isinstance(value, dict)
or set(value) != {"schema_version", "phases"}
or type(value["schema_version"]) is not int
or value["schema_version"] != 1
):
raise ValueError("Unsupported focused phase metadata schema")
phases = value["phases"]
if not isinstance(phases, list) or not 1 <= len(phases) <= 32:
raise ValueError("Focused phase catalog requires 132 definitions")
seen = set()
fields = {
"id",
"title",
"cwd",
"order_after",
"depends_on",
"resources",
"outputs",
"notes",
}
for phase in phases:
if not isinstance(phase, dict) or set(phase) != fields:
raise ValueError("Invalid focused phase metadata fields")
identity = phase["id"]
if (
not isinstance(identity, str)
or not re.fullmatch(r"[a-z][a-z0-9-]{0,63}", identity)
or identity in seen
):
raise ValueError("Invalid or duplicate focused phase ID")
if (
not isinstance(phase["title"], str)
or not phase["title"].strip()
or len(phase["title"]) > 256
or any(ord(char) < 32 for char in phase["title"])
):
raise ValueError("Invalid focused phase title")
if not isinstance(phase["cwd"], str) or phase["cwd"] not in {
"core",
"meta",
"core-webui",
"access-webui",
}:
raise ValueError("Unsupported focused phase working directory")
for field in ("order_after", "depends_on", "resources", "outputs", "notes"):
values = phase[field]
if (
not isinstance(values, list)
or len(values) > 64
or any(
not isinstance(item, str)
or not item
or len(item) > 4096
or "\0" in item
for item in values
)
or len(set(values)) != len(values)
):
raise ValueError("Invalid focused phase list: " + field)
if (set(phase["order_after"]) | set(phase["depends_on"])) - seen:
raise ValueError("Focused prerequisites must precede their consumer")
seen.add(identity)
return phases
def focused_phase_bodies(text: str, phases: list[dict]) -> dict[str, str]:
"""Only exact top-level registered wrappers are authoritative phase bodies.
Do not extract lookalike markers from heredocs, conditionals or unrelated
shell functions. This recognizes the maintained wrapper convention, not
arbitrary executable shell semantics.
"""
registered = {
"focused_phase_" + phase["id"].replace("-", "_"): phase["id"]
for phase in phases
}
lines, bodies, index, depth, heredoc = text.splitlines(), {}, 0, 0, None
while index < len(lines):
line = lines[index]
stripped = line.strip()
if heredoc is not None:
if stripped == heredoc:
heredoc = None
index += 1
continue
match = re.search(r"<<-?\s*['\"]?([A-Za-z_][A-Za-z0-9_]*)['\"]?", line)
if match:
heredoc = match[1]
index += 1
continue
function = re.fullmatch(r"(focused_phase_[a-z0-9_]+)\(\) \{", line)
if depth == 0 and function and function[1] in registered:
identity = registered[function[1]]
if (
identity in bodies
or index + 1 >= len(lines)
or lines[index + 1] != f"# devkit-phase: {identity} begin"
):
raise ValueError("Invalid or duplicate focused phase wrapper")
end = index + 2
while end < len(lines) and lines[end] != f"# devkit-phase: {identity} end":
end += 1
if end + 1 >= len(lines) or lines[end + 1] != "}":
raise ValueError("Unclosed focused phase wrapper")
bodies[identity] = "\n".join(lines[index + 2 : end]) + "\n"
index = end + 2
continue
if re.match(
r"(?:if|for|while|until|case|select)\b|(?:function\s+\w+|\w+\s*\(\s*\))",
stripped,
):
depth += 1
elif re.match(r"(?:fi|done|esac)\b|^}\s*;?$", stripped):
depth = max(0, depth - 1)
index += 1
if set(bodies) != {phase["id"] for phase in phases}:
raise ValueError("Focused phase metadata and marked implementations differ")
return {phase["id"]: bodies[phase["id"]] for phase in phases}
def _undeclared_source_note(project, workspace_root: Path) -> str | None:
"""Directory discovery is wider than registered Git input ownership."""
registered_paths = {repo.path.resolve() for repo in project.repositories}
children = list(islice(workspace_root.iterdir(), 4097))
if len(children) > 4096:
return "Workspace discovery exceeds its bounded ownership audit; native reuse is disabled."
for child in children:
if not child.name.startswith("govoplan") or child.resolve() in registered_paths:
continue
if any(
(child / name).exists() or (child / name).is_symlink()
for name in ("src", "webui")
):
return "Unregistered sibling src/WebUI inputs may be consumed by workspace discovery or PYTHONPATH; native reuse is disabled until their repository ownership is declared."
return None
def _apply_undeclared_source_note(checks: list[dict], note: str | None) -> None:
if note:
for check in checks:
check["reuse"] = "never"
check.setdefault("coverage_notes", []).append(note)
def _focused_ui_inputs(project) -> tuple[list[str] | None, str]:
names = {repo.name for repo in project.repositories}
if not {"govoplan", "govoplan-core"} <= names:
return None, "Missing Core/Meta ownership; input scope stays workspace-wide."
selected = {"govoplan", "govoplan-core"}
for repo in project.repositories:
root, webui = repo.path, repo.path / "webui"
if (
not root.is_dir()
or root.is_symlink()
or webui.is_symlink()
or webui.exists()
and not webui.is_dir()
):
return (
None,
"Missing or ambiguous repository/WebUI layout; input scope stays workspace-wide.",
)
if webui.is_dir():
selected.add(repo.name)
return (
sorted(selected),
"UI input scope includes whole Core/Meta and every registered WebUI repository, including helpers/configuration; it is not per-file dependency inference.",
)
def _apply_ui_inputs(check: dict, scope: tuple[list[str] | None, str]) -> None:
names, note = scope
if names is not None:
check["inputs"] = {"repos": list(names)}
check.setdefault("coverage_notes", []).append(note)
def _full_stages(project, workspace_root: Path, meta: Path, core: Path) -> list[dict]:
phases = focused_phases(meta)
ui_scope = _focused_ui_inputs(project)
directories = {
"core": core,
"meta": meta,
"core-webui": core / "webui",
"access-webui": workspace_root / "govoplan-access/webui",
}
checks = []
for phase in phases:
check = stage(
"focused." + phase["id"],
phase["title"],
[
"bash",
str(meta / "tools/checks/check-focused.sh"),
"--phase",
phase["id"],
],
directories[phase["cwd"]],
reason="Full retains every canonical phase in order; repository filters never narrow the required gate.",
deps=["focused." + value for value in phase["depends_on"]],
after=["focused." + value for value in phase["order_after"]],
resources=list(dict.fromkeys(["workspace:focused", *phase["resources"]])),
timeout_seconds=14400,
)
check["coverage_notes"] = list(phase["notes"])
check["phase_outputs"] = list(phase["outputs"])
if phase["id"] in {"core-ui", "module-builds", "browser", "module-ui"}:
check["resources"] = list(
dict.fromkeys(
[
*check["resources"],
*[f"webui:{repo.name}" for repo in project.repositories],
]
)
)
_apply_ui_inputs(check, ui_scope)
else:
check["coverage_notes"].append(
"Cross-module backend/tooling checks retain conservative whole-workspace inputs."
)
checks.append(check)
_apply_undeclared_source_note(
checks, _undeclared_source_note(project, workspace_root)
)
return checks
def _expanded_repositories(project, selected, *, changed: bool):
"""Conservative shared changes; declared interface consumers otherwise.
This is a selection aid, not a claim of exhaustive runtime dependency
analysis. The full profile always retains the canonical workspace gate.
"""
if not changed or not selected:
return selected, "explicit selection" if selected else "no changed repositories"
names = {repo.name for repo in selected}
if names & {"govoplan", "govoplan-core"}:
return list(
project.repositories
), "Core/Meta changed; all registered consumers conservatively selected"
# Reuse the release contract parser without importing module application
# code. If available declarations cannot be parsed, broaden selection.
import sys
meta = next(
(repo.path for repo in project.repositories if repo.name == "govoplan"), None
)
if meta is None:
return selected, "changed repositories; no GovOPlaN contract catalog"
release = meta / "tools" / "release"
if not (release / "govoplan_release" / "contracts.py").is_file():
return selected, "changed repositories; contract parser unavailable"
sys.path.insert(0, str(release))
try:
from govoplan_release.contracts import parse_manifest_contract
contracts = []
for repo in project.repositories:
for manifest in sorted((repo.path / "src").glob("*/backend/manifest.py")):
parsed = parse_manifest_contract(manifest, repo_name=repo.name)
if parsed is None:
return list(
project.repositories
), "unresolved manifest contract; conservative workspace selection"
contracts.append(parsed)
while True:
providers = {
item.name
for contract in contracts
if contract.repo in names
for item in contract.provides_interfaces
}
consumers = {
contract.repo
for contract in contracts
if any(item.name in providers for item in contract.requires_interfaces)
}
added = consumers - names
if not added:
break
names.update(added)
except (ImportError, AttributeError, OSError, SyntaxError, ValueError):
return list(
project.repositories
), "contract analysis unavailable; conservative workspace selection"
finally:
sys.path.remove(str(release))
return [
repo for repo in project.repositories if repo.name in names
], "changed repositories plus declared interface consumers"
def _module_ui_plan(repo, *, reason: str) -> tuple[list[dict], list[str]]:
"""Use package-owned direct Node test metadata, excluding shell/build chains.
Source structural scripts are also discoverable by the existing established
names. Unknown shell commands are deliberately not guessed or rewritten.
"""
webui = repo.path / "webui"
package_path = webui / "package.json"
if not package_path.is_file():
return [], []
scripts: dict[tuple[str, ...], list[str]] = {}
omitted = []
for item in [*declared_tests(repo, package_path), *discovered_sources(repo)]:
if item["component_suite"] is not None:
omitted.append(
f"{repo.name} {item['name']}: only covered by the explicit UI component batch; quick does not compile components"
)
elif item["name"] in {"test:module-permutations", "test:vite-cache-isolation"}:
omitted.append(
f"{repo.name} {item['name']}: separate environment/permutation verification, not run by this scoped profile"
)
elif item["_argv"]:
scripts[tuple(item["_argv"])] = item["_argv"]
else:
omitted.append(f"{repo.name} {item['name']}: {item['reason']}")
return [
stage(
source_stage_id(repo.name, argv),
f"{repo.name}: {Path(argv[-1]).stem}",
argv,
webui,
reason=reason,
resources=[f"webui:{repo.name}"],
)
for _, argv in sorted(scripts.items())
], omitted
def module_ui_stages(repo, *, reason: str) -> list[dict]:
return _module_ui_plan(repo, reason=reason)[0]
def build_stages(
workspace_root: Path,
profile: str,
repos: list[str],
changed: bool,
project: Path | None = None,
) -> list[dict]:
from .workspace import load_project, selected_repositories
if profile not in PROFILES:
raise ValueError(f"Unknown check profile: {profile}")
workspace_root = workspace_root.resolve()
loaded = load_project(workspace_root, project)
selected = selected_repositories(loaded, repos, changed=changed)
if project is not None:
return _custom_stages(
loaded, workspace_root, profile, selected, bool(repos or changed)
)
selected, reason = _expanded_repositories(loaded, selected, changed=changed)
mapping = {repo.name: repo.path for repo in loaded.repositories}
meta = mapping.get("govoplan", workspace_root / "govoplan")
core = mapping.get("govoplan-core", workspace_root / "govoplan-core")
if profile == "full":
return _full_stages(loaded, workspace_root, meta, core)
if changed and not selected:
return []
checks = []
for identity, command in (
(
"contracts",
[
"{python}",
str(meta / "tools/checks/check-contracts.py"),
"--workspace-root",
str(workspace_root),
"--no-impact",
],
),
(
"manifests",
[
"{python}",
str(meta / "tools/checks/check-manifest-shapes.py"),
"--workspace-root",
str(workspace_root),
"--require-architecture",
],
),
):
checks.append(
stage(
identity,
f"Workspace {identity}",
command,
meta,
reason="Shared manifest/interface invariants",
timeout_seconds=600,
)
)
if profile in {"quick", "ui"}:
ui_scope = _focused_ui_inputs(loaded)
coverage_notes = [
"Scoped source/component checks are not a complete module review or the full focused gate."
]
for identity in ("jsx-value-imports", "heading-help"):
checks.append(
stage(
identity,
f"Shared {identity} contract",
["{node}", str(meta / f"tools/checks/check-{identity}.mjs")],
meta,
reason="Existing source-only cross-module guard",
)
)
_apply_ui_inputs(checks[-1], ui_scope)
for repo in selected:
stages, omissions = _module_ui_plan(repo, reason=reason)
checks.extend(stages)
coverage_notes.extend(omissions)
checks[0]["coverage_notes"] = coverage_notes
if profile == "ui":
checks.append(
stage(
"core.component-batch",
"Core component suites (compile once)",
["{node}", str(core / "webui/scripts/run-component-tests.mjs")],
core / "webui",
reason="Shared components affect every UI consumer",
timeout_seconds=900,
)
)
_apply_ui_inputs(checks[-1], ui_scope)
if profile == "backend":
for repo in selected:
if (repo.path / "tests").is_dir():
checks.append(
stage(
f"{repo.name}.backend",
f"{repo.name} backend tests",
["{python}", "-m", "pytest", "-q", str(repo.path / "tests")],
repo.path,
reason=reason,
resources=["backend:test-state"],
timeout_seconds=1800,
)
)
_apply_undeclared_source_note(
checks, _undeclared_source_note(loaded, workspace_root)
)
return checks
def build_coverage(
workspace_root: Path,
profile: str,
repos: list[str],
changed: bool,
project: Path | None = None,
*,
stages: list[dict] | None = None,
) -> dict:
"""Read-only suite inventory; prebuilt stages avoid repeating selection queries."""
from .coverage import coverage_inventory
if profile not in PROFILES:
raise ValueError(f"Unknown check profile: {profile}")
if stages is None:
stages = build_stages(workspace_root, profile, repos, changed, project)
return coverage_inventory(workspace_root, profile, project, stages)
+244
View File
@@ -0,0 +1,244 @@
"""Independently verified check results; never an artifact/build-output cache."""
from __future__ import annotations
from copy import deepcopy
import threading
import re
from .common import digest, now
from .inputs import InputSnapshotter, FINGERPRINT_VERSION
CHECKPOINT_VERSION = 1
def validate_checkpoint_receipt(receipt):
"""New checkpoints are explicit; legacy status alone never certifies a phase."""
version = receipt.get("fingerprint_version")
if version is None:
return
if version != FINGERPRINT_VERSION:
raise ValueError("Unsupported check fingerprint version")
for stage in receipt["stages"]:
verified = stage.get("checkpoint_verified")
if type(verified) is not bool:
raise ValueError("Stage checkpoint verification must be boolean")
if stage["status"] == "passed" and not verified:
raise ValueError(
"Passing stage requires an independently verified checkpoint"
)
if not verified:
continue
if (
stage["status"] != "passed"
or stage.get("checkpoint_version") != CHECKPOINT_VERSION
):
raise ValueError("Invalid verified checkpoint state or version")
for field in (
"cache_key",
"input_fingerprint",
"stage_plan_fingerprint",
"log_sha256",
):
if not isinstance(stage.get(field), str) or not re.fullmatch(
r"[a-f0-9]{64}", stage[field]
):
raise ValueError(
"Verified checkpoint requires bounded content identities"
)
if (
not isinstance(stage.get("checkpoint_at"), str)
or not stage["checkpoint_at"]
):
raise ValueError("Verified checkpoint requires its recording time")
scope = stage.get("input_scope")
if (
not isinstance(scope, dict)
or scope.get("version") != 1
or scope.get("kind") not in {"workspace", "repositories"}
or type(scope.get("declared")) is not bool
):
raise ValueError("Verified checkpoint requires a versioned input scope")
names = scope.get("repos")
if (
not isinstance(names, list)
or not 1 <= len(names) <= 256
or any(not isinstance(name, str) for name in names)
or len(set(names)) != len(names)
):
raise ValueError(
"Verified checkpoint requires bounded repository identities"
)
class Checkpoints:
def __init__(self, project, workspace, plan, environment_probe, cancelled):
self.scanner = InputSnapshotter(project, workspace_root=workspace)
self.plan = {stage["id"]: stage for stage in plan}
self.environment_probe = environment_probe
self.cancelled = cancelled
self.lock = threading.RLock()
self.initial = None
self.environment = None
def source(self, stages=None):
with self.lock:
return self.scanner.snapshot(
list(self.plan.values()) if stages is None else stages
)
def probe_environment(self):
value = self.environment_probe()
self.check_cancelled()
return value
def check_cancelled(self):
if self.cancelled.is_set():
raise InterruptedError("Check cancelled during input verification")
def initialize(self):
self.initial = self.source()
self.check_cancelled()
self.environment = self.probe_environment()
return self.initial, self.environment
def closure(self, stage):
selected = {}
def include(item):
if item["id"] in selected:
return
selected[item["id"]] = item
for identity in item["deps"]:
include(self.plan[identity])
include(self.plan[stage["id"]])
return [selected[key] for key in sorted(selected)]
def identity(self, stage):
with self.lock:
closure = self.closure(stage)
snapshot = self.source(closure)
self.check_cancelled()
environment = self.probe_environment()
own = snapshot["stages"][stage["id"]]
key = digest(
{
"checkpoint_version": CHECKPOINT_VERSION,
"inputs": {
identity: value["fingerprint"]
for identity, value in snapshot["stages"].items()
},
"environment": environment,
}
)
return {
"checkpoint_version": CHECKPOINT_VERSION,
"cache_key": key,
"input_fingerprint": own["fingerprint"],
"input_scope": own["scope"],
"stage_plan_fingerprint": own["plan_fingerprint"],
"stage_environment_fingerprint": environment,
"dependency_input_fingerprints": {
item["id"]: snapshot["stages"][item["id"]]["fingerprint"]
for item in closure
if item["id"] != stage["id"]
},
}
def prepare(self, stage, prior, allow_reuse, verify_log):
before = self.identity(stage)
reason = "No previous verified checkpoint"
if stage.get("reuse", "verified") == "never":
reason = (
"This stage explicitly disables reuse (outputs/setup must be recreated)"
)
elif not allow_reuse:
reason = "A data dependency ran again; its consumers must run again"
elif (
prior
and prior.get("status") == "passed"
and prior.get("checkpoint_verified") is True
and prior.get("checkpoint_version") == CHECKPOINT_VERSION
):
if prior.get("cache_key") == before["cache_key"]:
# Receipt command text is never executed. Only the freshly planned
# stage runs; a cached log must independently match its content hash.
verify_log(prior)
result = {
**before,
**{
key: deepcopy(prior[key])
for key in (
"status",
"exit_code",
"duration_seconds",
"log_path",
"log_sha256",
"output_truncated",
"omitted_output_bytes",
"checkpoint_at",
)
if key in prior
},
}
result.update(
checkpoint_verified=True,
reuse_reason="Verified checkpoint matches current inputs, command, dependencies and environment",
)
return before, result
reason = (
"Declared inputs, command, dependency inputs or environment changed"
)
before["reuse_reason"] = reason
return before, None
def finish(self, stage, before, result):
result = {**result, **before, "checkpoint_verified": False}
if result["status"] != "passed" or result.get("exit_code") != 0:
return result
after = self.identity(stage)
if before["cache_key"] != after["cache_key"]:
result.update(
status="stale",
error="Stage inputs or environment changed during execution; no reusable checkpoint was recorded",
)
else:
result.update(checkpoint_verified=True, checkpoint_at=now())
return result
def finalize(self, stages):
final = self.source()
self.check_cancelled()
environment = self.probe_environment()
valid = (
final["observed_source_fingerprint"]
== self.initial["observed_source_fingerprint"]
and environment == self.environment
)
for stage in stages:
if stage["status"] != "passed":
valid = valid and stage["status"] != "stale"
continue
closure = self.closure(self.plan[stage["id"]])
key = digest(
{
"checkpoint_version": CHECKPOINT_VERSION,
"inputs": {
item["id"]: final["stages"][item["id"]]["fingerprint"]
for item in closure
},
"environment": environment,
}
)
if (
stage.get("checkpoint_verified") is not True
or stage.get("cache_key") != key
):
stage.update(
status="stale",
checkpoint_verified=False,
error="Final inputs no longer match this checkpoint",
)
valid = False
return valid, final, environment
+212
View File
@@ -0,0 +1,212 @@
"""A compact command catalog over maintained project tools."""
from __future__ import annotations
import argparse
import importlib
import json
from pathlib import Path
import sys
from . import __version__
from .common import META_ROOT, redact, safe_output
COMMAND_MODULES = (
"context",
"doctor",
"runner",
"review",
"docs",
"issues",
"release",
"maintenance",
)
def main(argv: list[str] | None = None) -> int:
argv = sys.argv[1:] if argv is None else argv
common = argparse.ArgumentParser(add_help=False, allow_abbrev=False)
common.add_argument(
"--workspace-root",
type=Path,
default=META_ROOT.parent,
help="Directory containing registered repositories",
)
common.add_argument(
"--project", type=Path, help="Explicit trusted portable-project JSON manifest"
)
common.add_argument(
"--state-dir",
type=Path,
help="Private evidence-state base (scoped again by workspace)",
)
common.add_argument("--format", choices=("summary", "json"), default="summary")
common.add_argument(
"--quiet",
action="store_true",
help="Suppress live progress on stderr; retain the final result",
)
common.add_argument(
"--json",
dest="format",
action="store_const",
const="json",
help="Alias for --format json",
)
parser = argparse.ArgumentParser(
prog="devkit",
description="Deterministic development workflows. Read-only previews by default for remote writes.",
parents=[common],
allow_abbrev=False,
)
parser.add_argument("--version", action="version", version=f"devkit {__version__}")
subparsers = parser.add_subparsers(dest="command", required=True)
for name in COMMAND_MODULES:
importlib.import_module("." + name, __package__).register(subparsers)
catalog = subparsers.add_parser("commands", help="Show the compact command catalog")
def commands(_):
entries = [
{
"command": "context",
"purpose": "Offline repository changes, ownership and instructions",
"effects": "read only",
},
{
"command": "doctor",
"purpose": "Local tool/dependency preflight and repair guidance",
"effects": "read only",
},
{
"command": "check",
"purpose": "Registered verification profiles with logs and source-bound receipts",
"effects": "tests/builds; --dry-run previews",
},
{
"command": "runs / latest / status / summary / logs",
"purpose": "Find runs and read progress, compact results and bounded live/final logs",
"effects": "read only",
},
{
"command": "coverage",
"purpose": "Explain declared suite coverage and exclusions for a check profile",
"effects": "read only",
},
{
"command": "resume / recover",
"purpose": "Resume verified identical work or recover an abandoned check run",
"effects": "local checks/state only",
},
{
"command": "review",
"purpose": "Module UI-review inventory and manual evidence checklist",
"effects": "local bundle only",
},
{
"command": "docs",
"purpose": "Existing documentation and translation audits",
"effects": "local checks/evidence",
},
{
"command": "issues",
"purpose": "Preview and explicitly publish deduplicated issue evidence",
"effects": "remote only with --apply",
},
{
"command": "release",
"purpose": "Existing durable release lifecycle, receipts and confirmations",
"effects": "explicit --apply and step confirmation",
},
{
"command": "git",
"purpose": "Frozen explicit-path commit and branch-push maintenance",
"effects": "explicit --apply; no bulk staging, force or tags",
},
]
return {
"commands": entries,
"summary": [
f"{item['command']}: {item['purpose']} ({item['effects']})"
for item in entries
],
}
catalog.set_defaults(handler=commands)
# Global flags work before or after the command, without copying defaults to every parser.
global_args, remaining = common.parse_known_args(argv)
args = parser.parse_args(remaining, namespace=global_args)
args.workspace_root = args.workspace_root.expanduser().resolve()
if args.project:
args.project = args.project.expanduser().absolute()
def progress(event):
if args.quiet:
return
if args.format == "json":
print(
json.dumps(safe_output(event), sort_keys=True, allow_nan=False),
file=sys.stderr,
flush=True,
)
else:
counts = event["counts"]
completed = sum(
count
for state, count in counts.items()
if state not in {"pending", "running"}
)
active = ", ".join(event["active_stages"])
print(
redact(
f"Run {event['run_id']}: {event['phase']} · {completed}/{event['total_stages']} stages · {event['elapsed_seconds']}s"
+ (f" · {active}" if active else "")
),
file=sys.stderr,
flush=True,
)
args.on_progress = progress
try:
result = args.handler(args)
if not isinstance(result, dict):
raise ValueError("Command did not return a result object")
code = int(result.pop("_exit_code", 0))
if args.format == "json":
print(
json.dumps(
safe_output(result),
sort_keys=True,
indent=2,
ensure_ascii=True,
allow_nan=False,
)
)
else:
summary = result.get("summary", [str(result.get("status", "Completed"))])
print(
"\n".join(redact(str(line)) for line in summary)
if isinstance(summary, list)
else redact(str(summary))
)
return code
except (ValueError, OSError, RuntimeError, ImportError) as exc:
message = redact(str(exc))
if args.format == "json":
print(
json.dumps(
{
"status": "error",
"error": message,
"error_type": type(exc).__name__,
}
)
)
else:
print(f"devkit: {message}", file=sys.stderr)
return 2
except KeyboardInterrupt:
print(
"devkit: interrupted; inspect the saved run before retrying",
file=sys.stderr,
)
return 130
+232
View File
@@ -0,0 +1,232 @@
"""Bounded local records and predictable output; no network or AI dependency."""
from __future__ import annotations
from contextlib import contextmanager
from datetime import datetime, timezone
import hashlib
import json
import os
from pathlib import Path
import re
import stat
import tempfile
import time
META_ROOT = Path(__file__).resolve().parents[3]
MAX_JSON_BYTES = 8 * 1024 * 1024
IDENTIFIER = re.compile(r"[a-zA-Z0-9][a-zA-Z0-9_.-]{0,127}\Z")
def now() -> str:
return datetime.now(timezone.utc).isoformat()
def canonical(value: object) -> bytes:
return json.dumps(
value, sort_keys=True, separators=(",", ":"), ensure_ascii=True, allow_nan=False
).encode()
def digest(value: object) -> str:
return hashlib.sha256(canonical(value)).hexdigest()
def identifier(value: str) -> str:
if (
not isinstance(value, str)
or not IDENTIFIER.fullmatch(value)
or value in {".", ".."}
):
raise ValueError("Invalid record identifier")
return value
def state_root(workspace_root: Path, state_dir: Path | None = None) -> Path:
base = (
state_dir
or Path(os.environ.get("XDG_STATE_HOME", str(Path.home() / ".local/state")))
/ "govoplan/devkit"
)
# Preserve spelling until symlink validation; resolving here would hide an unsafe alias.
base = Path(os.path.abspath(os.fspath(base.expanduser())))
return base / ("workspace-" + digest(str(workspace_root.resolve()))[:24])
def reject_symlinks(path: Path) -> None:
for item in (path, *path.parents):
if item.is_symlink():
raise ValueError("State and evidence paths must not contain symlinks")
def private_directory(path: Path) -> None:
reject_symlinks(path)
path.mkdir(parents=True, exist_ok=True, mode=0o700)
reject_symlinks(path)
metadata = path.stat()
if not stat.S_ISDIR(metadata.st_mode) or (
hasattr(os, "getuid") and metadata.st_uid != os.getuid()
):
raise ValueError("State directory must be owned by the current user")
path.chmod(0o700)
def read_bounded_bytes(path: Path, max_bytes: int = MAX_JSON_BYTES) -> bytes:
reject_symlinks(path)
descriptor = os.open(
path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0)
)
with os.fdopen(descriptor, "rb") as handle:
metadata = os.fstat(handle.fileno())
if not stat.S_ISREG(metadata.st_mode) or metadata.st_size > max_bytes:
raise ValueError("Evidence must be a bounded regular file")
encoded = handle.read(max_bytes + 1)
if len(encoded) > max_bytes:
raise ValueError("Evidence file exceeds its size bound")
return encoded
def read_json(path: Path, max_bytes: int = MAX_JSON_BYTES) -> object:
encoded = read_bounded_bytes(path, max_bytes)
def unique(pairs):
result = {}
for key, value in pairs:
if key in result:
raise ValueError("Duplicate JSON keys are not accepted")
result[key] = value
return result
try:
return json.loads(
encoded,
object_pairs_hook=unique,
parse_constant=lambda _: (_ for _ in ()).throw(
ValueError("Non-finite JSON number")
),
)
except (RecursionError, UnicodeError) as exc:
raise ValueError("JSON nesting or encoding is unsupported") from exc
def atomic_text(path: Path, value: str, max_bytes: int = MAX_JSON_BYTES) -> None:
encoded = value.encode("utf-8")
if len(encoded) > max_bytes:
raise ValueError("Output exceeds its size bound")
reject_symlinks(path.parent)
path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
reject_symlinks(path.parent)
reject_symlinks(path)
if path.exists() and not path.is_file():
raise ValueError("Output target must be a regular file")
descriptor, temporary = tempfile.mkstemp(prefix=".devkit-", dir=path.parent)
try:
with os.fdopen(descriptor, "wb") as handle:
os.fchmod(handle.fileno(), 0o600)
handle.write(encoded)
handle.flush()
os.fsync(handle.fileno())
os.replace(temporary, path)
directory = os.open(path.parent, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0))
try:
os.fsync(directory)
finally:
os.close(directory)
finally:
if os.path.exists(temporary):
os.unlink(temporary)
def atomic_json(path: Path, payload: object) -> None:
atomic_text(
path,
json.dumps(
payload, indent=2, sort_keys=True, ensure_ascii=True, allow_nan=False
)
+ "\n",
)
def redact(text: str) -> str:
"""Best-effort display hygiene, not permission to include secrets in commands."""
for key, value in os.environ.items():
if (
re.search(r"TOKEN|SECRET|PASSWORD|API_KEY|PRIVATE_KEY", key, re.I)
and len(value) >= 4
):
text = text.replace(value, "[redacted]")
text = re.sub(r"(?im)(authorization\s*[:=]\s*)([^\r\n]+)", r"\1[redacted]", text)
text = re.sub(r"(?i)(https?://)[^/\s:@]+:[^/\s@]+@", r"\1[redacted]@", text)
text = re.sub(
r"(?i)((?:token|password|secret|api[_-]?key)\s*[=:]\s*)[^\s,;]+",
r"\1[redacted]",
text,
)
return text
def redact_argv(argv: list[str]) -> list[str]:
result, hide_next = [], False
for argument in argv:
if hide_next:
result.append("[redacted]")
hide_next = False
continue
if re.fullmatch(
r"--?(?:password|passwd|token|secret|api[-_]key|access[-_]token|authorization)",
argument,
re.I,
):
hide_next = True
result.append(redact(argument))
return result
def safe_output(value):
"""Redact presentation, not immutable identity hashes or execution inputs."""
if isinstance(value, dict):
return {
key: redact_argv(item)
if key == "argv"
and isinstance(item, list)
and all(isinstance(arg, str) for arg in item)
else "[redacted]"
if re.fullmatch(
r"password|passwd|token|secret|api[_-]?key|authorization",
str(key),
re.I,
)
else safe_output(item)
for key, item in value.items()
}
if isinstance(value, list):
return [safe_output(item) for item in value]
return redact(value) if isinstance(value, str) else value
@contextmanager
def resource_lock(directory: Path, name: str, timeout: float = 0):
"""Host-local advisory lock, released by the OS even after a process crash."""
import fcntl
private_directory(directory)
path = directory / (hashlib.sha256(name.encode()).hexdigest() + ".lock")
reject_symlinks(path)
descriptor = os.open(
path, os.O_CREAT | os.O_RDWR | getattr(os, "O_NOFOLLOW", 0), 0o600
)
try:
if not stat.S_ISREG(os.fstat(descriptor).st_mode):
raise ValueError("Lock target must be a regular file")
deadline = time.monotonic() + timeout
while True:
try:
fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
break
except BlockingIOError:
if time.monotonic() >= deadline:
raise RuntimeError(f"Resource is busy: {name}") from None
time.sleep(min(0.1, max(0, deadline - time.monotonic())))
yield
finally:
os.close(descriptor)
+90
View File
@@ -0,0 +1,90 @@
"""Small read-only context bundles; no automatic source/credential dumping."""
from __future__ import annotations
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from .common import META_ROOT, now, read_json
from .workspace import inspect_repository, load_project, selected_repositories
def build_context(
workspace_root: Path, project_file: Path | None, names: list[str], changed: bool
) -> dict:
project = load_project(workspace_root, project_file)
repos = selected_repositories(project, names)
with ThreadPoolExecutor(max_workers=8) as pool:
states = list(pool.map(inspect_repository, repos))
if changed:
states = [
state
for state in states
if state["errors"]
or state["dirty_entries"]
or state["ahead"]
or (state["head"] and not state["upstream"])
]
inventory = {}
if not project_file:
path = META_ROOT / "docs/project/ui-review-issue-inventory.json"
if path.is_file():
payload = read_json(path)
for item in payload.get("issues", []) if isinstance(payload, dict) else []:
if isinstance(item, dict):
inventory[item.get("repository")] = item
for state in states:
root = Path(state["path"])
state["instructions"] = [
str(path) for path in (root / "AGENTS.md",) if path.is_file()
]
state["documentation"] = [
str(path)
for path in (
root / "README.md",
root / "docs/README.md",
root / "docs/MODULE_ARCHITECTURE.md",
)
if path.is_file()
]
state["change_entry_count"] = len(state["dirty_entries"])
state["review_issue"] = inventory.get(state["name"], {}).get("url")
state["suggested_check"] = (
f"./devkit check --repo {state['name']} --profile quick --dry-run"
)
summary = [
f"{project.name}: {len(states)} repositories selected (offline; upstream counts may be stale)."
]
for state in states:
errors = f" ERROR: {'; '.join(state['errors'])}" if state["errors"] else ""
summary.append(
f"{state['name']}: {state['branch'] or '(detached/unborn)'}; {state['change_entry_count']} change entries; ahead={state['ahead']} behind={state['behind']}{errors}"
)
return {
"schema_version": 1,
"generated_at": now(),
"workspace_root": str(workspace_root),
"project": project.name,
"remote_checked": False,
"repositories": states,
"summary": summary,
"_exit_code": 1 if any(state["errors"] for state in states) else 0,
}
def register(subparsers):
parser = subparsers.add_parser(
"context",
help="Offline repository changes, ownership and relevant instruction paths",
)
parser.add_argument("--repo", action="append", default=[])
parser.add_argument(
"--changed",
action="store_true",
help="Show dirty or locally ahead repositories, retaining inspection errors",
)
parser.set_defaults(
handler=lambda args: build_context(
args.workspace_root, args.project, args.repo, args.changed
)
)
+453
View File
@@ -0,0 +1,453 @@
"""Explicit suite-plan coverage; never execute or guess nested shell/npm flows."""
from __future__ import annotations
import hashlib
import os
from pathlib import Path
import re
import shlex
import stat
from .common import digest, redact_argv, reject_symlinks
from .package_tests import CORE_COMPONENT_SUITES, declared_tests, discovered_sources
from .workspace import load_project
DISPOSITIONS = ("planned", "covered_elsewhere", "excluded", "unsupported")
MAX_SUITES = 4096
def _read_canonical(path: Path) -> str:
reject_symlinks(path)
descriptor = os.open(
path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0)
)
with os.fdopen(descriptor, "rb") as handle:
metadata = os.fstat(handle.fileno())
if not stat.S_ISREG(metadata.st_mode) or metadata.st_size > 1024 * 1024:
raise ValueError("Canonical focused gate must be a bounded regular file")
encoded = handle.read(1024 * 1024 + 1)
if len(encoded) > 1024 * 1024:
raise ValueError("Canonical focused gate exceeds its size bound")
return encoded.decode("utf-8")
def _node_argv(argv: list[str], cwd: Path) -> tuple[str, ...] | None:
if (
not argv
or argv[0] not in {"node", "{node}", "$NODE"}
and Path(argv[0]).name not in {"node", "nodejs"}
):
return None
offset = 2 if len(argv) > 1 and argv[1] == "--test" else 1
if len(argv) != offset + 1:
return None
target = Path(argv[offset])
if "$" in str(target):
return None
return ("node", *argv[1:offset], str((cwd / target).resolve()))
def canonical_invocations(workspace_root: Path, meta: Path, core: Path) -> dict:
"""Recognize direct commands in exact registered phase wrappers or legacy top level.
Shell is not evaluated. Bodies, conditionals, loops, functions, npm hooks and
recursive commands do not grant coverage. Unknown cwd invalidates matches.
The registered marked wrappers are the only function bodies admitted.
"""
path = meta / "tools/checks/check-focused.sh"
try:
text = _read_canonical(path)
except (OSError, ValueError):
return {
"path": str(path),
"sha256": None,
"npm": [],
"node": [],
"node_phases": [],
"notes": [
"Canonical focused script is unavailable/unreadable; no package coverage inferred."
],
}
values = {
"WORKSPACE_ROOT": str(workspace_root),
"META_ROOT": str(meta),
"ROOT": str(core),
}
def expand(value: str) -> str | None:
value = re.sub(
r"\$\{(WORKSPACE_ROOT|META_ROOT|ROOT)\}|\$(WORKSPACE_ROOT|META_ROOT|ROOT)(?![A-Za-z0-9_])",
lambda match: values[match[1] or match[2]],
value,
)
return None if "$" in value or "`" in value else value
result = {
"path": str(path),
"sha256": hashlib.sha256(text.encode()).hexdigest(),
"npm": [],
"node": [],
"node_phases": [],
"notes": [],
}
metadata = meta / "tools/checks/focused-phases.json"
blocks = {None: text}
if metadata.exists() or metadata.is_symlink() or "# devkit-phase:" in text:
from .catalog import focused_phases, focused_phase_bodies
try:
phases = focused_phases(meta)
blocks = focused_phase_bodies(text, phases)
result["phase_ids"] = [phase["id"] for phase in phases]
result["metadata_sha256"] = digest(phases)
except (OSError, ValueError):
result["notes"].append(
"Focused phase metadata/marked bodies are invalid or unavailable; no phase coverage inferred."
)
return result
cwd: Path | None = meta
body_end, nesting = None, 0
current_phase = object()
for phase_id, line in (
(phase_id, line)
for phase_id, block in blocks.items()
for line in block.replace("\\\n", " ").splitlines()
):
if current_phase != phase_id:
cwd, body_end, nesting = meta, None, 0
current_phase = phase_id
stripped = line.strip()
if body_end is not None:
if stripped == body_end:
body_end = None
continue
heredoc = re.search(r"<<-?\s*['\"]?([A-Za-z_][A-Za-z0-9_]*)['\"]?", line)
if heredoc:
body_end = heredoc[1]
continue
if not stripped or stripped.startswith("#"):
continue
if re.match(
r"(?:if|for|while|until|case|select)\b|(?:function\s+\w+|\w+\s*\(\s*\))",
stripped,
):
nesting += 1
continue
if re.match(r"(?:fi|done|esac)\b|^}\s*;?$", stripped):
nesting = max(0, nesting - 1)
continue
if nesting:
if re.search(r"\bcd\s", stripped):
cwd = None
continue
try:
parts = shlex.split(line, comments=True)
except ValueError:
result["notes"].append(
"An unparseable shell line grants no suite coverage."
)
continue
if not parts:
continue
if parts[0] == "cd":
destination = expand(parts[1]) if len(parts) == 2 else None
candidate = (
(cwd / destination).resolve()
if destination and cwd
else Path(destination).resolve()
if destination and Path(destination).is_absolute()
else None
)
cwd = (
candidate
if candidate and candidate.is_relative_to(workspace_root)
else None
)
continue
if cwd is None:
continue
if (
parts[0] in {"$NPM", "${NPM}", "npm"}
and len(parts) >= 3
and parts[1] == "run"
and re.fullmatch(r"test(?::[A-Za-z0-9_.:-]+)?", parts[2])
):
arguments = parts[3:]
if arguments and (
arguments[0] != "--"
or any(
not re.fullmatch(r"[A-Za-z0-9_.-]+", item) for item in arguments[1:]
)
):
continue
result["npm"].append(
{
"cwd": str(cwd),
"name": parts[2],
"args": arguments[1:] if arguments else [],
"phase": phase_id,
}
)
elif parts[0] in {"$NODE", "${NODE}", "node"}:
expanded = [expand(part) for part in parts[1:]]
if all(part is not None for part in expanded):
command = _node_argv(["node", *expanded], cwd)
if command:
result["node"].append(command)
result["node_phases"].append({"argv": command, "phase": phase_id})
result["notes"] = list(dict.fromkeys(result["notes"]))[:8]
return result
def coverage_inventory(
workspace_root: Path, profile: str, project_file: Path | None, stages: list[dict]
) -> dict:
workspace_root = workspace_root.resolve()
project = load_project(workspace_root, project_file)
stage_map = {item["id"]: item for item in stages}
direct = {}
explicit_npm = {}
for stage in stages:
argv, cwd = stage["argv"], Path(stage["cwd"])
normalized = _node_argv(argv, cwd)
if normalized:
direct[normalized] = stage["id"]
if (
len(argv) == 3
and (argv[0] in {"npm", "{npm}"} or Path(argv[0]).name == "npm")
and argv[1] == "run"
):
explicit_npm[(str(cwd.resolve()), argv[2])] = stage["id"]
paths = {repo.name: repo.path for repo in project.repositories}
canonical = (
canonical_invocations(
workspace_root,
paths.get("govoplan", workspace_root / "govoplan"),
paths.get("govoplan-core", workspace_root / "govoplan-core"),
)
if project_file is None
else None
)
canonical_stages = {}
if canonical is not None:
legacy = stage_map.get("focused-workspace")
if legacy and legacy["argv"] == ["bash", canonical["path"]]:
canonical_stages[None] = "focused-workspace"
for identity in canonical.get("phase_ids", []):
check = stage_map.get("focused." + identity)
if check and check["argv"] == [
"bash",
canonical["path"],
"--phase",
identity,
]:
canonical_stages[identity] = check["id"]
full = bool(canonical_stages)
invocations = (
[
{**item, "covering_stage": canonical_stages[item.get("phase")]}
for item in canonical["npm"]
if item.get("phase") in canonical_stages
]
if canonical
else []
)
canonical_nodes = (
{
tuple(item["argv"]): canonical_stages[item.get("phase")]
for item in canonical.get("node_phases", [])
if item.get("phase") in canonical_stages
}
if canonical
else {}
)
suites, notes = [], []
if canonical:
notes.extend(canonical["notes"])
core_package = (
paths.get("govoplan-core", workspace_root / "govoplan-core")
/ "webui/package.json"
)
core_requests = [
item
for item in invocations
if item["cwd"] == str(core_package.parent) and item["name"] == "test:components"
]
requested_components = set()
for item in core_requests:
requested_components.update(
CORE_COMPONENT_SUITES
if not item["args"] or item["args"] == ["all"]
else item["args"]
)
requested_components.intersection_update(CORE_COMPONENT_SUITES)
for repo in project.repositories:
for location in ("package.json", "webui/package.json"):
package = repo.path / location
if not package.exists() and not package.is_symlink():
continue
entries = declared_tests(repo, package)
if location == "webui/package.json":
declared_argv = {
tuple(item["_argv"]) for item in entries if item["_argv"]
}
entries.extend(
item
for item in discovered_sources(repo)
if not item["_argv"] or tuple(item["_argv"]) not in declared_argv
)
for item in entries:
private_argv = item.pop("_argv")
component = item["component_suite"]
identity = (str(package.parent), item["name"])
normalized = (
_node_argv(private_argv, package.parent) if private_argv else None
)
matching = [
value
for value in invocations
if (value["cwd"], value["name"]) == identity
]
item.update(disposition="excluded", covering_stage=None)
if component is not None:
all_or_selected = (
component == "all" or component in requested_components
)
if "core.component-batch" in stage_map:
item.update(
disposition="planned"
if component == "all"
else "covered_elsewhere",
covering_stage="core.component-batch",
reason="UI explicitly runs the shared component batch; quick never compiles it.",
)
elif full and core_requests and all_or_selected:
item.update(
disposition="planned"
if component == "all"
else "covered_elsewhere",
covering_stage=core_requests[0]["covering_stage"],
reason=f"Canonical focused gate explicitly selects {len(requested_components)}/{len(CORE_COMPONENT_SUITES)} component suites; this is not the complete component batch.",
)
else:
item["reason"] = (
"No component compilation in quick/backend; UI runs all components. Full runs only its explicitly named subset."
)
if component == "all":
selected = (
list(CORE_COMPONENT_SUITES)
if "core.component-batch" in stage_map
else sorted(requested_components)
)
item.update(
covered_components=selected,
excluded_components=[
name
for name in CORE_COMPONENT_SUITES
if name not in selected
],
)
elif identity in explicit_npm:
item.update(
disposition="planned",
covering_stage=explicit_npm[identity],
reason="Explicit project check invokes this exact package suite.",
)
elif matching:
if any(not value["args"] for value in matching):
item.update(
disposition="planned",
covering_stage=next(
value["covering_stage"]
for value in matching
if not value["args"]
),
reason="The canonical focused script directly invokes this exact package suite.",
)
else:
item["reason"] = (
"Canonical invocation supplies arguments; complete suite coverage cannot be inferred."
)
elif normalized in direct:
item.update(
disposition="planned",
covering_stage=direct[normalized],
reason="A selected stage runs this exact direct source-test command.",
)
elif normalized in canonical_nodes:
item.update(
disposition="covered_elsewhere",
covering_stage=canonical_nodes[normalized],
reason="Canonical focused script directly runs this suite's exact Node target.",
)
elif private_argv is None:
item["disposition"] = "unsupported"
elif item["name"] in {
"test:module-permutations",
"test:vite-cache-isolation",
}:
item["reason"] = (
"Separate environment/permutation suite; absent from the selected stage plan."
)
else:
item["reason"] = (
"Not directly present in this profile's stage plan; nested npm scripts/hooks are not inferred."
)
suites.append(item)
if len(suites) > MAX_SUITES:
raise ValueError("Coverage inventory exceeds its bounded suite count")
if project_file is not None:
for check in project.config.get("checks", []):
identity = check["id"]
suites.append(
{
"repo": ",".join(check.get("repos", [])) or "project",
"name": identity,
"kind": "project-check",
"package_path": None,
"argv": redact_argv(check["argv"]),
"command_sha256": digest(check["argv"]),
"disposition": "planned" if identity in stage_map else "excluded",
"covering_stage": identity if identity in stage_map else None,
"reason": "Selected declared check or dependency."
if identity in stage_map
else "Declared check is outside this profile/selection.",
}
)
if len(suites) > MAX_SUITES:
raise ValueError("Coverage inventory exceeds its bounded suite count")
counts = {
kind: sum(item["disposition"] == kind for item in suites)
for kind in DISPOSITIONS
}
notes.append(
"Coverage describes planned suite invocations, not passing tests, per-test coverage, or completed UI review."
)
if full:
notes.append(
"Full is the canonical focused gate, not every declared package test; recursive commands and npm hooks are deliberately not guessed."
)
return {
"schema_version": 1,
"profile": profile,
"scope": "Declared root/webui package test scripts, discovered UI structural checks, and custom project checks across registered repositories.",
"suite_count": len(suites),
"stages": list(stage_map),
"counts": counts,
"suites": suites,
"canonical_gate": {
key: canonical[key]
for key in ("path", "sha256", "metadata_sha256")
if key in canonical
}
if canonical
else None,
"notes": notes,
"summary": [
"Suite coverage: "
+ ", ".join(f"{value} {key}" for key, value in counts.items()),
*notes,
],
}
+133
View File
@@ -0,0 +1,133 @@
"""Recorded documentation checks built from existing owning-module contracts."""
from __future__ import annotations
from pathlib import Path
import uuid
from .catalog import stage
from .common import state_root
from .workspace import load_project, selected_repositories
LIMITATIONS = [
"Static baseline, marker coverage and known display slots do not prove complete workflow or linguistic coverage.",
"Computed labels, configured documentation providers and runtime module/permission combinations still require manual review.",
"Audit findings do not edit documentation, translate text, publish evidence or close a review issue.",
]
def build_doc_stages(args) -> list[dict]:
workspace = Path(args.workspace_root).resolve()
project = load_project(workspace, getattr(args, "project", None))
if getattr(args, "project", None):
raise ValueError(
"The docs audit currently targets GovOPlaN manifest/locale contracts. Register another project's documentation checks in its check profiles."
)
selected = selected_repositories(
project, getattr(args, "repo", []), changed=getattr(args, "changed", False)
)
if getattr(args, "changed", False) and not selected:
return []
meta = next(repo.path for repo in project.repositories if repo.name == "govoplan")
core = next(
repo.path for repo in project.repositories if repo.name == "govoplan-core"
)
output = (
state_root(workspace, getattr(args, "state_dir", None))
/ "artifacts"
/ f"docs-{uuid.uuid4().hex}"
)
reason = "Reuse the existing owning-module documentation and translation checks"
labels = [
"{node}",
str(meta / "tools/devkit/audit-display-labels.mjs"),
"--workspace-root",
str(workspace),
]
for repo in selected:
labels.extend(["--repo", repo.name])
stages = [
stage(
"docs.manifests",
"Static user/admin documentation and manifest contracts",
[
"{python}",
str(meta / "tools/checks/check-manifest-shapes.py"),
"--workspace-root",
str(workspace),
"--require-architecture",
],
meta,
reason=reason,
timeout_seconds=600,
),
stage(
"docs.interface-inventory",
"Existing EN/DE markers, high-risk help and interface declarations",
[
"{python}",
str(meta / "tools/inventory/platform-interface-inventory.py"),
"--workspace-root",
str(workspace),
"--strict",
"--strict-declarations",
"--strict-endpoints",
"--output-dir",
str(output),
],
meta,
reason="Shared inventory remains workspace-wide; --repo narrows only the additional plain-label audit",
timeout_seconds=600,
),
stage(
"docs.translation-structure",
"Existing translation key/structural-value guard",
["{node}", str(core / "webui/scripts/audit-i18n-structural.mjs")],
core / "webui",
reason=reason,
),
stage(
"docs.plain-display-labels",
"Plain display labels and owning catalog registration",
labels,
meta,
reason="Supplement the marker inventory with known display slots; dynamic cases are review candidates",
timeout_seconds=600,
),
]
# These constraints belong to the saved evidence, not only the immediate
# CLI response. The publisher already validates bounded coverage_notes.
stages[0]["coverage_notes"] = list(LIMITATIONS)
return stages
def audit(args) -> dict:
from .runner import run_checks
result = run_checks(args, build_doc_stages(args))
result["limitations"] = list(LIMITATIONS)
result.setdefault("_exit_code", 0)
summary = result.setdefault("summary", [])
summary.extend(note for note in LIMITATIONS if note not in summary)
return result
def register(subparsers) -> None:
docs = subparsers.add_parser(
"docs", help="Audit existing documentation and translation contracts"
)
commands = docs.add_subparsers(dest="docs_command", required=True)
command = commands.add_parser(
"audit", help="Run recorded checks; never edit or generate translations"
)
command.add_argument(
"--repo",
action="append",
default=[],
help="Repository/alias for plain-label checks; shared checks stay workspace-wide",
)
command.add_argument("--changed", action="store_true")
command.add_argument("--jobs", type=int, default=2)
command.add_argument("--dry-run", action="store_true")
command.set_defaults(handler=audit, profile="docs")
+232
View File
@@ -0,0 +1,232 @@
"""Read-only environment preflight with actionable, never automatic repairs."""
from __future__ import annotations
from dataclasses import replace
import os
from pathlib import Path
import socket
from .common import now
from .environment import execution_environment, resolve_tools, tool_version
from .workspace import inspect_repository, load_project, selected_repositories
from .process import require_capture
def _required_tools(project, workspace_root, selected, *, filtered, profile=None):
"""Use the catalog's selected commands, including dependencies, without running them."""
from .catalog import _custom_stages
if profile is None:
# Doctor covers all declared profiles by default, or all declared checks
# when the project only uses the context/doctor commands so far.
profiles = project.config.get("profiles", {})
records = project.config.get("checks", [])
if not isinstance(profiles, dict) or not isinstance(records, list):
raise ValueError(
"Project profiles and checks must have their declared shapes"
)
identities = []
if profiles:
for values in profiles.values():
if not isinstance(values, list) or any(
not isinstance(value, str) for value in values
):
raise ValueError("A project profile must be a list of check IDs")
identities.extend(values)
else:
if any(not isinstance(record, dict) for record in records):
raise ValueError("Project checks must be objects")
identities = [record.get("id") for record in records]
if any(not isinstance(identity, str) for identity in identities):
raise ValueError("Project checks must have string IDs")
project = replace(
project,
config={
**project.config,
"profiles": {"quick": list(dict.fromkeys(identities))},
},
)
profile = "quick"
stages = _custom_stages(project, workspace_root, profile, selected, filtered)
# Python is required by the runner and its environment fingerprint even if
# all selected check commands use another interpreter. Explicitly configured
# tools also declare requirements for commands hidden inside project scripts.
required = {"python", *project.config.get("tools", {})}
for stage in stages:
argv = stage["argv"]
for name in ("node", "npm", "python"):
if any("{" + name + "}" in value for value in argv):
required.add(name)
executable = Path(argv[0]).name
if executable in {"node", "nodejs"}:
required.add("node")
elif executable in {"npm", "npx"}:
required.update({"npm", "node"})
if "npm" in required:
required.add("node")
return required
def diagnose(args) -> dict:
project = load_project(args.workspace_root, args.project)
tools = resolve_tools(args.workspace_root, project)
env = execution_environment(args.workspace_root, project, tools)
selected = selected_repositories(project, args.repo)
required = (
_required_tools(
project,
args.workspace_root,
selected,
filtered=bool(args.repo),
profile=getattr(args, "profile", None),
)
if args.project
else set(tools)
)
checks = []
for name, executable in tools.items():
if name not in required:
checks.append(
{
"id": name,
"status": "not_required",
"detail": "Not required by the selected declared project checks; not probed.",
"path": executable,
"repair": "For indirect dependencies inside scripts, declare the tool under project tools.",
}
)
continue
version = tool_version(executable, env)
checks.append(
{
"id": name,
"status": "passed" if version != "unavailable" else "blocked",
"detail": version,
"path": executable,
"repair": f"Install/configure {name}; set {name.upper()} or project tools.{name}. No installation was attempted.",
}
)
for repo in selected:
snapshot = inspect_repository(repo)
checks.append(
{
"id": repo.name,
"status": "blocked" if snapshot["errors"] else "passed",
"detail": "; ".join(snapshot["errors"])
or "Git checkout readable (dirty work is allowed).",
"repair": "Use tools/repo/bootstrap-repositories.py --check; review missing checkouts before cloning.",
}
)
for package_dir in (repo.path, repo.path / "webui"):
if (package_dir / "package.json").is_file():
present = (package_dir / "node_modules").is_dir()
checks.append(
{
"id": f"dependencies:{repo.name}:{package_dir.name}",
"status": "passed" if present else "warning",
"detail": "Dependency directory exists; availability is not a full dependency audit."
if present
else "No local node_modules directory; workspace-hoisted packages may still resolve.",
"repair": "Inspect the owning package lock and install instructions before running npm ci.",
}
)
if not args.project:
script = args.workspace_root / "govoplan/tools/repo/sync-python-environment.py"
if script.is_file():
result = require_capture(
[
tools["python"],
str(script),
"--check",
"--requirements",
str(args.workspace_root / "govoplan/requirements-dev.txt"),
"--python",
tools["python"],
],
timeout=60,
max_stdout=65536,
env=env,
)
checks.append(
{
"id": "python-environment-sync",
"status": "passed" if result.returncode == 0 else "warning",
"detail": "Environment synchronization fingerprint is current."
if result.returncode == 0
else "Environment sync check did not pass; inspect its --dry-run output.",
"repair": "./.venv/bin/python tools/repo/sync-python-environment.py --dry-run --requirements requirements-dev.txt --python ./.venv/bin/python",
}
)
browser_roots = [
Path.home() / ".cache/ms-playwright",
Path.home() / ".var/app/com.vscodium.codium/cache/ms-playwright",
]
explicit_browser = os.environ.get("PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH")
browser_available = (
(Path(explicit_browser).is_file() and os.access(explicit_browser, os.X_OK))
if explicit_browser
else any(
root.is_dir() and any(root.glob("chromium-*/chrome-linux*/chrome"))
for root in browser_roots
)
)
checks.append(
{
"id": "browser",
"status": "passed" if browser_available else "warning",
"detail": "Configured/cached Chromium found."
if browser_available
else "Chromium executable not found; check PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH or install the pinned Playwright browser.",
"repair": "Follow Core WebUI conformance setup; do not start a development server to repair this.",
}
)
with socket.socket() as probe:
probe.settimeout(0.25)
busy = probe.connect_ex(("127.0.0.1", 4174)) == 0
checks.append(
{
"id": "browser-test-port",
"status": "warning" if busy else "passed",
"detail": "Port 4174 is occupied; do not kill another run."
if busy
else "Port 4174 is free now (not a reservation).",
"repair": "Wait for the owning check run; use devkit resource coordination.",
}
)
summary = [
f"Environment preflight: {sum(item['status'] == 'blocked' for item in checks)} blockers, {sum(item['status'] == 'warning' for item in checks)} warnings."
]
summary.extend(
f"{item['id']}: {item['status']}{item['detail']}"
for item in checks
if item["status"] != "passed"
)
summary.append(
"Read-only: no installations, configuration repairs or servers started."
)
if args.project:
summary.append(
"Required tools follow declared checks and explicit project tools; indirect script dependencies must be declared explicitly."
)
return {
"schema_version": 1,
"generated_at": now(),
"checks": checks,
"required_tools": sorted(required),
"summary": summary,
"_exit_code": 1 if any(item["status"] == "blocked" for item in checks) else 0,
}
def register(subparsers):
parser = subparsers.add_parser(
"doctor", help="Read-only environment preflight and exact repair guidance"
)
parser.add_argument("--repo", action="append", default=[])
parser.add_argument(
"--profile",
choices=("quick", "ui", "backend", "full"),
help="For portable projects, limit tool requirements to this check profile (default: all declared profiles)",
)
parser.set_defaults(handler=diagnose)
+271
View File
@@ -0,0 +1,271 @@
"""Resolve local tools once, without installation, network access or server startup."""
from __future__ import annotations
import hashlib
from itertools import islice
import os
from pathlib import Path
import shutil
import stat
import subprocess
import sys
from .common import digest
from .workspace import Project
from .process import require_capture
MAX_DISCOVERY_CHILDREN = 4096
def resolve_tools(workspace_root: Path, project: Project) -> dict[str, str]:
configured = project.config.get("tools", {})
if not isinstance(configured, dict) or set(configured) - {"python", "node", "npm"}:
raise ValueError("Project tools must configure only python, node and npm")
tools = {}
python_env = (
Path(
os.environ.get("GOVOPLAN_VENV_ROOT", str(workspace_root / "govoplan/.venv"))
)
/ "bin/python"
)
for name in ("python", "node", "npm"):
default = (
str(python_env)
if name == "python" and python_env.is_file()
else sys.executable
if name == "python"
else name
)
value = configured.get(name) or os.environ.get(name.upper()) or default
if not isinstance(value, str) or not value or "\0" in value:
raise ValueError(f"Invalid executable configuration for {name}")
executable = shutil.which(value)
if executable:
# Keep venv executable symlinks: resolving them loses Python's environment.
tools[name] = os.path.abspath(executable)
else:
tools[name] = value
return tools
def execution_environment(
workspace_root: Path, project: Project, tools: dict[str, str]
) -> dict[str, str]:
env = dict(os.environ)
if (
project.config.get("organization") == "GovOPlaN"
and "schema_version" not in project.config
):
# Managed GovOPlaN checks must not inherit another checkout's scope.
env["GOVOPLAN_WORKSPACE_ROOT"] = str(workspace_root.resolve())
core = next(
repo.path for repo in project.repositories if repo.name == "govoplan-core"
)
env["GOVOPLAN_CORE_ROOT"] = str(core)
env["GOVOPLAN_CORE_SOURCE_ROOT"] = str(core)
directories = [
str(Path(tools[name]).parent)
for name in ("python", "node", "npm")
if Path(tools[name]).is_absolute()
]
core_bins = workspace_root / "govoplan-core/webui/node_modules/.bin"
if core_bins.is_dir():
directories.insert(0, str(core_bins))
env["PATH"] = os.pathsep.join([*directories, env.get("PATH", "")])
env.update({name.upper(): value for name, value in tools.items()})
sources = [
str(repo.path / "src")
for repo in project.repositories
if (repo.path / "src").is_dir()
]
if sources:
env["PYTHONPATH"] = os.pathsep.join(
sources + ([env["PYTHONPATH"]] if env.get("PYTHONPATH") else [])
)
env["GIT_OPTIONAL_LOCKS"] = "0"
env["PYTHONDONTWRITEBYTECODE"] = "1"
return env
def tool_version(executable: str, env: dict[str, str]) -> str:
try:
result = require_capture(
[executable, "--version"],
timeout=15,
max_stdout=4096,
max_stderr=4096,
env=env,
)
return (
(result.stdout or result.stderr).decode(errors="replace").strip()[:256]
if result.returncode == 0
else "unavailable"
)
except (OSError, subprocess.TimeoutExpired, ValueError):
return "unavailable"
def _file_identity(metadata: os.stat_result) -> tuple[int, ...]:
return (
metadata.st_dev,
metadata.st_ino,
metadata.st_mode,
metadata.st_size,
metadata.st_ctime_ns,
metadata.st_mtime_ns,
)
def _environment_file_hash(path: Path, maximum: int) -> str | None:
"""Read a stable, bounded regular target without changing executable spelling.
Venv executables and package directories may legitimately be symlinks. Only
the read target is resolved; both names and the open descriptor are verified
again afterwards. Missing optional metadata stays absent, but a present
malformed or concurrently changing entry cannot certify an environment.
"""
try:
original = path.lstat()
except FileNotFoundError:
return None
try:
target = path.resolve(strict=True)
target_metadata = target.lstat()
descriptor = os.open(
target,
os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0),
)
with os.fdopen(descriptor, "rb") as handle:
before = os.fstat(handle.fileno())
if not stat.S_ISREG(before.st_mode) or before.st_size > maximum:
raise ValueError("Environment input must be a bounded regular file")
identity = _file_identity(before)
if identity != _file_identity(target_metadata):
raise ValueError("Environment input changed before fingerprinting")
hasher, size = hashlib.sha256(), 0
for chunk in iter(
lambda: handle.read(min(1024 * 1024, maximum - size + 1)), b""
):
size += len(chunk)
if size > maximum:
raise ValueError("Environment input grew beyond its size bound")
hasher.update(chunk)
if (
_file_identity(os.fstat(handle.fileno())) != identity
or _file_identity(target.lstat()) != identity
or _file_identity(path.lstat()) != _file_identity(original)
or path.resolve(strict=True) != target
):
raise ValueError("Environment input changed during fingerprinting")
return hasher.hexdigest()
except (OSError, RuntimeError) as exc:
raise ValueError("Environment input cannot be fingerprinted safely") from exc
def _discovery_path_shape(path: Path) -> dict:
"""Directory membership, not timestamps changed by ordinary build outputs."""
try:
metadata = path.lstat()
except (FileNotFoundError, NotADirectoryError):
return {"kind": "missing"}
try:
resolved = path.resolve()
try:
target_kind = stat.S_IFMT(resolved.lstat().st_mode)
except (FileNotFoundError, NotADirectoryError):
target_kind = "missing"
if target_kind == stat.S_IFLNK:
# Newer pathlib versions can retain a loop with strict=False.
raise ValueError("Native source discovery cannot be resolved safely")
return {
"kind": stat.S_IFMT(metadata.st_mode),
"resolved": str(resolved),
"target_kind": target_kind,
}
except (OSError, RuntimeError) as exc:
raise ValueError("Native source discovery cannot be resolved safely") from exc
def _native_discovery_fingerprint(workspace_root: Path, project: Project) -> str:
"""Bind glob discovery and registered ownership omitted by narrow Git scopes."""
children = list(islice(workspace_root.iterdir(), MAX_DISCOVERY_CHILDREN + 1))
if len(children) > MAX_DISCOVERY_CHILDREN:
raise ValueError("Native source discovery exceeds its bounded ownership audit")
def shape(path):
return {
name: _discovery_path_shape(path / name if name else path)
for name in ("", "src", "webui")
}
return digest(
{
"version": 1,
"siblings": [
{"name": child.name, "shape": shape(child)}
for child in sorted(children)
if child.name.startswith("govoplan")
],
"registered": [
{"name": repo.name, "path": str(repo.path), "shape": shape(repo.path)}
for repo in sorted(project.repositories, key=lambda item: item.name)
],
}
)
def environment_fingerprint(
workspace_root: Path, project: Project, tools: dict[str, str], env: dict[str, str]
) -> str:
identity = {
"environment": {
key: value
for key, value in env.items()
if key not in {"_", "SHLVL", "PWD", "OLDPWD"}
},
"tools": {},
}
if (
project.config.get("organization") == "GovOPlaN"
and "schema_version" not in project.config
):
identity["native_source_discovery"] = _native_discovery_fingerprint(
workspace_root, project
)
for name, executable in tools.items():
binary_hash = _environment_file_hash(Path(executable), 512 * 1024 * 1024)
identity["tools"][name] = {
"path": executable,
"version": tool_version(executable, env),
"sha256": binary_hash,
}
installed = {}
for repo in project.repositories:
for suffix in (
"node_modules/.package-lock.json",
"webui/node_modules/.package-lock.json",
".venv/pyvenv.cfg",
):
path = repo.path / suffix
value = _environment_file_hash(path, 32 * 1024 * 1024)
if value is not None:
installed[str(path)] = value
try:
result = require_capture(
[
tools["python"],
"-c",
"import importlib.metadata,json; print(json.dumps(sorted((d.metadata['Name'],d.version) for d in importlib.metadata.distributions())))",
],
timeout=30,
max_stdout=1024 * 1024,
env=env,
)
if result.returncode:
raise ValueError("Cannot fingerprint installed Python distributions")
installed["python_distributions"] = hashlib.sha256(result.stdout).hexdigest()
except (OSError, subprocess.TimeoutExpired) as exc:
raise ValueError("Cannot fingerprint Python environment") from exc
identity["installed"] = installed
return digest(identity)
+417
View File
@@ -0,0 +1,417 @@
"""Versioned repository-scoped input identities with run-local content memoization.
No stored receipt supplies commands to this engine. Every snapshot re-reads Git
HEAD/index/flags/membership and opens its inputs; only stable file-content hashes
are memoized, never repository snapshots or prior-run results.
"""
from __future__ import annotations
import hashlib
import os
from pathlib import Path
import re
import stat
from .common import META_ROOT, canonical, digest, identifier
from .workspace import Project, git_bytes
FINGERPRINT_VERSION = "repository-inputs-v1"
MAX_SOURCE_BYTES = 64 * 1024 * 1024
MAX_CACHE_FILES = 200_000
MAX_REPOSITORY_ENTRIES = 200_000
_INDEX_RECORD = re.compile(rb"[A-Za-z] [0-7]{6} [a-fA-F0-9]{40,64} [0-3]\t(.*)\Z", re.S)
_RUNTIME_FIELDS = {
"status",
"exit_code",
"duration_seconds",
"log_path",
"log_sha256",
"error",
"started_at",
"finished_at",
"reused_from",
"output_truncated",
"checkpoint_verified",
"checkpoint_at",
"cache_key",
"input_fingerprint",
"reuse_reason",
"omitted_output_bytes",
}
def validate_input_declaration(value: object, repository_names) -> dict:
"""Pure validation shared with planning; does not inspect files or run Git."""
if not isinstance(value, dict) or set(value) != {"repos"}:
raise ValueError(
"Stage inputs must contain only the required repos declaration"
)
names = value["repos"]
if not isinstance(names, list) or not 1 <= len(names) <= 256:
raise ValueError(
"Input repos must be a nonempty bounded list of canonical repository names"
)
for name in names:
identifier(name)
if len(set(names)) != len(names):
raise ValueError("Duplicate input repository reference")
if set(names) - set(repository_names):
raise ValueError(
"Input repos reference an unknown/noncanonical repository name"
)
return {"repos": sorted(names)}
def _file_identity(metadata: os.stat_result) -> tuple[int, ...]:
return (
metadata.st_dev,
metadata.st_ino,
metadata.st_mode,
metadata.st_size,
metadata.st_ctime_ns,
metadata.st_mtime_ns,
)
def _stats() -> dict[str, int]:
return {
name: 0
for name in (
"repositories",
"git_calls",
"entries",
"files",
"bytes",
"cache_hits",
"tooling_files",
"tooling_bytes",
"tooling_cache_hits",
)
}
class InputSnapshotter:
"""One run's in-memory memo; create a fresh instance for each invocation."""
def __init__(self, project: Project, *, workspace_root: Path):
self.project = project
self.workspace_root = Path(workspace_root).resolve()
self.repositories = {}
for repo in project.repositories:
identifier(repo.name)
if repo.name in self.repositories:
raise ValueError("Duplicate input repository name")
if not repo.path.is_absolute() or not repo.path.resolve().is_relative_to(
self.workspace_root
):
raise ValueError("Input repository path escapes its workspace")
self.repositories[repo.name] = repo
if not self.repositories or len(self.repositories) > 256:
raise ValueError("Input project requires 1256 repositories")
# One most recent stable identity per path. This is deliberately not
# serializable/persisted, and hashes are never reused on mtime alone.
self._files: dict[Path, tuple[tuple[int, ...], bytes]] = {}
def _repo_names(self, names: object) -> list[str]:
return validate_input_declaration({"repos": names}, self.repositories)["repos"]
def scope(self, stage: dict) -> dict:
"""Validate one explicit scope; absent inputs means the whole workspace."""
if "inputs" not in stage:
return {
"version": 1,
"kind": "workspace",
"declared": False,
"repos": sorted(self.repositories),
}
value = validate_input_declaration(stage["inputs"], self.repositories)
return {
"version": 1,
"kind": "repositories",
"declared": True,
"repos": value["repos"],
}
def _file_hash(
self, path: Path, statistics: dict, *, expected=None, tooling: bool = False
) -> bytes:
prefix = "tooling_" if tooling else ""
maximum = 4 * 1024 * 1024 if tooling else MAX_SOURCE_BYTES
descriptor = os.open(
path,
os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0),
)
with os.fdopen(descriptor, "rb") as handle:
before = os.fstat(handle.fileno())
if not stat.S_ISREG(before.st_mode) or before.st_size > maximum:
raise ValueError("Input must be a bounded regular file")
identity = _file_identity(before)
if expected is not None and _file_identity(expected) != identity:
raise ValueError(
"Input changed before its source identity could be established"
)
statistics[prefix + "files"] += 1
cached = self._files.get(path)
if cached is not None and cached[0] == identity:
value = cached[1]
statistics[prefix + "cache_hits"] += 1
else:
hasher, size = hashlib.sha256(), 0
for chunk in iter(
lambda: handle.read(min(1024 * 1024, maximum - size + 1)), b""
):
size += len(chunk)
if size > maximum:
raise ValueError("Input grew beyond its source identity limit")
hasher.update(chunk)
statistics[prefix + "bytes"] += size
value = hasher.digest()
after = os.fstat(handle.fileno())
# Rechecking the pathname also rejects replacement after the open;
# a stable old FD is not evidence about a newly replaced source file.
current = path.lstat()
if _file_identity(after) != identity or _file_identity(current) != identity:
raise ValueError("Input changed while calculating its source identity")
if path not in self._files and len(self._files) >= MAX_CACHE_FILES:
self._files.clear() # Eviction only causes extra reads, never reuse.
self._files[path] = (identity, value)
return value
def _entry(self, repo, encoded_name: bytes, statistics: dict) -> dict:
relative = Path(os.fsdecode(encoded_name))
if relative.is_absolute() or ".." in relative.parts:
raise ValueError("Unsafe source membership path")
path = repo.path / relative
statistics["entries"] += 1
try:
metadata = path.lstat()
except FileNotFoundError:
return {"kind": "missing"}
if not path.parent.resolve().is_relative_to(repo.path.resolve()):
raise ValueError("Input path escapes its declared repository")
result = {"mode": stat.S_IMODE(metadata.st_mode)}
if stat.S_ISLNK(metadata.st_mode):
target_name = os.readlink(path)
try:
target = path.resolve()
except (OSError, RuntimeError) as exc:
raise ValueError("Input symlink cannot be resolved safely") from exc
if not target.is_relative_to(repo.path.resolve()):
raise ValueError(
"Input symlink target escapes its repository; undeclared target bytes cannot be reused"
)
result.update(
kind="symlink",
target_sha256=hashlib.sha256(os.fsencode(target_name)).hexdigest(),
)
try:
target_metadata = target.lstat()
except FileNotFoundError:
result["target_state"] = "missing"
else:
if not stat.S_ISREG(target_metadata.st_mode):
raise ValueError(
"Input symlink must target a regular file inside its repository"
)
result.update(
target_state="file",
target_mode=stat.S_IMODE(target_metadata.st_mode),
target_content=self._file_hash(
target, statistics, expected=target_metadata
).hex(),
)
if (
_file_identity(path.lstat()) != _file_identity(metadata)
or os.readlink(path) != target_name
):
raise ValueError(
"Input symlink changed while calculating source identity"
)
elif stat.S_ISREG(metadata.st_mode):
result.update(
kind="file",
content=self._file_hash(path, statistics, expected=metadata).hex(),
)
else:
raise ValueError(
"Unsupported source entry; scoped repository inputs require files or in-repository file symlinks"
)
return result
def _repository(self, name: str, statistics: dict) -> str:
repo = self.repositories[name]
statistics["repositories"] += 1
identity = {
"version": FINGERPRINT_VERSION,
"repo": name,
"path": str(repo.path),
}
if not repo.path.exists():
return digest({**identity, "state": "missing"})
if (
not repo.path.resolve().is_relative_to(self.workspace_root)
or not (repo.path / ".git").exists()
):
raise ValueError(
"Cannot establish scoped source identity for an escaped/non-Git repository"
)
statistics["git_calls"] += 1
head = git_bytes(
repo.path, "rev-parse", "--verify", "HEAD", allow_failure=True
).strip()
statistics["git_calls"] += 1
# This preserves stage numbers and assume-unchanged/skip-worktree flags,
# while also listing untracked membership in the same bounded process.
index = git_bytes(
repo.path,
"ls-files",
"--stage",
"-v",
"--cached",
"--others",
"--exclude-standard",
"-z",
)
members = set()
for record in index.split(b"\0"):
if not record:
continue
if record.startswith(b"? "):
members.add(record[2:])
else:
match = _INDEX_RECORD.fullmatch(record)
if not match:
raise ValueError(
"Unsupported Git membership record; cannot establish scoped source identity"
)
members.add(match[1])
if len(members) > MAX_REPOSITORY_ENTRIES:
raise ValueError(
"Repository source membership exceeds its bounded entry count"
)
hasher = hashlib.sha256(
canonical(
{
**identity,
"head_sha256": hashlib.sha256(head).hexdigest(),
"index_sha256": hashlib.sha256(index).hexdigest(),
}
)
)
for member in sorted(members):
hasher.update(
canonical(
{
"name_sha256": hashlib.sha256(member).hexdigest(),
"value": self._entry(repo, member, statistics),
}
)
)
hasher.update(b"\0")
return hasher.hexdigest()
def _source_identity(self, names: list[str], identities: dict[str, str]) -> str:
return digest(
{
"version": FINGERPRINT_VERSION,
"workspace_root": str(self.workspace_root),
"repositories": {name: identities[name] for name in names},
}
)
def _observe(self, names: list[str]) -> dict:
statistics = _stats()
identities = {name: self._repository(name, statistics) for name in names}
complete = set(names) == self.repositories.keys()
return {
"schema_version": 1,
"fingerprint_version": FINGERPRINT_VERSION,
"workspace_root": str(self.workspace_root),
"repository_fingerprints": identities,
"observed_source_fingerprint": self._source_identity(names, identities),
"observed_scope": {
"version": 1,
"kind": "workspace" if complete else "repositories",
"repos": names,
},
"complete_workspace": complete,
"scan_stats": statistics,
}
def source_snapshot(self, repos: list[str] | None = None) -> dict:
"""Compare observed source only, without interpreting any stored commands."""
return self._observe(
sorted(self.repositories) if repos is None else self._repo_names(repos)
)
def _tooling_identity(self, statistics: dict) -> str:
package = Path(__file__).resolve().parent
paths = set()
for path in package.rglob("*.py"):
paths.add(path)
if len(paths) > 256:
raise ValueError("Devkit tooling source exceeds its bounded inventory")
paths.update(
{
package.parent / "devkit.py",
package.parent / "project.schema.json",
META_ROOT / "devkit",
}
)
return digest(
{
str(path.relative_to(META_ROOT)): self._file_hash(
path, statistics, tooling=True
).hex()
for path in sorted(paths)
}
)
def snapshot(self, stages: list[dict], *, tooling_fingerprint: str = "") -> dict:
if not isinstance(stages, list) or len(stages) > 512:
raise ValueError("Input snapshot requires a bounded stage list")
if not isinstance(tooling_fingerprint, str) or len(tooling_fingerprint) > 4096:
raise ValueError("Invalid supplied tooling identity")
scopes = {}
for stage in stages:
if not isinstance(stage, dict):
raise ValueError("Input stages must be objects")
if set(stage) & _RUNTIME_FIELDS:
raise ValueError(
"Input identities require freshly planned stages, not mutable receipt/runtime fields"
)
identity = identifier(stage.get("id"))
if identity in scopes:
raise ValueError("Duplicate input stage identity")
scopes[identity] = self.scope(stage)
names = sorted({name for scope in scopes.values() for name in scope["repos"]})
observed = self._observe(names)
tools = digest(
{
"declared_tools": self.project.config.get("tools", {}),
"devkit_source": self._tooling_identity(observed["scan_stats"]),
"supplied_tooling": tooling_fingerprint,
}
)
entries = {}
for stage in stages:
scope = scopes[stage["id"]]
source = self._source_identity(
scope["repos"], observed["repository_fingerprints"]
)
plan = digest(stage)
entries[stage["id"]] = {
"source_fingerprint": source,
"plan_fingerprint": plan,
"scope": scope,
"fingerprint": digest(
{
"version": FINGERPRINT_VERSION,
"source": source,
"plan": plan,
"tooling": tools,
}
),
}
return {**observed, "tooling_fingerprint": tools, "stages": entries}
+547
View File
@@ -0,0 +1,547 @@
"""Append-only, idempotent Gitea evidence notes; preview is entirely offline."""
from __future__ import annotations
from contextlib import ExitStack
from dataclasses import dataclass
import hashlib
import html
import os
from pathlib import Path
import re
import subprocess
import sys
from typing import Any
from urllib.parse import urlsplit
from .common import META_ROOT, atomic_json, digest, identifier, read_json, redact, resource_lock, state_root
from .inputs import FINGERPRINT_VERSION
_GITEA_PATH = str(META_ROOT / "tools/gitea")
if _GITEA_PATH not in sys.path:
sys.path.insert(0, _GITEA_PATH)
from gitea_common import GiteaClient, RepoTarget, _parse_remote, repo_path # noqa: E402
MAX_NOTE_BYTES = 128 * 1024
MAX_TARGETS = 256
MAX_COVERAGE_NOTES = 2048
MARKER_PREFIX = "<!-- govoplan-devkit:evidence:v1:"
STATUSES = {"planned", "pending", "running", "passed", "failed", "timed_out", "skipped", "cancelled", "interrupted", "blocked", "partial", "stale"}
@dataclass(frozen=True)
class NoteTarget:
root: Path
base_url: str
owner: str
repository: str
issue: int
issue_id: int | None = None
@property
def url(self) -> str:
return f"{self.base_url}/{self.owner}/{self.repository}/issues/{self.issue}"
@property
def path(self) -> str:
return repo_path(self.owner, self.repository, f"/issues/{self.issue}")
def record(self) -> dict:
return {"root": str(self.root), "base_url": self.base_url, "owner": self.owner,
"repository": self.repository, "issue": self.issue, "url": self.url,
**({"issue_id": self.issue_id} if self.issue_id is not None else {})}
def register(subparsers) -> None:
parser = subparsers.add_parser("issues", help="Preview or append evidence to exact Gitea issues")
commands = parser.add_subparsers(dest="issues_command", required=True)
note = commands.add_parser("note", help="Append a deduplicated note; offline dry run by default")
note.add_argument("--root", type=Path, help="Target repository root or child directory")
note.add_argument("--issue", type=int)
note.add_argument("--target-plan", type=Path, help="JSON with schema_version=1 and exact root/issue/url targets")
note.add_argument("--remote", default="origin")
note.add_argument("--env-file", type=Path, help="Private GITEA_TOKEN dotenv file; target overrides are ignored")
note.add_argument("--evidence", help="Local run ID or explicit receipt JSON path")
note.add_argument("--key", default="verification", help="Stable note purpose within an evidence run")
note.add_argument("--summary", dest="note_summary", action="append", default=[])
note.add_argument("--next", dest="next_steps", action="append", default=[])
note.add_argument("--body-file", type=Path, help="Additional Markdown, never executed")
note.add_argument("--note-file", type=Path, help="Structured JSON: summary[], next[], body")
note.add_argument("--apply", action="store_true", help="Explicitly authorize serial comment creation")
note.add_argument("--retry-uncertain", action="store_true", help="After reconciliation, explicitly retry a still-unconfirmed earlier POST")
note.set_defaults(handler=handle_note)
def _text(value: Any, *, maximum: int = 16384) -> str:
if not isinstance(value, str) or len(value.encode("utf-8")) > maximum:
raise ValueError("Note/evidence text must be a bounded string")
if any(ord(character) < 32 and character not in "\n\t\r" for character in value):
raise ValueError("Note/evidence text contains control characters")
if MARKER_PREFIX in value:
raise ValueError("Evidence markers are reserved for the publisher")
return redact(value)
def _positive(value: Any) -> int:
if type(value) is not int or value <= 0:
raise ValueError("Issue and comment identities must be positive integers")
return value
def _base_url(value: str) -> str:
parsed = urlsplit(value)
if (parsed.scheme not in {"http", "https"} or not parsed.hostname or parsed.username
or parsed.password or parsed.query or parsed.fragment or "\\" in value
or any(ord(char) < 33 for char in value)):
raise ValueError("Gitea target must be a credential-free HTTP(S) URL")
if any(part in {".", ".."} or "%" in part for part in parsed.path.split("/")):
raise ValueError("Gitea base URL contains an ambiguous path")
try:
parsed.port
except ValueError as exc:
raise ValueError("Invalid Gitea port") from exc
return value.rstrip("/")
def _name(value: str) -> str:
if not isinstance(value, str) or not re.fullmatch(r"[A-Za-z0-9_.-]+", value) or value in {".", ".."}:
raise ValueError("Invalid Gitea owner or repository name")
return value
def resolve_target(root: Path, issue: int, workspace_root: Path, *, remote: str = "origin",
expected_url: str | None = None, issue_id: int | None = None) -> NoteTarget:
requested = root if root.is_absolute() else workspace_root / root
resolved = requested.resolve()
if not resolved.is_relative_to(workspace_root.resolve()) or not resolved.is_dir():
raise ValueError("Issue target must be an existing repository inside the selected workspace")
if not re.fullmatch(r"[A-Za-z0-9_.-]+", remote):
raise ValueError("Invalid Git remote name")
command = subprocess.run(["git", "-C", str(resolved), "rev-parse", "--show-toplevel"],
capture_output=True, text=True, timeout=15)
if command.returncode:
raise ValueError("Issue target is not a Git checkout")
actual = Path(command.stdout.strip()).resolve()
if not actual.is_relative_to(workspace_root.resolve()):
raise ValueError("Resolved issue repository escapes the workspace")
result = subprocess.run(["git", "-C", str(actual), "remote", "get-url", remote],
capture_output=True, text=True, timeout=15)
if result.returncode or len(result.stdout) > 8192:
raise ValueError("Target repository has no usable Git remote")
remote_url = result.stdout.strip()
parsed = urlsplit(remote_url)
if (any(ord(char) < 33 for char in remote_url) or "\\" in remote_url
or parsed.query or parsed.fragment):
raise ValueError("Ambiguous Git remote URL cannot bind an issue target")
if parsed.scheme in {"http", "https"} and (parsed.username or parsed.password):
raise ValueError("Credential-bearing Git remotes cannot be used for issue publishing")
base, owner, repository = _parse_remote(remote_url)
target = NoteTarget(actual, _base_url(base), _name(owner), _name(repository), _positive(issue),
_positive(issue_id) if issue_id is not None else None)
if expected_url is not None and expected_url != target.url:
raise ValueError("Target plan issue URL does not match the exact repository remote and issue number")
return target
def _targets(args) -> list[NoteTarget]:
workspace_root = Path(args.workspace_root).resolve()
if args.target_plan:
if args.root is not None or args.issue is not None:
raise ValueError("Use either --root/--issue or --target-plan")
payload = read_json(args.target_plan, max_bytes=1024 * 1024)
if not isinstance(payload, dict) or type(payload.get("schema_version")) is not int or payload["schema_version"] != 1:
raise ValueError("Target plan requires schema_version 1")
records = payload.get("targets")
if not isinstance(records, list) or not 1 <= len(records) <= MAX_TARGETS:
raise ValueError("Target plan must contain 1256 exact targets")
targets = []
for record in records:
if (not isinstance(record, dict) or not isinstance(record.get("root"), str)
or not isinstance(record.get("url"), str)):
raise ValueError("Each target plan entry requires root, issue and URL")
target = resolve_target(Path(record["root"]), record.get("issue"), workspace_root,
remote=args.remote, expected_url=record["url"], issue_id=record.get("issue_id"))
for key in ("base_url", "owner", "repository"):
if key in record and record[key] != getattr(target, key):
raise ValueError("Target plan repository identity is inconsistent")
targets.append(target)
else:
if args.root is None or args.issue is None:
raise ValueError("An exact --root and --issue are required")
targets = [resolve_target(args.root, args.issue, workspace_root, remote=args.remote)]
if len({target.url for target in targets}) != len(targets):
raise ValueError("Duplicate issue targets are not accepted")
if len({target.base_url for target in targets}) != 1:
raise ValueError("A credentialed target plan must be confined to one exact Gitea base URL")
return targets
def validate_receipt(payload: Any, workspace_root: Path) -> dict:
if not isinstance(payload, dict) or type(payload.get("schema_version")) is not int or payload["schema_version"] != 1:
raise ValueError("Evidence requires receipt schema_version 1")
identifier(payload.get("run_id"))
recorded_workspace = payload.get("workspace_root")
if not isinstance(recorded_workspace, str) or Path(recorded_workspace).resolve() != workspace_root.resolve():
raise ValueError("Evidence belongs to another workspace")
fingerprint = payload.get("source_fingerprint")
if not isinstance(fingerprint, str) or not re.fullmatch(r"[a-f0-9]{64}", fingerprint):
raise ValueError("Evidence requires a recorded source fingerprint")
if not isinstance(payload.get("status"), str) or payload["status"] not in STATUSES:
raise ValueError("Unknown evidence status")
_text(payload.get("generated_at"), maximum=128)
if payload.get("finished_at") is not None:
_text(payload["finished_at"], maximum=128)
stages = payload.get("stages")
if not isinstance(stages, list) or len(stages) > 1000:
raise ValueError("Evidence requires a bounded stage list")
seen = set()
for stage in stages:
if not isinstance(stage, dict):
raise ValueError("Invalid evidence stage")
identity = _text(stage.get("id"), maximum=256)
if not identity or identity in seen or "\n" in identity:
raise ValueError("Evidence stage IDs must be unique")
seen.add(identity)
if not isinstance(stage.get("status"), str) or stage["status"] not in STATUSES:
raise ValueError("Unknown evidence stage status")
code = stage.get("exit_code")
if code is not None and type(code) is not int:
raise ValueError("Evidence exit code must be an integer or null")
duration = stage.get("duration_seconds")
if duration is not None and (type(duration) not in {int, float} or not 0 <= duration < 31536000):
raise ValueError("Invalid evidence stage duration")
if stage.get("log_path") is not None:
_text(stage["log_path"], maximum=4096)
if stage["status"] == "passed" and code != 0:
raise ValueError("Passed evidence stage must have exit code zero")
if payload["status"] == "passed" and (not stages or any(stage["status"] != "passed" for stage in stages)):
raise ValueError("Passed evidence is inconsistent with its stages")
if payload["status"] == "passed" and payload.get("snapshot_verified") is not True:
raise ValueError("Passed evidence requires a verified source snapshot")
coverage_notes(stages)
from .checkpoints import validate_checkpoint_receipt
validate_checkpoint_receipt(payload)
return payload
def coverage_notes(stages: list[dict]) -> list[str]:
"""Keep declared coverage limits visible without interpreting them as commands."""
notes, total, size = [], 0, 0
for stage in stages:
values = stage.get("coverage_notes", [])
if not isinstance(values, list):
raise ValueError("Evidence coverage_notes must be a bounded list of strings")
total += len(values)
if total > MAX_COVERAGE_NOTES:
raise ValueError("Evidence coverage_notes exceed their count bound")
for value in values:
text = _text(value, maximum=4096).strip()
if not text:
raise ValueError("Evidence coverage notes cannot be empty")
size += len(text.encode())
if size > MAX_NOTE_BYTES:
raise ValueError("Evidence coverage_notes exceed their size bound")
notes.append(text)
return list(dict.fromkeys(notes))
def evidence_record(value: str | None, args) -> dict | None:
if not value:
return None
workspace_root = Path(args.workspace_root).resolve()
external = value.endswith(".json") or "/" in value or "\\" in value
if external:
path = Path(value).expanduser().absolute()
payload = read_json(path)
origin = "external-unverified"
else:
from .runner import read_receipt
identity = identifier(value)
payload = read_receipt(workspace_root, args.state_dir, identity)
path = state_root(workspace_root, args.state_dir) / "runs" / identity / "receipt.json"
origin = "local-integrity-checked"
receipt = validate_receipt(payload, workspace_root)
from .workspace import load_project, source_fingerprint
expected_project = str(args.project.resolve()) if getattr(args, "project", None) else None
source_state = "not-compared"
current = None
if receipt.get("project_file") == expected_project:
try:
project = load_project(workspace_root, getattr(args, "project", None))
if receipt.get("fingerprint_version") == FINGERPRINT_VERSION:
from .inputs import InputSnapshotter
scope = receipt.get("source_scope", {})
if not isinstance(scope, dict) or not isinstance(scope.get("repos"), list):
raise ValueError("Scoped evidence requires an explicit recorded repository scope")
current = InputSnapshotter(project, workspace_root=workspace_root).source_snapshot(scope["repos"])["observed_source_fingerprint"]
else:
current = source_fingerprint(project)
source_state = "matches-current" if current == receipt["source_fingerprint"] else "historical-source-differs"
except (OSError, ValueError, subprocess.SubprocessError):
source_state = "current-source-unavailable"
else:
source_state = "different-project-not-compared"
return {"run_id": redact(receipt["run_id"]), "status": receipt["status"], "origin": origin,
"receipt_path": redact(str(path)), "source_fingerprint": receipt["source_fingerprint"],
"current_source_fingerprint": current, "source_state": source_state,
"snapshot_verified": receipt.get("snapshot_verified") is True,
"coverage_notes": coverage_notes(receipt["stages"]) + ([
"Source comparison is limited to the recorded repository input scope; this is not a whole-workspace or artifact verification."
] if receipt.get("fingerprint_version") == FINGERPRINT_VERSION and not receipt.get("source_scope", {}).get("complete_workspace") else []),
"fingerprint_version": receipt.get("fingerprint_version"),
"source_scope": receipt.get("source_scope"),
"generated_at": redact(receipt["generated_at"]), "finished_at": redact(receipt["finished_at"]) if receipt.get("finished_at") else None,
"stages": [{**{key: redact(stage[key]) if isinstance(stage.get(key), str) else stage.get(key)
for key in ("id", "status", "exit_code", "duration_seconds", "log_path")},
"coverage_notes": coverage_notes([stage])} for stage in receipt["stages"]],
"attestation": "Receipt metadata is not an independent attestation, a live verification, or a completed module review."}
def _structured_note(args) -> dict:
payload = read_json(args.note_file, max_bytes=MAX_NOTE_BYTES) if args.note_file else {}
if not isinstance(payload, dict) or set(payload) - {"summary", "next", "body"}:
raise ValueError("Structured note accepts only summary[], next[] and body")
result = {}
for key, values in (("summary", args.note_summary), ("next", args.next_steps)):
supplied = payload.get(key, [])
if not isinstance(supplied, list) or len(supplied) + len(values) > 100:
raise ValueError("Summary and next steps must be bounded lists")
result[key] = [_text(item).strip() for item in [*supplied, *values]]
body = _text(payload.get("body", ""), maximum=MAX_NOTE_BYTES)
if args.body_file:
# Reuse the bounded, no-symlink file reader by wrapping no content in code.
from .common import reject_symlinks
import stat
reject_symlinks(args.body_file)
descriptor = os.open(args.body_file, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0))
with os.fdopen(descriptor, "rb") as handle:
metadata = os.fstat(handle.fileno())
if not stat.S_ISREG(metadata.st_mode) or metadata.st_size > MAX_NOTE_BYTES:
raise ValueError("Additional body must be a bounded regular file")
text = handle.read(MAX_NOTE_BYTES + 1).decode("utf-8")
body += "\n\n" + _text(text, maximum=MAX_NOTE_BYTES)
result["body"] = body.strip()
return result
def _cell(value: Any) -> str:
return html.escape(str(value if value is not None else "")).replace("|", "&#124;").replace("`", "&#96;").replace("\n", " ")
def render_note(note: dict, evidence: dict | None, target: NoteTarget, key: str) -> tuple[str, str]:
identity = {"target": target.url, "key": identifier(key),
"evidence": evidence["run_id"] if evidence else digest(note)}
marker = MARKER_PREFIX + digest(identity) + " -->"
lines = [marker, "## Development evidence", "", f"Issue: {target.url}", ""]
for field, title in (("summary", "Summary"), ("next", "Next / remaining")):
if note[field]:
lines += [f"### {title}", "", *["- " + item for item in note[field]], ""]
if note["body"]:
lines += [note["body"], ""]
if evidence:
lines += ["### Recorded check evidence", "", f"Run: `{evidence['run_id']}`; reported result: **{evidence['status']}**.",
f"Receipt origin: `{evidence['origin']}`; source comparison: `{evidence['source_state']}`.",
f"Recorded source fingerprint: `{evidence['source_fingerprint']}`.",
f"Finished: {_cell(evidence['finished_at'])}; receipt: `{_cell(evidence['receipt_path'])}`.", "",
"| Stage | Reported status | Exit | Seconds | Local log reference |", "| --- | --- | --- | --- | --- |"]
for stage in evidence["stages"]:
lines.append("| " + " | ".join(_cell(stage.get(field)) for field in ("id", "status", "exit_code", "duration_seconds", "log_path")) + " |")
if evidence.get("coverage_notes"):
lines += ["", "### Coverage limitations / checks not included", "",
"A passing recorded stage does not mean these omitted checks ran.", "",
*["- " + _cell(value) for value in evidence["coverage_notes"]]]
lines += ["", evidence["attestation"], "Local logs are referenced only; their content has not been read or uploaded.", ""]
lines += ["This comment does not close the issue, complete its review, or change its checklist."]
body = redact("\n".join(lines).rstrip() + "\n")
if len(body.encode()) > MAX_NOTE_BYTES:
raise ValueError("Rendered note exceeds its size bound")
return marker, body
def _token(env_file: Path | None) -> str:
values = {}
if env_file is not None:
from .common import reject_symlinks
reject_symlinks(env_file)
import stat
descriptor = os.open(env_file, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0))
with os.fdopen(descriptor, "rb") as handle:
metadata = os.fstat(handle.fileno())
if not stat.S_ISREG(metadata.st_mode) or metadata.st_size > 65536:
raise ValueError("Credential file must be a bounded regular file")
content = handle.read(65537)
if len(content) > 65536:
raise ValueError("Credential file exceeds its size limit")
for line in content.decode("utf-8").splitlines():
line = line.strip()
if not line or line.startswith("#"):
continue
if line.startswith("export "):
line = line[7:].strip()
key, separator, value = line.partition("=")
if not separator or not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", key.strip()):
raise ValueError("Invalid credential file format")
value = value.strip()
if len(value) >= 2 and value[0] == value[-1] and value[0] in {"'", '"'}:
value = value[1:-1]
if key.strip() in values:
raise ValueError("Duplicate credential file keys")
values[key.strip()] = value
token = os.environ.get("GITEA_TOKEN") or values.get("GITEA_TOKEN")
if not token or len(token) > 8192 or any(char.isspace() for char in token):
raise ValueError("GITEA_TOKEN is required for --apply; use the environment or --env-file")
return token
def make_client(target: NoteTarget, token: str):
return GiteaClient(RepoTarget(target.base_url, target.owner, target.repository), token)
def _issue(client, target: NoteTarget) -> dict:
issue = client.request_json("GET", target.path)
if (not isinstance(issue, dict) or issue.get("number") != target.issue or issue.get("html_url") != target.url
or issue.get("pull_request") is not None):
raise ValueError("Remote issue identity does not match the exact target")
_positive(issue.get("id"))
if target.issue_id is not None and issue["id"] != target.issue_id:
raise ValueError("Remote issue ID changed from the target plan")
return issue
def _comments(client, target: NoteTarget) -> list[dict]:
comments, seen = [], set()
for page in range(1, 10001):
values = client.request_json("GET", target.path + "/comments", query={"page": page, "limit": 50})
if not isinstance(values, list):
raise ValueError("Remote comment pagination did not return a list")
if not values:
return comments
for comment in values:
if not isinstance(comment, dict) or not isinstance(comment.get("body"), str):
raise ValueError("Remote comment has an invalid shape")
identity = _positive(comment.get("id"))
if identity in seen:
raise ValueError("Repeated comment pagination; cannot establish a complete duplicate check")
seen.add(identity)
comments.append(comment)
if len(comments) > 100000:
raise ValueError("Remote comments exceed the bounded duplicate-check limit")
raise ValueError("Remote comment pagination did not terminate")
def _existing(comments: list[dict], marker: str, body: str) -> dict | None:
matches = [comment for comment in comments if marker in comment["body"]]
if len(matches) > 1 or (matches and matches[0]["body"] != body):
raise ValueError("Evidence marker collision; existing comments are preserved")
return matches[0] if matches else None
def _readback(client, target: NoteTarget, comment: dict, body: str) -> dict:
identity = _positive(comment.get("id"))
fresh = client.request_json("GET", repo_path(target.owner, target.repository, f"/issues/comments/{identity}"))
issue_api_url = target.base_url + "/api/v1" + target.path
if not isinstance(fresh, dict) or fresh.get("id") != identity or fresh.get("body") != body:
raise ValueError("Posted comment read-back did not match")
if not fresh.get("html_url") and not fresh.get("issue_url"):
raise ValueError("Comment read-back has no issue binding")
if fresh.get("issue_url") and fresh["issue_url"] != issue_api_url:
raise ValueError("Comment read-back belongs to another issue")
if fresh.get("html_url") and fresh["html_url"].split("#", 1)[0] != target.url:
raise ValueError("Comment read-back URL belongs to another issue")
return fresh
def handle_note(args) -> dict:
targets = _targets(args)
note = _structured_note(args)
evidence = evidence_record(args.evidence, args)
if not evidence and not any((note["summary"], note["next"], note["body"])):
raise ValueError("Provide evidence or a nonempty structured note")
if args.retry_uncertain and not args.apply:
raise ValueError("--retry-uncertain requires --apply")
prepared = [(target, *render_note(note, evidence, target, args.key)) for target in targets]
result = {"schema_version": 1, "operation": "issues.note", "apply": bool(args.apply),
"evidence": evidence, "targets": [{**target.record(), "marker": marker, "body": body, "status": "would-post"} for target, marker, body in prepared],
"summary": [f"{'Apply' if args.apply else 'Offline dry run'}: {len(targets)} exact issue target(s); issue bodies and states are preserved."]}
if evidence and evidence["coverage_notes"]:
result["summary"].append(f"Evidence has {len(evidence['coverage_notes'])} coverage limitation(s), retained in the note; omitted checks are not claimed as passed.")
if not args.apply:
return result
token = _token(args.env_file)
# Tokens loaded from an explicit file are not inserted into the process environment.
for record in result["targets"]:
if token in record["body"]:
raise ValueError("A credential occurs in note content; refusing publication")
state = state_root(Path(args.workspace_root), args.state_dir)
with ExitStack() as stack:
for target, marker, _body in sorted(prepared, key=lambda item: item[0].url):
stack.enter_context(resource_lock(state / "locks", "issues.note:" + marker))
clients, bindings, journals = {}, {}, {}
try:
# Validate every target and every existing marker before the first write.
for target, marker, body in prepared:
client = make_client(target, token)
stack.callback(client.close)
clients[target.url] = client
bindings[target.url] = _issue(client, target)["id"]
_existing(_comments(client, target), marker, body)
journal_path = state / "issue-notes" / (hashlib.sha256(marker.encode()).hexdigest() + ".json")
prior = read_json(journal_path) if journal_path.exists() else None
if prior is not None and (not isinstance(prior, dict) or prior.get("schema_version") != 1
or prior.get("target") != target.url or prior.get("body_digest") != digest(body)
or prior.get("issue_id") != bindings[target.url]
or prior.get("status") not in {"posting", "uncertain", "verified"}):
raise ValueError("Local evidence journal identity collision; inspect before retrying")
journals[target.url] = (journal_path, prior)
for index, (target, marker, body) in enumerate(prepared):
client = clients[target.url]
output = result["targets"][index]
journal_path, prior = journals[target.url]
issue = _issue(client, target)
if issue["id"] != bindings[target.url]:
raise ValueError("Issue identity changed after preflight")
existing = _existing(_comments(client, target), marker, body)
if existing:
verified = _readback(client, target, existing, body)
output.update(status="existing-verified", comment_id=verified["id"])
atomic_json(journal_path, {"schema_version": 1, "target": target.url, "issue_id": issue["id"], "body_digest": digest(body), "status": "verified", "comment_id": verified["id"]})
continue
if prior and prior.get("status") in {"uncertain", "posting", "verified"} and not args.retry_uncertain:
output["status"] = "uncertain-retry-required"
result["_exit_code"] = 2
result["summary"].append("An earlier POST is not visible after complete reconciliation; explicit --retry-uncertain is required. No further posts attempted.")
break
journal = {"schema_version": 1, "target": target.url, "issue_id": issue["id"], "body_digest": digest(body), "status": "posting"}
atomic_json(journal_path, journal)
try:
posted = client.request_json("POST", target.path + "/comments", body={"body": body})
verified = _readback(client, target, posted, body)
unique = _existing(_comments(client, target), marker, body)
if unique is None or unique["id"] != verified["id"]:
raise ValueError("New comment is not uniquely visible in its issue")
output.update(status="posted-verified", comment_id=verified["id"])
except Exception:
# POST is never replayed automatically, even after a timeout or bad response.
atomic_json(journal_path, {**journal, "status": "uncertain"})
try:
observed = _existing(_comments(client, target), marker, body)
verified = _readback(client, target, observed, body) if observed else None
except Exception:
verified = None
if verified is None:
output["status"] = "uncertain"
result["_exit_code"] = 2
result["summary"].append("POST/read-back could not be confirmed. No automatic retry and no further posts; reconcile this target before continuing.")
break
output.update(status="reconciled-verified", comment_id=verified["id"])
atomic_json(journal_path, {**journal, "status": "verified", "comment_id": output["comment_id"]})
except Exception:
# HTTP response/error bodies may contain secrets; never return them to a report.
result["_exit_code"] = 2
result["summary"].append("Gitea validation or local journal checks failed; existing issues/comments were preserved. No further posts attempted.")
finally:
token = ""
for record in result["targets"]:
if record["status"] == "would-post":
record["status"] = "not-attempted"
result["summary"].append("; ".join(f"{record['url']}: {record['status']}" for record in result["targets"]))
return result
File diff suppressed because it is too large Load Diff
+251
View File
@@ -0,0 +1,251 @@
"""Read-only run discovery and bounded provisional output, never passing evidence."""
from __future__ import annotations
from datetime import datetime
import os
from pathlib import Path
import re
import time
from .common import (
atomic_json,
identifier,
now,
read_json,
redact,
reject_symlinks,
state_root,
)
MAX_LIVE_BYTES = 65536
MAX_HISTORY_ENTRIES = 10000
MAX_HISTORY_SCAN = 500
ANSI = re.compile(r"\x1b(?:\[[0-?]*[ -/]*[@-~]|\][^\x07\x1b]*(?:\x07|\x1b\\))")
CONTROLS = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f]")
def display_text(value: str) -> str:
return redact(CONTROLS.sub("", ANSI.sub("", value)))
def _closed_lines(value: bytes) -> bytes:
boundary = value.rfind(b"\n")
return value[: boundary + 1] if boundary >= 0 else b""
def capture_text(snapshot, *, provisional: bool = False) -> str:
"""Do not expose partial secret/context fragments at live or retention boundaries."""
data = snapshot.stdout
omitted = snapshot.omitted_stdout_bytes
if omitted:
head = _closed_lines(data[: snapshot.stdout_head_bytes])
tail = data[snapshot.stdout_head_bytes :]
# A rolling tail can start inside Authorization or a secret: drop that fragment.
tail = tail.partition(b"\n")[2]
if provisional:
tail = _closed_lines(tail)
return (
display_text(head.decode("utf-8", errors="replace"))
+ f"\n[Output truncated: {omitted} bytes omitted between retained head and tail; cut boundary lines are withheld.]\n"
+ display_text(tail.decode("utf-8", errors="replace"))
)
if provisional:
data = _closed_lines(data)
return display_text(data.decode("utf-8", errors="replace"))
def bounded_display(value: str, maximum: int, *, tail_only: bool = False) -> str:
if type(maximum) is not int or maximum < 1:
raise ValueError("Display bound must be a positive integer")
encoded = value.encode("utf-8")
if len(encoded) <= maximum:
return value
if tail_only:
return encoded[-maximum:].decode("utf-8", errors="ignore")
marker = b"\n[Display bound: middle omitted; retained beginning and final output follow.]\n"
if maximum <= len(marker):
return encoded[-maximum:].decode("utf-8", errors="ignore")
available = max(0, maximum - len(marker))
head = available // 2
return (
encoded[:head].decode("utf-8", errors="ignore")
+ marker.decode()
+ encoded[-(available - head) :].decode("utf-8", errors="ignore")
)
def write_live(log_path: Path, stage_id: str, snapshot, started: float) -> None:
# Complete lines only, even at final callback: the separately finalized log
# may include an unterminated final line after normal redaction.
excerpt = bounded_display(
capture_text(snapshot, provisional=True), MAX_LIVE_BYTES, tail_only=True
)
atomic_json(
log_path.with_suffix(".live.json"),
{
"schema_version": 1,
"run_id": log_path.parent.name,
"stage_id": stage_id,
"provisional": True,
"updated_at": now(),
"elapsed_seconds": round(time.monotonic() - started, 3),
"output_truncated": snapshot.truncated,
"omitted_bytes": snapshot.omitted_stdout_bytes,
"excerpt": excerpt,
},
)
def read_live(
workspace_root: Path, state_dir: Path | None, run_id: str, stage_id: str
) -> dict:
identifier(run_id)
identifier(stage_id)
path = (
state_root(workspace_root, state_dir)
/ "runs"
/ run_id
/ (stage_id + ".live.json")
)
if not path.exists() and not path.is_symlink():
return {"provisional": True, "excerpt": "", "live_available": False}
payload = read_json(path, max_bytes=MAX_LIVE_BYTES * 6 + 8192)
if (
not isinstance(payload, dict)
or payload.get("schema_version") != 1
or payload.get("run_id") != run_id
or payload.get("stage_id") != stage_id
or payload.get("provisional") is not True
or not isinstance(payload.get("excerpt"), str)
or len(payload["excerpt"].encode("utf-8")) > MAX_LIVE_BYTES
or not isinstance(payload.get("updated_at"), str)
or type(payload.get("elapsed_seconds")) not in {int, float}
or payload["elapsed_seconds"] < 0
or type(payload.get("output_truncated")) is not bool
):
raise ValueError("Invalid provisional log snapshot")
return {
"provisional": True,
"live_available": True,
"excerpt": display_text(payload["excerpt"]),
"updated_at": payload.get("updated_at"),
"elapsed_seconds": payload.get("elapsed_seconds"),
"output_truncated": payload.get("output_truncated"),
}
def elapsed_seconds(receipt: dict) -> float | None:
try:
started = datetime.fromisoformat(receipt["generated_at"])
finished = datetime.fromisoformat(receipt.get("finished_at") or now())
return max(0, round((finished - started).total_seconds(), 3))
except (KeyError, TypeError, ValueError):
return None
def list_runs(args) -> dict:
from .runner import read_receipt
limit = getattr(args, "limit", 10)
before = getattr(args, "before", None)
if type(limit) is not int or not 1 <= limit <= 100:
raise ValueError("Run history limit must be between 1 and 100")
if before:
identifier(before)
base = state_root(args.workspace_root, args.state_dir) / "runs"
reject_symlinks(base)
names = []
if base.exists():
with os.scandir(base) as entries:
for index, entry in enumerate(entries):
if index >= MAX_HISTORY_ENTRIES:
raise ValueError(
"Run history exceeds its directory bound; archive old evidence explicitly before listing"
)
if not entry.name.startswith("."):
identifier(entry.name)
names.append(entry.name)
names = sorted(
(name for name in names if before is None or name < before), reverse=True
)
rows, examined, cursor = [], 0, None
wanted_project = (
str(args.project.resolve()) if getattr(args, "project", None) else None
)
for identity in names:
examined += 1
cursor = identity
try:
record = read_receipt(args.workspace_root, args.state_dir, identity)
if wanted_project and record.get("project_file") != wanted_project:
if examined >= MAX_HISTORY_SCAN:
break
continue
rows.append(
{
"run_id": identity,
"status": record["status"],
"phase": record.get("phase"),
"profile": record.get("profile"),
"generated_at": record.get("generated_at"),
"elapsed_seconds": elapsed_seconds(record),
"snapshot_verified": record["snapshot_verified"],
"passed_stages": sum(
item["status"] == "passed" for item in record["stages"]
),
"total_stages": len(record["stages"]),
}
)
except (OSError, ValueError) as exc:
# Corrupt/newest evidence is visible, never silently replaced with an older pass.
rows.append(
{
"run_id": display_text(identity),
"status": "invalid",
"error": display_text(str(exc)),
}
)
if len(rows) >= limit or examined >= MAX_HISTORY_SCAN:
break
next_cursor = cursor if examined < len(names) else None
lines = [
f"{item['run_id']}: {item['status']}"
+ (
f" ({item.get('profile')}; {item.get('elapsed_seconds')}s)"
if item["status"] != "invalid"
else "" + item["error"]
)
for item in rows
]
if not lines:
lines = ["No matching check runs found; no verification is implied."]
if next_cursor:
lines.append("More history: use --before " + display_text(next_cursor))
return {
"runs": rows,
"next_cursor": next_cursor,
"examined": examined,
"summary": lines,
"_exit_code": 1 if any(item["status"] == "invalid" for item in rows) else 0,
}
def latest_run(args) -> dict:
from argparse import Namespace
from .runner import read_receipt, summarize
selected = list_runs(Namespace(**{**vars(args), "limit": 1}))
if not selected["runs"]:
return {
"status": "not_found",
"summary": selected["summary"],
"next_cursor": selected["next_cursor"],
"_exit_code": 1,
}
row = selected["runs"][0]
if row["status"] == "invalid":
raise ValueError(
"Latest run is invalid; inspect run history instead of assuming an older pass"
)
return summarize(read_receipt(args.workspace_root, args.state_dir, row["run_id"]))
+173
View File
@@ -0,0 +1,173 @@
"""Bounded package metadata and conservative direct-Node test discovery."""
from __future__ import annotations
from pathlib import Path
import re
import shlex
from .common import digest, read_json, redact_argv, reject_symlinks
MAX_PACKAGE_BYTES = 1024 * 1024
MAX_SCRIPTS = 512
CORE_COMPONENT_SUITES = (
"data-grid-actions",
"dialog-focus",
"explorer-tree",
"icon-button",
"layout-primitives",
"mail-components",
"metric-card",
"page-layout",
"workspace-layout",
"people-picker",
"password-field",
"resource-access",
"action-blocker",
"documentation-help",
"selection-list",
"wysiwyg-editor",
)
CORE_RUNNER = "scripts/run-component-tests.mjs"
def read_package(path: Path) -> dict:
try:
package = read_json(path, max_bytes=MAX_PACKAGE_BYTES)
except (OSError, ValueError) as exc:
raise ValueError(f"Cannot safely read package metadata: {path}") from exc
if not isinstance(package, dict):
raise ValueError(f"Package metadata must be an object: {path}")
scripts = package.get("scripts", {})
if not isinstance(scripts, dict) or len(scripts) > MAX_SCRIPTS:
raise ValueError(f"Package scripts must be a bounded object: {path}")
for name, command in scripts.items():
if (
not isinstance(name, str)
or not 1 <= len(name) <= 128
or any(char in name for char in "\0\r\n")
):
raise ValueError(f"Package script names must be bounded strings: {path}")
if (
not isinstance(command, str)
or not 1 <= len(command) <= 8192
or "\0" in command
):
raise ValueError(
f"Package script commands must be bounded nonempty strings: {path}"
)
return package
def core_component_alias(
repo_name: str, name: str, command: str, package_path: Path
) -> str | None:
if repo_name != "govoplan-core" or package_path.parent.name != "webui":
return None
if name == "test:components" and command == f"node {CORE_RUNNER}":
return "all"
for suite in CORE_COMPONENT_SUITES:
if name == "test:" + suite and command == f"node {CORE_RUNNER} {suite}":
return suite
return None
def direct_node(command: str, package_path: Path) -> tuple[list[str] | None, str]:
try:
parts = shlex.split(command)
except ValueError:
return None, "Malformed command quoting; no command was guessed or executed."
if not parts or parts[0] != "node":
return (
None,
"Not a direct Node source test; use its explicitly reviewed owning workflow.",
)
offset = 2 if len(parts) > 1 and parts[1] == "--test" else 1
if len(parts) != offset + 1 or not re.fullmatch(
r"(?:scripts|tests)/[A-Za-z0-9_.-]+\.mjs", parts[offset]
):
return (
None,
"Compound command, flags or arguments are unsupported by scoped source discovery.",
)
if parts[offset] == CORE_RUNNER:
return (
None,
"Only exact known Core component aliases belong to the shared UI batch.",
)
target = package_path.parent / parts[offset]
try:
reject_symlinks(target)
if not target.is_file() or not target.resolve().is_relative_to(
package_path.parent.resolve()
):
return None, "Declared test target is missing or escapes its package."
except (OSError, ValueError):
return None, "Declared test target is not a safe regular package file."
return [
"{node}",
*parts[1:offset],
str(target),
], "Direct package-owned source test."
def declared_tests(repo, package_path: Path) -> list[dict]:
package = read_package(package_path)
result = []
for name, command in sorted(package.get("scripts", {}).items()):
if name != "test" and not name.startswith("test:"):
continue
component = core_component_alias(repo.name, name, command, package_path)
argv, reason = (
direct_node(command, package_path)
if component is None
else (None, "Exact known Core component alias.")
)
result.append(
{
"repo": repo.name,
"package_path": str(package_path),
"name": name,
"command_sha256": digest(command),
"argv": redact_argv(argv) if argv else None,
"component_suite": component,
"reason": reason,
"_argv": argv,
}
)
return result
def discovered_sources(repo) -> list[dict]:
webui = repo.path / "webui"
found = {}
for folder in ("scripts", "tests"):
for pattern in ("test-interface-pattern*.mjs", "*structure*.mjs"):
for target in (webui / folder).glob(pattern):
command = (
"node "
+ ("--test " if target.name.endswith(".test.mjs") else "")
+ target.relative_to(webui).as_posix()
)
argv, reason = direct_node(command, webui / "package.json")
found[str(target)] = {
"repo": repo.name,
"package_path": str(webui / "package.json"),
"name": "file:" + target.relative_to(webui).as_posix(),
"discovered": True,
"command_sha256": digest(command),
"argv": redact_argv(argv) if argv else None,
"component_suite": None,
"reason": reason,
"_argv": argv,
}
if len(found) > MAX_SCRIPTS:
raise ValueError(
f"Too many discovered source checks in {repo.name}"
)
return [found[key] for key in sorted(found)]
def source_stage_id(repo_name: str, argv: list[str]) -> str:
stem = Path(argv[-1]).stem
return f"{repo_name}.{stem}"[:110] + "." + digest(argv)[:12]
+300
View File
@@ -0,0 +1,300 @@
"""Bounded subprocess capture shared by execution, discovery and environment probes."""
from __future__ import annotations
from dataclasses import dataclass
import os
from pathlib import Path
import selectors
import signal
import subprocess
import threading
import time
from typing import Callable
@dataclass(frozen=True)
class OutputSnapshot:
"""Immutable bounded output; raw bytes are decoded only at the display boundary."""
stdout: bytes
stderr: bytes
truncated: bool
omitted_stdout_bytes: int
omitted_stderr_bytes: int
stdout_head_bytes: int
stderr_head_bytes: int
final: bool
def text(self, stream: str = "stdout") -> str:
"""Render each retained segment separately, never joining cut UTF-8 sequences.
This is not secret redaction. Callers must redact before publishing or
persisting a snapshot, including withholding partial live lines as needed.
"""
if stream not in {"stdout", "stderr"}:
raise ValueError("Output stream must be stdout or stderr")
data = getattr(self, stream)
omitted = getattr(self, "omitted_" + stream + "_bytes")
if not omitted:
return data.decode("utf-8", errors="replace")
split = getattr(self, stream + "_head_bytes")
return (
data[:split].decode("utf-8", errors="replace")
+ f"\n[{omitted} output bytes omitted between retained segments]\n"
+ data[split:].decode("utf-8", errors="replace")
)
@dataclass(frozen=True)
class Capture:
stdout: bytes
stderr: bytes
returncode: int
status: str
truncated: bool
omitted_stdout_bytes: int = 0
omitted_stderr_bytes: int = 0
stdout_head_bytes: int = 0
stderr_head_bytes: int = 0
def snapshot(self) -> OutputSnapshot:
return OutputSnapshot(
self.stdout,
self.stderr,
self.truncated,
self.omitted_stdout_bytes,
self.omitted_stderr_bytes,
self.stdout_head_bytes,
self.stderr_head_bytes,
True,
)
class _OutputBuffer:
def __init__(self, limit: int, mode: str):
self.limit, self.mode, self.total = limit, mode, 0
self.head, self.tail = bytearray(), bytearray()
self.head_limit = limit if mode == "prefix" else limit // 2
self.tail_limit = limit - self.head_limit
def append(self, data: bytes) -> None:
self.total += len(data)
count = min(len(data), self.head_limit - len(self.head))
self.head.extend(data[:count])
remaining = data[count:]
if self.tail_limit and remaining:
if len(remaining) >= self.tail_limit:
self.tail[:] = remaining[-self.tail_limit :]
else:
excess = max(0, len(self.tail) + len(remaining) - self.tail_limit)
del self.tail[:excess]
self.tail.extend(remaining)
@property
def omitted(self) -> int:
return self.total - len(self.head) - len(self.tail)
def value(self) -> bytes:
return bytes(self.head) + bytes(self.tail)
def group_exists(pid: int) -> bool:
try:
os.killpg(pid, 0)
return True
except ProcessLookupError:
return False
def stop_group(process: subprocess.Popen) -> None:
try:
os.killpg(process.pid, signal.SIGTERM)
except ProcessLookupError:
return
deadline = time.monotonic() + 0.5
while time.monotonic() < deadline and group_exists(process.pid):
process.poll()
time.sleep(0.02)
try:
os.killpg(process.pid, signal.SIGKILL)
except ProcessLookupError:
pass
process.wait(timeout=3)
def run_captured(
argv: list[str],
*,
cwd: Path | str | None = None,
env: dict[str, str] | None = None,
timeout: float = 30,
max_stdout: int = 1024 * 1024,
max_stderr: int = 65536,
input_bytes: bytes | None = None,
cancelled: threading.Event | None = None,
merge_stderr: bool = False,
terminate_on_limit: bool = True,
capture_mode: str = "prefix",
on_output: Callable[[OutputSnapshot], None] | None = None,
) -> Capture:
"""No shell, capped memory, finite deadline, owned process-group cleanup.
Prefix capture preserves probe semantics. Head/tail capture retains the
beginning and actual latest output within the same byte bound. Optional
callbacks receive a first-data snapshot, at most one dirty update per second,
and a final snapshot. Callback failures propagate after owned-process cleanup.
This is not a sandbox: a deliberately detached new process session is outside
the original process group. Only run trusted project commands.
"""
if capture_mode not in {"prefix", "head_tail"}:
raise ValueError("Capture mode must be prefix or head_tail")
if any(type(value) is not int or value < 0 for value in (max_stdout, max_stderr)):
raise ValueError("Output bounds must be nonnegative integers")
buffers = {
"stdout": _OutputBuffer(max_stdout, capture_mode),
"stderr": _OutputBuffer(max_stderr, capture_mode),
}
last_notified, notified_total = None, -1
def snapshot(final: bool) -> OutputSnapshot:
out, err = buffers["stdout"], buffers["stderr"]
return OutputSnapshot(
out.value(),
err.value(),
bool(out.omitted or err.omitted),
out.omitted,
err.omitted,
len(out.head),
len(err.head),
final,
)
def notify(final: bool = False) -> None:
nonlocal last_notified, notified_total
if on_output is None:
return
total = sum(buffer.total for buffer in buffers.values())
instant = time.monotonic()
if final or (
total > 0
and total != notified_total
and (last_notified is None or instant - last_notified >= 1)
):
on_output(snapshot(final))
last_notified, notified_total = instant, total
process = subprocess.Popen(
argv,
cwd=cwd,
env=env,
stdin=subprocess.PIPE if input_bytes is not None else subprocess.DEVNULL,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT if merge_stderr else subprocess.PIPE,
start_new_session=True,
)
pending_input = memoryview(input_bytes or b"")
selector = None
try:
selector = selectors.DefaultSelector()
assert process.stdout is not None
selector.register(process.stdout, selectors.EVENT_READ, "stdout")
if process.stderr:
selector.register(process.stderr, selectors.EVENT_READ, "stderr")
if process.stdin:
if pending_input:
selector.register(process.stdin, selectors.EVENT_WRITE, "stdin")
else:
process.stdin.close()
except BaseException:
stop_group(process)
if selector is not None:
selector.close()
for handle in (process.stdin, process.stdout, process.stderr):
if handle and not handle.closed:
handle.close()
raise
deadline = time.monotonic() + timeout
exited_at = None
state = None
try:
while selector.get_map() or process.poll() is None:
if cancelled and cancelled.is_set():
state = "interrupted"
break
if time.monotonic() >= deadline:
state = "timed_out"
break
for key, _ in selector.select(
timeout=min(0.1, max(0, deadline - time.monotonic()))
):
if key.data == "stdin":
try:
written = os.write(key.fd, pending_input[:4096])
pending_input = pending_input[written:]
except BrokenPipeError:
pending_input = memoryview(b"")
if not pending_input:
selector.unregister(key.fileobj)
key.fileobj.close()
continue
data = os.read(key.fd, 65536)
if not data:
selector.unregister(key.fileobj)
continue
target = buffers[key.data]
target.append(data)
if target.omitted and terminate_on_limit:
state = "output_limit"
break
notify()
if state:
break
if process.poll() is not None:
exited_at = exited_at or time.monotonic()
if selector.get_map() and time.monotonic() - exited_at >= 0.5:
state = "leaked_process"
break
if state:
stop_group(process)
else:
process.wait(timeout=3)
# A child can redirect all output then outlive an otherwise successful parent.
grace = time.monotonic() + 0.1
while group_exists(process.pid) and time.monotonic() < grace:
time.sleep(0.01)
if group_exists(process.pid):
state = "leaked_process"
stop_group(process)
else:
state = "passed" if process.returncode == 0 else "failed"
notify(final=True)
finally:
if process.poll() is None or group_exists(process.pid):
stop_group(process)
selector.close()
for handle in (process.stdin, process.stdout, process.stderr):
if handle and not handle.closed:
handle.close()
output = snapshot(True)
return Capture(
output.stdout,
output.stderr,
process.returncode,
state,
output.truncated,
output.omitted_stdout_bytes,
output.omitted_stderr_bytes,
output.stdout_head_bytes,
output.stderr_head_bytes,
)
def require_capture(argv: list[str], **kwargs) -> Capture:
result = run_captured(argv, **kwargs)
if result.status in {"timed_out", "interrupted", "output_limit", "leaked_process"}:
raise ValueError(
f"Bounded subprocess did not complete normally: {result.status}"
)
return result
+366
View File
@@ -0,0 +1,366 @@
"""Headless adapter to the existing, receipt-bound release console lifecycle.
The ASGI application runs in this process; no HTTP socket or background server
is started. Critical release orchestration remains in the existing service.
"""
from __future__ import annotations
import argparse
import asyncio
import importlib
from pathlib import Path
import re
import secrets
import sys
from typing import Any
from urllib.parse import quote
from .common import redact
META_ROOT = Path(__file__).resolve().parents[3]
RELEASE_ROOT = META_ROOT / "tools/release"
_REPOSITORY = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,127}\Z")
_VERSION = re.compile(r"[0-9]+\.[0-9]+\.[0-9]+(?:[-+][0-9A-Za-z][0-9A-Za-z.-]*)?\Z")
_RUN = re.compile(r"rr-(?:[0-9]{8}T[0-9]{6}Z-[0-9a-f]{12}|request-[0-9a-f]{64})\Z")
_STEP = re.compile(r"[A-Za-z0-9][A-Za-z0-9._:@-]{0,159}\Z")
_REQUEST = re.compile(r"[A-Za-z0-9][A-Za-z0-9._:@/-]{7,127}\Z")
def _typed(pattern: re.Pattern[str], label: str):
def parse(value: str) -> str:
if not pattern.fullmatch(value):
raise argparse.ArgumentTypeError(f"Invalid {label}.")
return value
return parse
def _limit(value: str) -> int:
try:
result = int(value)
except ValueError as exc:
raise argparse.ArgumentTypeError("Limit must be an integer from 1 to 100.") from exc
if not 1 <= result <= 100:
raise argparse.ArgumentTypeError("Limit must be an integer from 1 to 100.")
return result
def _planning_options(parser: argparse.ArgumentParser, *, selection: bool) -> None:
if selection:
parser.add_argument("--repo", action="append", default=[], type=_typed(_REPOSITORY, "repository name"))
parser.add_argument("--repo-version", action="append", default=[], metavar="REPO=VERSION")
parser.add_argument("--target-version", type=_typed(_VERSION, "target version"))
parser.add_argument("--channel", default="stable")
parser.add_argument("--online", action="store_true", help="Allow the existing remote/catalog checks.")
parser.add_argument("--remote-tags", action="store_true", help="Explicitly inspect remote Git tags.")
parser.add_argument("--public-catalog", action="store_true", help="Explicitly inspect the public catalog.")
parser.add_argument("--include-migrations", action="store_true", help="Run migration audits; never applies migrations.")
def register(subparsers: Any) -> None:
"""Register release commands without importing FastAPI, HTTPX or Core."""
parser = subparsers.add_parser("release", help="Plan and operate durable GovOPlaN release runs.")
commands = parser.add_subparsers(dest="release_command", required=True)
for name in ("plan", "status", "create"):
command = commands.add_parser(name)
_planning_options(command, selection=name != "status")
if name == "status":
command.add_argument("--include-website", action="store_true")
if name == "create":
command.add_argument("--request-id", required=True, type=_typed(_REQUEST, "request ID"))
command.add_argument("--apply", action="store_true", help="Persist the frozen run; otherwise preview only.")
command.set_defaults(handler=handle)
listing = commands.add_parser("list", help="List bounded, workspace-scoped durable run history.")
listing.add_argument("--limit", type=_limit, default=20)
listing.add_argument("--cursor")
listing.set_defaults(handler=handle)
for name in ("show", "preview", "execute", "resume", "retry", "reconcile"):
command = commands.add_parser(name)
command.add_argument("run_id", type=_typed(_RUN, "run ID"))
if name in {"preview", "execute", "retry", "reconcile"}:
command.add_argument("step_id", type=_typed(_STEP, "step ID"))
if name in {"execute", "resume", "retry", "reconcile"}:
command.add_argument("--request-id", required=True, type=_typed(_REQUEST, "request ID"))
command.add_argument("--apply", action="store_true", help="Apply this explicit durable transition; otherwise preview only.")
if name in {"execute", "reconcile"}:
command.add_argument("--confirm", default="", help="Exact confirmation required by the existing release service.")
if name == "execute":
command.add_argument("--signing-key", action="append", default=[], metavar="KEY_ID=PRIVATE_KEY_FILE")
if name == "reconcile":
command.add_argument("--outcome", required=True, choices=("effect_absent", "effect_succeeded", "unresolved"))
command.set_defaults(handler=handle)
def _selection(args: argparse.Namespace) -> tuple[list[str], dict[str, str]]:
versions: dict[str, str] = {}
for item in args.repo_version:
repo, separator, version = item.partition("=")
if not separator or not _REPOSITORY.fullmatch(repo) or not _VERSION.fullmatch(version):
raise ValueError("--repo-version must be REPO=VERSION with a valid repository and version.")
if repo in versions and versions[repo] != version:
raise ValueError(f"Conflicting target versions for {repo}.")
versions[repo] = version
repos = list(dict.fromkeys([*args.repo, *versions]))
if not repos:
raise ValueError("Select at least one --repo or --repo-version explicitly.")
for repo in repos:
if repo not in versions and args.target_version:
versions[repo] = args.target_version
if args.release_command == "create" and any(repo not in versions for repo in repos):
raise ValueError("Creating a run requires an explicit version for every selected repository.")
return repos, versions
def _planning_query(args: argparse.Namespace) -> dict[str, Any]:
return {
"channel": args.channel,
"online": args.online,
"remote_tags": args.remote_tags,
"public_catalog": args.public_catalog or args.online,
"include_migrations": args.include_migrations,
**({"target_version": args.target_version} if args.target_version else {}),
}
def _load_application(args: argparse.Namespace) -> tuple[Any, str]:
# The generic CLI/help path stays dependency-light. The backend still checks
# its operator-controlled runtime and registered source origins on apply.
for name, loaded in list(sys.modules.items()):
if name in {"server", "govoplan_release"} or name.startswith(("server.", "govoplan_release.")):
expected = RELEASE_ROOT / ("server" if name.startswith("server") else "govoplan_release")
source = getattr(loaded, "__file__", None)
if not isinstance(source, str) or not Path(source).resolve().is_relative_to(expected.resolve()):
raise ValueError("A foreign module shadows the trusted GovOPlaN release service.")
# Reprioritize even when the path was added previously below an unrelated
# working directory. Validate cached packages before importing any submodule.
sys.path[:] = [str(RELEASE_ROOT), *(path for path in sys.path if path != str(RELEASE_ROOT))]
module = importlib.import_module("server.app")
if Path(module.__file__).resolve() != (RELEASE_ROOT / "server/app.py").resolve():
raise ValueError("A foreign server.app module shadows the GovOPlaN release service.")
token = secrets.token_urlsafe(32)
state_dir = getattr(args, "state_dir", None)
app = module.create_app(
workspace_root=Path(args.workspace_root).expanduser().resolve(),
token=token,
run_state_root=Path(state_dir) / "release-console" if state_dir is not None else None,
)
return app, token
def _error_detail(payload: Any) -> str:
detail = payload.get("detail") if isinstance(payload, dict) else None
if isinstance(detail, str):
return detail
if isinstance(detail, list):
# Validation input may contain signing-key arguments. Never echo it.
return "; ".join(
".".join(str(part) for part in item.get("loc", [])) + ": " + str(item.get("msg", "Invalid request"))
for item in detail if isinstance(item, dict)
)
return "The release service did not return a valid successful response."
class _ServiceError(Exception):
def __init__(self, status: int, payload: Any):
super().__init__(_error_detail(payload))
self.status = status
def _brief(value: Any, *, limit: int = 240) -> str:
"""Bound display-only fields; never serialize an executor or its arguments."""
if not isinstance(value, (str, int, float, bool)):
return "unknown"
text = " ".join(redact(str(value)).split())
return text if len(text) <= limit else text[:limit - 1] + ""
def _status(payload: dict[str, Any]) -> str:
status = payload.get("status")
for key in ("summary", "state"):
if isinstance(payload.get(key), dict):
status = payload[key].get("status", status)
if isinstance(payload.get("state_step"), dict):
status = payload["state_step"].get("state", status)
# Return the semantic value unchanged: display redaction must never affect
# failure exit codes, even when an environment secret happens to equal it.
return status if isinstance(status, str) else "ok"
def _summary_lines(name: str, payload: dict[str, Any], note: str | None = None) -> list[str]:
"""Compact, bounded projection; the unchanged JSON result retains details."""
lines = [f"Release {name}: {_brief(_status(payload))}."]
if note:
lines.append(note)
plan = payload.get("immutable", {}).get("plan", {}) if isinstance(payload.get("immutable"), dict) else payload
if not isinstance(plan, dict):
plan = {}
if isinstance(plan.get("source_preflight_ready"), bool):
prefix = "Frozen plan source" if "immutable" in payload else "Source"
lines.append(f"{prefix} preflight ready: {str(plan['source_preflight_ready']).lower()}.")
units = plan.get("units", [])
if isinstance(units, list):
for unit in units[:12]:
if isinstance(unit, dict):
lines.append(f"{_brief(unit.get('repo'))}: {_brief(unit.get('status', 'planned'))}; target {_brief(unit.get('target_version'))}.")
if len(units) > 12:
lines.append(f"{len(units) - 12} more selected repositories; use --json for every repository.")
dashboard = payload.get("summary")
if isinstance(dashboard, dict):
counts = [f"{dashboard[key]} {label}" for key, label in (
("repository_count", "repositories"), ("missing_count", "missing"),
("dirty_count", "dirty"), ("ahead_count", "ahead"),
("behind_count", "behind"), ("error_count", "errors"),
) if isinstance(dashboard.get(key), int)]
if counts:
lines.append("Repository status: " + ", ".join(counts) + ".")
findings = plan.get("gate_findings", payload.get("collection_errors", []))
if isinstance(findings, list):
for finding in findings[:4]:
if isinstance(finding, dict):
scope = f" ({_brief(finding['repo'])})" if finding.get("repo") else ""
lines.append(f"Gate {_brief(finding.get('code'))}{scope}: {_brief(finding.get('message'))}")
if len(findings) > 4:
lines.append(f"{len(findings) - 4} more gate findings; use --json for details.")
state = payload.get("state", {})
steps = state.get("steps", []) if isinstance(state, dict) else []
if isinstance(payload.get("state_step"), dict):
steps = [payload["state_step"]]
if isinstance(steps, list) and steps:
counts: dict[str, int] = {}
for step in steps:
if isinstance(step, dict):
status = _brief(step.get("state", "unknown"))
counts[status] = counts.get(status, 0) + 1
lines.append("Steps: " + ", ".join(f"{count} {state}" for state, count in sorted(counts.items())) + ".")
relevant = [step for step in steps if isinstance(step, dict) and step.get("state") != "succeeded"]
for step in relevant[:3]:
lines.append(f"Step {_brief(step.get('id'))}: {_brief(step.get('state'))}." +
(f" {_brief(step['disabled_reason'])}" if step.get("disabled_reason") else ""))
execution = payload.get("execution_result")
if isinstance(execution, dict):
lines.append(f"Executor result: {_brief(execution.get('status', 'recorded'))}.")
recommendation = payload.get("recommended_next", plan.get("recommended_action"))
if isinstance(recommendation, dict) and recommendation:
step = f" [{_brief(recommendation['step_id'])}]" if recommendation.get("step_id") else ""
lines.append(f"Next: {_brief(recommendation.get('id'))}{step}{_brief(recommendation.get('title'))}.")
if recommendation.get("remediation"):
lines.append(_brief(recommendation["remediation"]))
runs = payload.get("runs")
if isinstance(runs, list):
lines.append(f"{len(runs)} release runs in this page.")
for run in runs[:12]:
if isinstance(run, dict):
lines.append(f"{_brief(run.get('run_id'))}: {_brief(_status(run))}.")
if payload.get("next_cursor"):
lines.append("More history available; use --json for the next cursor.")
return lines
async def _run(args: argparse.Namespace) -> dict[str, Any]:
# Resolve CLI-only validation before loading the service or creating state.
if getattr(args, "project", None) is not None:
raise ValueError("Release uses the authoritative GovOPlaN catalog and does not accept --project overrides; select the registered --workspace-root instead.")
name = args.release_command
selection = _selection(args) if name in {"plan", "create"} else None
keys = getattr(args, "signing_key", [])
if len(keys) > 8 or any(not re.fullmatch(r"[A-Za-z0-9._-]{1,128}=.+", key) or "-----BEGIN" in key or "\n" in key or "\r" in key or len(key) > 4096 for key in keys):
raise ValueError("Provide at most eight --signing-key KEY_ID=PRIVATE_KEY_FILE arguments, never key material.")
import httpx
app, token = _load_application(args)
metadata = {
"workspace_root": str(app.state.workspace_root),
"state_location": str(app.state.release_runs.root),
"candidate_location": str(app.state.release_candidate_root),
}
transport = httpx.ASGITransport(app=app, raise_app_exceptions=False)
async with httpx.AsyncClient(
transport=transport, base_url="http://govoplan-devkit.invalid",
headers={"X-Release-Console-Token": token}, timeout=None,
follow_redirects=False,
) as client:
async def request(method: str, path: str, **kwargs: Any) -> dict[str, Any]:
response = await client.request(method, path, **kwargs)
try:
payload = response.json()
except ValueError:
payload = None
if response.status_code >= 400 or not isinstance(payload, dict):
raise _ServiceError(response.status_code, payload)
return payload
def result(payload: dict[str, Any], *, dry_run: bool = False, summary: str | None = None) -> dict[str, Any]:
status = _status(payload)
execution_failed = isinstance(payload.get("execution_result"), dict) and payload["execution_result"].get("status") == "failed"
return {
**metadata, "operation": name, "dry_run": dry_run,
"result": payload,
"_exit_code": 1 if execution_failed or status in {"blocked", "failed", "interrupted"} else 0,
"summary": [*_summary_lines(name, payload, summary), f"Durable state: {metadata['state_location']}"],
}
async def preview(run: dict[str, Any], step_id: str) -> dict[str, Any]:
plan = run.get("immutable", {}).get("plan", {})
plan_step = next((step for step in plan.get("dry_run_steps", []) if step.get("id") == step_id), None)
state_step = next((step for step in run.get("state", {}).get("steps", []) if step.get("id") == step_id), None)
if plan_step is None or state_step is None:
raise _ServiceError(404, {"detail": "Release run step was not found."})
if state_step.get("executor", {}).get("kind") == "catalog_publish":
return await request("POST", f"{run_path}/steps/{quote(step_id, safe='')}/preview", json={"remote": "origin"})
return {
"run_id": run["run_id"], "plan_step": plan_step, "state_step": state_step,
"note": "Frozen-plan inspection only; no executor was called and no live preflight is claimed.",
}
if name == "status":
payload = await request("GET", "/api/dashboard", params={**_planning_query(args), "include_website": args.include_website})
return result(payload)
if name == "list":
params = {"limit": args.limit, **({"cursor": args.cursor} if args.cursor else {})}
return result(await request("GET", "/api/release-runs", params=params))
if name in {"plan", "create"}:
assert selection is not None
repos, versions = selection
query = {
**_planning_query(args), "repos": ",".join(repos),
"repo_versions": ",".join(f"{repo}={version}" for repo, version in versions.items()),
}
if name == "plan" or not args.apply:
payload = await request("GET", "/api/selective-plan", params=query)
return result(payload, dry_run=True, summary="Release plan inspected; no run was created and no release step executed.")
body = {key: value for key, value in _planning_query(args).items() if key != "target_version"}
payload = await request("POST", "/api/release-runs", json={**body, "request_id": args.request_id, "repo_versions": versions})
return result(payload)
run_path = "/api/release-runs/" + quote(args.run_id, safe="")
if name == "show":
return result(await request("GET", run_path))
if name == "preview" or not args.apply:
run = await request("GET", run_path)
payload = await preview(run, args.step_id) if name in {"preview", "execute"} else run
return result(payload, dry_run=True, summary=f"Release {name} inspected; no durable transition or executor was invoked.")
body = {"request_id": args.request_id}
if name in {"execute", "reconcile"}:
body["confirm"] = args.confirm
if name == "execute":
body.update({"remote": "origin", "signing_keys": keys})
if name == "reconcile":
body["outcome"] = args.outcome
path = f"{run_path}/resume" if name == "resume" else f"{run_path}/steps/{quote(args.step_id, safe='')}/{name}"
return result(await request("POST", path, json=body))
def handle(args: argparse.Namespace) -> dict[str, Any]:
"""Return the common devkit JSON/summary envelope; never retry mutations."""
try:
return asyncio.run(_run(args))
except _ServiceError as exc:
summary = [f"Release {args.release_command} failed (HTTP {exc.status}): {exc}"]
if args.release_command in {"create", "execute", "resume", "retry", "reconcile"}:
summary.append("No automatic retry occurred. Inspect the run; reuse the same request ID for a known replay, or resume/reconcile an uncertain effect before a new attempt.")
return {"_exit_code": 1, "status": "error", "http_status": exc.status, "summary": summary}
except ModuleNotFoundError as exc:
return {"_exit_code": 2, "status": "unavailable", "summary": [f"Release commands need the GovOPlaN development dependencies ({exc.name} is unavailable)."]}
except ValueError as exc:
return {"_exit_code": 2, "status": "invalid", "summary": [str(exc)]}
+244
View File
@@ -0,0 +1,244 @@
"""Local, source-derived review guidance. Gitea remains the only review state log."""
from __future__ import annotations
import hashlib
import os
from pathlib import Path
import re
import stat
from urllib.parse import urlsplit
from .common import atomic_json, now, read_json, redact, reject_symlinks
from .issues import coverage_notes, evidence_record, _base_url, _name, _positive
from .workspace import inspect_repository, load_project, selected_repositories, source_fingerprint
MAX_SOURCES = 10000
MANUAL_CHECKS = (
("surfaces", "Confirm every route, pane, dialog, settings surface, widget, public form and contributed interface; source discovery is only a starting list."),
("display-edit", "Show compact readable data first; edit coherent groups in scoped dialogs. Record a reason for a deliberate bulk editing mode or other exception."),
("help", "Check documentation books beside meaningful visible headings/labels, including widgets, loading and configuration states; preserve optional Docs fallback."),
("actions", "Check predictable action order, Save/Cancel, unsaved drafts, destructive consequences and truthful unavailable-action explanations."),
("geometry", "Check shared cards/tables/dialogs, column resizing, pagination, long data, narrow layouts, zoom and the reported wide-window configuration."),
("states", "Exercise loading, empty, partial, error, stale, conflict and permission-denied states; verify no accidental mutation on read or cancel."),
("accessibility", "Exercise keyboard order, visible focus, accessible names, dialog focus restoration and non-color-only feedback."),
("language-docs", "Review English and German, long headings and module-owned user/admin documentation for every changed workflow and limitation."),
("boundaries", "Exercise tenant/authorization boundaries and optional-module absence; inspect headless modules' contributed interfaces rather than marking them complete automatically."),
("propagation", "Record applied principle revision, exceptions, fixes, evidence and remaining work in the module issue; propagate new rules to already-reviewed modules through the central issue."),
)
def register(subparsers) -> None:
parser = subparsers.add_parser("review", help="Assemble a module's local UI-review guidance (does not complete a review)")
parser.add_argument("module", help="Repository name, alias, or review inventory scope ID")
parser.add_argument("bundle_module", nargs="?", help="Also accepts: review bundle MODULE")
parser.add_argument("--profile", default="ui", help="Plan this registered check profile, without running it")
parser.add_argument("--evidence", help="Existing local run ID or explicit receipt JSON path")
parser.add_argument("--output", type=Path, help="Optional local JSON artifact; not a progress tracker")
parser.set_defaults(handler=handle_review)
def _project_path(root: Path, value: str) -> Path:
if not isinstance(value, str):
raise ValueError("Review inventory/principles paths must be strings")
raw = Path(value)
if raw.is_absolute() or ".." in raw.parts:
raise ValueError("Review configuration paths must be relative within the workspace")
path = root / raw
reject_symlinks(path)
if not path.resolve().is_relative_to(root.resolve()):
raise ValueError("Review input escapes the workspace")
return path
def _link(record: dict) -> dict:
if not isinstance(record, dict):
raise ValueError("Review issue link must be an object")
repo = _name(record.get("repository"))
number = _positive(record.get("number"))
url = record.get("url")
if not isinstance(url, str):
raise ValueError("Review issue URL is missing")
parsed = urlsplit(_base_url(url))
parts = parsed.path.rstrip("/").split("/")
if len(parts) < 5 or parts[-3:] != [repo, "issues", str(number)]:
raise ValueError("Review issue URL does not match its repository and number")
_name(parts[-4])
# Snapshot status/operation fields are deliberately not projected as live state.
return {"repository": repo, "number": number, "url": url}
def _issue_inventory(path: Path | None) -> tuple[list[dict], dict | None]:
if path is None or not path.exists():
return [], None
payload = read_json(path, max_bytes=2 * 1024 * 1024)
if not isinstance(payload, dict) or payload.get("schema_version") != 1:
raise ValueError("Review issue inventory requires schema_version 1")
rows = payload.get("issues")
if not isinstance(rows, list) or len(rows) > 1024:
raise ValueError("Review issue inventory must contain a bounded issues list")
issues, scopes, urls = [], set(), set()
for row in rows:
link = _link(row)
scope = row.get("scope_id")
if not isinstance(scope, str) or not re.fullmatch(r"[A-Za-z0-9_:.-]{1,128}", scope) or scope in scopes or link["url"] in urls:
raise ValueError("Review issue inventory has an invalid or duplicate scope/issue")
scopes.add(scope)
urls.add(link["url"])
issues.append({**link, "scope_id": scope, "name": str(row.get("name", scope)), "kind": str(row.get("kind", "unspecified"))})
epic = _link(payload["epic"]) if payload.get("epic") else None
return issues, epic
def _read_text(path: Path) -> str:
reject_symlinks(path)
descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0))
with os.fdopen(descriptor, "rb") as handle:
metadata = os.fstat(handle.fileno())
if not stat.S_ISREG(metadata.st_mode) or metadata.st_size > 1024 * 1024:
raise ValueError("Principles must be a bounded regular text file")
encoded = handle.read(1024 * 1024 + 1)
if len(encoded) > 1024 * 1024:
raise ValueError("Principles exceed the file-size bound")
return encoded.decode("utf-8")
def _principles(path: Path | None) -> dict:
if path is None or not path.exists():
return {"path": str(path) if path else None, "available": False, "revision": None, "rules": []}
text = _read_text(path)
revision = re.search(r"\bUI-\d{4}-\d{2}-\d{2}\b", text)
rules = [{"id": match.group(1), "title": match.group(2).strip()}
for match in re.finditer(r"^##\s+(UI-\d{2})\s*[—–:-]\s*(.+)$", text, re.MULTILINE)]
return {"path": str(path), "available": True, "revision": revision.group(0) if revision else None,
"content_sha256": hashlib.sha256(text.encode()).hexdigest(), "rules": rules}
def source_inventory(root: Path) -> dict:
"""Enumerate filenames only; never execute module manifests or import optional modules."""
groups = {name: [] for name in ("pages-and-navigation", "dialogs-and-embedded-editors", "settings-and-administration", "widgets-and-public-surfaces", "other-ui-sources")}
manifests, skipped = [], []
count = 0
for base, mode in ((root / "webui/src", "ui"), (root / "src", "backend")):
try:
reject_symlinks(base)
except ValueError:
skipped.append(str(base.relative_to(root)))
continue
for directory, dirs, names in os.walk(base, followlinks=False):
folder = Path(directory)
safe_dirs = []
for name in sorted(dirs):
child = folder / name
if child.is_symlink():
skipped.append(str(child.relative_to(root)))
elif name not in {"node_modules", ".git", "__pycache__", ".venv", "dist"}:
safe_dirs.append(name)
dirs[:] = safe_dirs
for name in sorted(names):
path = folder / name
if path.is_symlink():
skipped.append(str(path.relative_to(root)))
continue
if mode == "backend":
if name == "manifest.py":
manifests.append(str(path.relative_to(root)))
continue
if path.suffix not in {".tsx", ".jsx", ".vue", ".svelte"} and name not in {"index.ts", "module.ts", "routes.ts"}:
continue
count += 1
if count > MAX_SOURCES:
raise ValueError("Module UI source inventory exceeds its size bound")
relative = str(path.relative_to(root))
lower = relative.lower()
group = ("dialogs-and-embedded-editors" if any(part in lower for part in ("dialog", "modal", "editor")) else
"settings-and-administration" if any(part in lower for part in ("setting", "admin")) else
"widgets-and-public-surfaces" if any(part in lower for part in ("widget", "public")) else
"pages-and-navigation" if any(part in lower for part in ("/pages/", "page.", "navigation", "routes.ts")) else "other-ui-sources")
groups[group].append(relative)
return {"basis": "Static filename discovery, not a complete runtime surface inventory or a review result.",
"ui_source_count": count, "groups": groups, "manifest_paths": manifests,
"skipped_symlinks": skipped, "module_code_imported": False}
def _redact_tree(value):
if isinstance(value, str):
return redact(value)
if isinstance(value, list):
return [_redact_tree(item) for item in value]
if isinstance(value, dict):
return {key: _redact_tree(item) for key, item in value.items()}
return value
def handle_review(args) -> dict:
name = args.module
if name == "bundle":
if not args.bundle_module:
raise ValueError("review bundle requires a module")
name = args.bundle_module
elif args.bundle_module:
raise ValueError("Review accepts one module; use review MODULE")
workspace_root = Path(args.workspace_root).resolve()
project = load_project(workspace_root, args.project)
configured = project.config.get("review", {}) if args.project else {}
if not isinstance(configured, dict) or set(configured) - {"issue_inventory", "principles"}:
raise ValueError("Project review configuration accepts issue_inventory and principles paths")
meta = next((repo.path for repo in project.repositories if repo.name == "govoplan"), None)
core = next((repo.path for repo in project.repositories if repo.name == "govoplan-core"), None)
inventory_path = (_project_path(workspace_root, configured["issue_inventory"]) if "issue_inventory" in configured else
meta / "docs/project/ui-review-issue-inventory.json" if meta and not args.project else None)
principles_path = (_project_path(workspace_root, configured["principles"]) if "principles" in configured else
core / "docs/UI_DESIGN_PRINCIPLES.md" if core and not args.project else None)
issues, epic = _issue_inventory(inventory_path)
scope_matches = [item for item in issues if item["scope_id"] == name]
selected = selected_repositories(project, [scope_matches[0]["repository"] if scope_matches else name])
if len(selected) != 1:
raise ValueError("Review must resolve to exactly one registered repository")
repo = selected[0]
links = [item for item in issues if item["repository"] == repo.name]
inventory = source_inventory(repo.path)
principles = _principles(principles_path)
state = inspect_repository(repo)
warnings = []
if not links:
warnings.append("No module issue discovery link is configured; locate/create the canonical Gitea review issue before recording review work.")
if not principles["available"] or not principles["revision"]:
warnings.append("The principle document or dated UI revision is unavailable; confirm the governing rules before reviewing.")
if not inventory["ui_source_count"]:
warnings.append("No UI source filenames were discovered. Check contributed/headless/placeholder scope manually; this is not automatic N/A or completion.")
if inventory["skipped_symlinks"]:
warnings.append("Symlinked source paths were not traversed; the source starting inventory is incomplete.")
if state["errors"]:
warnings.append("Repository inspection reported errors; missing or unreadable source is not a clean review.")
try:
fingerprint = source_fingerprint(project)
except (OSError, ValueError):
fingerprint = None
warnings.append("Current workspace source identity could not be established; do not present attached historical evidence as current verification.")
from .catalog import build_stages
checks = build_stages(workspace_root, args.profile, [repo.name], False, project=args.project)
if not checks:
warnings.append("No automated stages are planned for this selection; this is not a passing verification result.")
evidence = evidence_record(args.evidence, args)
limits = list(dict.fromkeys([*coverage_notes(checks), *(evidence["coverage_notes"] if evidence else [])]))
warnings.extend("Coverage limitation: " + note for note in limits[:8])
if len(limits) > 8:
warnings.append(f"{len(limits) - 8} additional coverage limitations are retained in the JSON bundle.")
result = {"schema_version": 1, "operation": "review.bundle", "generated_at": now(), "workspace_root": str(workspace_root),
"project": project.name, "module": repo.name, "source_fingerprint": fingerprint, "repository": state,
"issue_links": links, "central_issue": epic, "issue_inventory_path": str(inventory_path) if inventory_path else None,
"state_authority": "Live Gitea issues are the canonical backlog and review state log. Discovery links do not report current issue state.",
"inventory": inventory, "principles": principles, "check_plan": {"profile": args.profile, "executed": False, "stages": checks},
"manual_checklist": [{"id": identity, "prompt": prompt} for identity, prompt in MANUAL_CHECKS],
"review_completion": "Not assessed. Automated checks and this bundle never complete a module review or update issue checklists.",
"evidence": evidence, "coverage_notes": limits, "warnings": warnings,
"summary": [f"Review bundle: {repo.name}; {inventory['ui_source_count']} UI source files, {len(checks)} planned checks (not executed).",
"Principle revision: " + (principles["revision"] or "unavailable"),
*["Module issue: " + item["url"] for item in links],
*(["Central issue: " + epic["url"]] if epic else []),
"Manual review remains unassessed; record outcomes and remaining work in Gitea.", *warnings]}
result = _redact_tree(result)
if args.output:
result["summary"].append("Local bundle artifact: " + redact(str(args.output)))
atomic_json(args.output, result)
return result
File diff suppressed because it is too large Load Diff
+142
View File
@@ -0,0 +1,142 @@
"""Dependency-free validation against the published portable-project schema."""
from __future__ import annotations
import math
from pathlib import Path
import re
from .common import META_ROOT, canonical, read_json
def _schema_value(value, schema: dict, definitions: dict, path: str) -> None:
if "$ref" in schema:
return _schema_value(
value,
definitions[schema["$ref"].removeprefix("#/$defs/")],
definitions,
path,
)
kind = schema.get("type")
numeric = type(value) is int or type(value) is float and math.isfinite(value)
matches = {
"object": isinstance(value, dict),
"array": isinstance(value, list),
"string": isinstance(value, str),
"number": numeric,
"integer": numeric and int(value) == value,
}
if kind and not matches[kind]:
raise ValueError(f"{path} must be a {kind}")
if "const" in schema and value != schema["const"]:
raise ValueError(f"{path} has an unsupported value")
if "enum" in schema and value not in schema["enum"]:
raise ValueError(f"{path} has an unsupported value")
if isinstance(value, dict):
properties = schema.get("properties", {})
unknown = set(value) - set(properties)
if schema.get("additionalProperties") is False and unknown:
names = ", ".join(key[:80] for key in sorted(unknown)[:4])
raise ValueError(f"Unknown field in {path}: {names}")
if set(schema.get("required", [])) - set(value):
raise ValueError(f"{path} is missing a required field")
for key, item in value.items():
if key in properties:
_schema_value(item, properties[key], definitions, f"{path}.{key}")
if isinstance(value, list):
if (
not schema.get("minItems", 0)
<= len(value)
<= schema.get("maxItems", len(value))
):
raise ValueError(f"{path} exceeds its item bounds")
if schema.get("uniqueItems") and len(
{canonical(item) for item in value}
) != len(value):
raise ValueError(f"Duplicate value in {path}")
prefix = schema.get("prefixItems", [])
for index, item in enumerate(value):
_schema_value(
item,
prefix[index] if index < len(prefix) else schema.get("items", {}),
definitions,
f"{path}[{index}]",
)
if isinstance(value, str):
if (
not schema.get("minLength", 0)
<= len(value)
<= schema.get("maxLength", len(value))
):
raise ValueError(f"{path} exceeds its string bounds")
if "pattern" in schema and re.search(schema["pattern"], value) is None:
raise ValueError(
f"{path} contains an invalid identifier, path or character"
)
if type(value) in {int, float}:
if (
"maximum" in schema
and value > schema["maximum"]
or "exclusiveMinimum" in schema
and value <= schema["exclusiveMinimum"]
):
raise ValueError(f"{path} exceeds its numeric bounds")
def validate_project(payload: dict, workspace_root: Path) -> None:
"""Validate every declaration, not just the selected profile/dependencies."""
schema = read_json(META_ROOT / "tools/devkit/project.schema.json")
_schema_value(payload, schema, schema["$defs"], "project")
root = workspace_root.resolve()
records = payload["repositories"]
names = {record["name"] for record in records}
if len(names) != len(records):
raise ValueError("Duplicate project repository name")
paths, aliases = set(), set()
for record in records:
path = (root / record["path"]).resolve()
if not path.is_relative_to(root):
raise ValueError("Repository path escapes the workspace")
if path in paths:
raise ValueError("Duplicate project repository path")
paths.add(path)
keys = {record["name"], *record.get("aliases", [])}
if aliases & keys:
raise ValueError("Repository names and aliases must be unambiguous")
aliases.update(keys)
checks = payload.get("checks", [])
identities = {item["id"] for item in checks}
if len(identities) != len(checks):
raise ValueError("Duplicate project check ID")
for item in checks:
if set(item.get("repos", [])) - names:
raise ValueError(f"Unknown repository reference in check {item['id']}")
if set(item.get("deps", [])) - identities:
raise ValueError(f"Unknown check dependency in {item['id']}")
if set(item.get("after", [])) - identities:
raise ValueError(f"Unknown check ordering reference in {item['id']}")
if set(item.get("deps", [])) & set(item.get("after", [])):
raise ValueError(f"Duplicate dependency/ordering reference in {item['id']}")
if set(item.get("inputs", {}).get("repos", [])) - names:
raise ValueError(f"Unknown input repository reference in {item['id']}")
if not (root / item.get("cwd", ".")).resolve().is_relative_to(root):
raise ValueError(f"Check {item['id']} cwd escapes the workspace")
for name, selected in payload.get("profiles", {}).items():
if set(selected) - identities:
raise ValueError(f"Unknown profile check in {name}")
remaining = {
item["id"]: set(item.get("deps", [])) | set(item.get("after", []))
for item in checks
}
while remaining:
ready = {identity for identity, deps in remaining.items() if not deps}
if not ready:
raise ValueError("Cyclic check dependency in project declarations")
remaining = {
identity: deps - ready
for identity, deps in remaining.items()
if identity not in ready
}
for value in payload.get("review", {}).values():
if not (root / value).resolve().is_relative_to(root):
raise ValueError("Project review path escapes the workspace")
+273
View File
@@ -0,0 +1,273 @@
"""Repository discovery, offline snapshots and exact source fingerprints."""
from __future__ import annotations
from dataclasses import dataclass
import hashlib
import os
from pathlib import Path
import stat
import sys
from .common import META_ROOT, digest, identifier, read_json
from .process import require_capture
@dataclass(frozen=True)
class Repository:
name: str
path: Path
aliases: tuple[str, ...] = ()
@dataclass(frozen=True)
class Project:
name: str
repositories: tuple[Repository, ...]
config: dict
def load_project(workspace_root: Path, project: Path | None = None) -> Project:
root = workspace_root.resolve()
payload = read_json(project or META_ROOT / "repositories.json")
if not isinstance(payload, dict):
raise ValueError("Project manifest must be an object")
if project:
from .validation import validate_project
validate_project(payload, root)
records = payload.get("repositories")
if not isinstance(records, list) or not records or len(records) > 256:
raise ValueError("Project manifest requires 1256 repositories")
repos, names = [], set()
for record in records:
if not isinstance(record, dict) or not isinstance(record.get("path"), str):
raise ValueError("Invalid repository record")
if not record["path"] or (
project and set(record) - {"name", "path", "aliases"}
):
raise ValueError("Repository requires a nonempty path and known fields")
name = identifier(record.get("name"))
raw = Path(record["path"])
if raw.is_absolute() or ".." in raw.parts:
raise ValueError("Repository paths must be relative within the workspace")
path = (root / raw).resolve()
if not path.is_relative_to(root):
raise ValueError("Repository path escapes the workspace")
aliases = (
record.get("aliases", []) if project else [name.removeprefix("govoplan-")]
)
if not isinstance(aliases, list) or any(
not isinstance(alias, str) for alias in aliases
):
raise ValueError("Repository aliases must be strings")
if len(aliases) != len(set(aliases)):
raise ValueError("Duplicate repository alias")
keys = {name, *[identifier(alias) for alias in aliases]}
if names.intersection(keys):
raise ValueError("Repository names and aliases must be unambiguous")
names.update(keys)
repos.append(Repository(name, path, tuple(aliases)))
return Project(
payload.get("name", "Project" if project else "GovOPlaN"), tuple(repos), payload
)
def inspect_repository(repo: Repository) -> dict:
# Reuse the established read-only model, including distinct Git errors.
sys.path.insert(0, str(META_ROOT / "tools/release")) if str(
META_ROOT / "tools/release"
) not in sys.path else None
from govoplan_release.git_state import collect_repository_snapshot
from govoplan_release.model import RepositorySpec
unsafe = unsafe_git_environment()
if unsafe:
return {
"name": repo.name,
"path": str(repo.path),
"exists": repo.path.exists(),
"is_git": (repo.path / ".git").exists(),
"branch": None,
"head": None,
"upstream": None,
"ahead": None,
"behind": None,
"remote_checked": False,
"dirty_entries": [],
"errors": ["Git environment overrides prevent scoped inspection"],
"safe_directory_required": False,
}
snapshot = collect_repository_snapshot(
RepositorySpec(
name=repo.name,
category="module",
subtype="",
remote="",
path=str(repo.path),
),
workspace_root=repo.path.parent,
target_tag=None,
online=False,
)
return {
"name": repo.name,
"path": str(repo.path),
"exists": snapshot.exists,
"is_git": snapshot.is_git,
"branch": snapshot.branch,
"head": snapshot.head,
"upstream": snapshot.upstream,
"ahead": snapshot.ahead,
"behind": snapshot.behind,
"remote_checked": False,
"dirty_entries": list(snapshot.dirty_entries),
"errors": list(snapshot.errors),
"safe_directory_required": snapshot.safe_directory_required,
}
def selected_repositories(
project: Project, names: list[str], changed: bool = False
) -> list[Repository]:
selected = list(project.repositories)
if names:
wanted = set(names)
known = {key for repo in selected for key in (repo.name, *repo.aliases)}
if wanted - known:
raise ValueError(
"Unknown repository filter: " + ", ".join(sorted(wanted - known))
)
selected = [
repo for repo in selected if wanted.intersection((repo.name, *repo.aliases))
]
if changed:
result = []
for repo in selected:
state = inspect_repository(repo)
if (
state["errors"]
or state["dirty_entries"]
or state["ahead"]
or (state["head"] and not state["upstream"])
):
result.append(repo)
selected = result
return selected
def unsafe_git_environment() -> set[str]:
return {
key
for key in os.environ
if key.startswith("GIT_")
and key not in {"GIT_OPTIONAL_LOCKS", "GIT_TERMINAL_PROMPT", "GIT_PAGER"}
}
def git_bytes(path: Path, *argv: str, allow_failure: bool = False) -> bytes:
if unsafe_git_environment():
raise ValueError(
"Git environment overrides prevent a scoped source fingerprint"
)
result = require_capture(
["git", "--no-pager", "-C", str(path), *argv],
timeout=30,
max_stdout=32 * 1024 * 1024,
env={**os.environ, "GIT_OPTIONAL_LOCKS": "0", "GIT_TERMINAL_PROMPT": "0"},
)
if result.returncode:
if allow_failure:
return b"unborn"
raise ValueError(
f"Could not fingerprint Git state in {path.name}; inspect context first"
)
return result.stdout
def source_fingerprint(project: Project) -> str:
"""Bind HEAD, index and every tracked/untracked working file, including hidden changes."""
overall = hashlib.sha256()
overall.update(digest(project.config).encode())
for repo in sorted(project.repositories, key=lambda item: item.name):
overall.update(repo.name.encode() + b"\0" + str(repo.path).encode() + b"\0")
if not repo.path.exists():
overall.update(b"missing\0")
continue
if not (repo.path / ".git").exists():
raise ValueError(
f"Cannot establish source identity for non-Git repository {repo.name}"
)
overall.update(
git_bytes(
repo.path, "rev-parse", "--verify", "HEAD", allow_failure=True
).strip()
)
overall.update(git_bytes(repo.path, "ls-files", "--stage", "-z"))
overall.update(git_bytes(repo.path, "ls-files", "-v", "-z"))
tracked = git_bytes(repo.path, "ls-files", "--cached", "-z")
untracked = git_bytes(
repo.path, "ls-files", "--others", "--exclude-standard", "-z"
)
for encoded_name in sorted(
set(tracked.split(b"\0") + untracked.split(b"\0")) - {b""}
):
name = os.fsdecode(encoded_name)
relative = Path(name)
if relative.is_absolute() or ".." in relative.parts:
raise ValueError("Unsafe repository file name")
path = repo.path / relative
overall.update(encoded_name + b"\0")
if not path.exists() and not path.is_symlink():
overall.update(b"deleted\0")
continue
metadata = path.lstat()
overall.update(str(stat.S_IMODE(metadata.st_mode)).encode() + b"\0")
if path.is_symlink():
overall.update(os.fsencode(os.readlink(path)))
elif path.is_file():
if not path.resolve().is_relative_to(repo.path):
raise ValueError("Fingerprint input escapes its repository")
maximum = 64 * 1024 * 1024
descriptor = os.open(
path,
os.O_RDONLY
| getattr(os, "O_NOFOLLOW", 0)
| getattr(os, "O_NONBLOCK", 0),
)
with os.fdopen(descriptor, "rb") as handle:
before = os.fstat(handle.fileno())
if not stat.S_ISREG(before.st_mode) or before.st_size > maximum:
raise ValueError(
f"Fingerprint input must be a bounded regular file in {repo.name}"
)
file_hash, count = hashlib.sha256(), 0
for chunk in iter(
lambda: handle.read(min(1024 * 1024, maximum - count + 1)), b""
):
count += len(chunk)
if count > maximum:
raise ValueError("File grew beyond fingerprint limit")
file_hash.update(chunk)
after = os.fstat(handle.fileno())
if (
before.st_ino,
before.st_size,
before.st_mtime_ns,
before.st_ctime_ns,
) != (
after.st_ino,
after.st_size,
after.st_mtime_ns,
after.st_ctime_ns,
):
raise ValueError(
"File changed while calculating source identity"
)
overall.update(file_hash.digest())
else:
raise ValueError(
"Unsupported changed-file type; cannot establish source identity"
)
overall.update(b"\0")
return overall.hexdigest()
+67
View File
@@ -0,0 +1,67 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"title": "Devkit portable project configuration",
"description": "The runtime additionally checks reference existence, cycles, resolved path confinement and cross-repository alias/path uniqueness. cwd defaults to the workspace when omitted.",
"type": "object",
"additionalProperties": false,
"required": ["schema_version", "repositories"],
"properties": {
"schema_version": {"type": "integer", "const": 1},
"name": {"type": "string", "minLength": 1, "maxLength": 128, "pattern": "^[^\\u0000\\r\\n]+$"},
"repositories": {
"type": "array", "minItems": 1, "maxItems": 256,
"items": {
"type": "object", "additionalProperties": false, "required": ["name", "path"],
"properties": {
"name": {"$ref": "#/$defs/identifier"},
"path": {"$ref": "#/$defs/relativePath"},
"aliases": {"type": "array", "maxItems": 128, "uniqueItems": true, "items": {"$ref": "#/$defs/identifier"}}
}
}
},
"tools": {
"type": "object", "additionalProperties": false,
"properties": {"python": {"$ref": "#/$defs/executable"}, "node": {"$ref": "#/$defs/executable"}, "npm": {"$ref": "#/$defs/executable"}}
},
"review": {
"type": "object", "additionalProperties": false,
"properties": {
"issue_inventory": {"$ref": "#/$defs/relativePath"},
"principles": {"$ref": "#/$defs/relativePath"}
}
},
"checks": {
"type": "array", "maxItems": 512,
"items": {
"type": "object", "additionalProperties": false, "required": ["id", "argv"],
"properties": {
"id": {"$ref": "#/$defs/identifier"},
"title": {"type": "string", "maxLength": 1024, "pattern": "^[^\\u0000]*$"},
"argv": {"type": "array", "minItems": 1, "maxItems": 256, "prefixItems": [{"type": "string", "minLength": 1, "maxLength": 8192, "pattern": "^[^\\u0000]+$"}], "items": {"type": "string", "maxLength": 8192, "pattern": "^[^\\u0000]*$"}},
"cwd": {"$ref": "#/$defs/relativePath"},
"deps": {"$ref": "#/$defs/profile"},
"after": {"$ref": "#/$defs/profile"},
"reuse": {"type": "string", "enum": ["verified", "never"]},
"inputs": {"$ref": "#/$defs/inputs"},
"resources": {"type": "array", "maxItems": 256, "uniqueItems": true, "items": {"type": "string", "minLength": 1, "maxLength": 256, "pattern": "^[^\\u0000\\r\\n]+$"}},
"repos": {"type": "array", "maxItems": 256, "uniqueItems": true, "items": {"$ref": "#/$defs/identifier"}},
"timeout_seconds": {"type": "number", "exclusiveMinimum": 0, "maximum": 43200}
}
}
},
"profiles": {
"type": "object", "additionalProperties": false,
"properties": {
"quick": {"$ref": "#/$defs/profile"}, "ui": {"$ref": "#/$defs/profile"},
"backend": {"$ref": "#/$defs/profile"}, "full": {"$ref": "#/$defs/profile"}
}
}
},
"$defs": {
"inputs": {"type": "object", "additionalProperties": false, "required": ["repos"], "properties": {"repos": {"type": "array", "minItems": 1, "maxItems": 256, "uniqueItems": true, "items": {"$ref": "#/$defs/identifier"}}}},
"identifier": {"type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$(?![\\s\\S])"},
"executable": {"type": "string", "minLength": 1, "maxLength": 4096, "pattern": "^[^\\u0000\\r\\n]+$"},
"relativePath": {"type": "string", "minLength": 1, "maxLength": 4096, "pattern": "^(?!/)(?!\\.\\.(?:/|$))(?!.*?/\\.\\.(?:/|$))[^\\u0000\\r\\n]+$"},
"profile": {"type": "array", "maxItems": 512, "uniqueItems": true, "items": {"$ref": "#/$defs/identifier"}}
}
}
+386
View File
@@ -0,0 +1,386 @@
#!/usr/bin/env python3
"""Inventory all module UI review scopes and safely create missing Gitea tracks.
Dry-run by default. Existing issues are never rewritten or closed. The optional
one-time epic link initialization refuses to replace an edited or completed list.
"""
from __future__ import annotations
import argparse
from concurrent.futures import ThreadPoolExecutor
from contextlib import contextmanager
import importlib.util
import json
from pathlib import Path
import re
import socket
import sys
from typing import Any, Iterator
from gitea_common import (
GiteaClient, GiteaError, RepoTarget, load_dotenv,
org_path, repo_path, require_token,
)
META_ROOT = Path(__file__).resolve().parents[2]
BASE_URL = "https://git.add-ideas.de"
OWNER = "GovOPlaN"
EPIC_MARKER = "<!-- govoplan-ui-review:v1:epic -->"
LIST_START = "<!-- govoplan-ui-review:module-list:start -->"
LIST_END = "<!-- govoplan-ui-review:module-list:end -->"
INITIAL_LIST = "The linked inventory is being initialized. All 77 tracks are pending; no checkboxes are complete."
PRINCIPLES_URL = f"{BASE_URL}/{OWNER}/govoplan-core/src/branch/main/docs/UI_DESIGN_PRINCIPLES.md"
PROCESS_URL = f"{BASE_URL}/{OWNER}/govoplan/src/branch/main/docs/project/UI_REVIEW_PROGRAM.md"
PRINCIPLES = (
("UI-01", "Help icons beside the relevant heading/label, not action-button rows"),
("UI-02", "Display-first with scoped edit dialogs; explicit bulk-grid editing exception"),
("UI-03", "Shared page actions, Reload/New ordering, Save/Cancel and destructive separation"),
("UI-04", "Full-width table/card geometry, visible actions, pagination and two-way resizing"),
("UI-05", "Scoped loading/error feedback, useful progress and retained state"),
("UI-06", "Predictable tree selection, expansion, grouping and reordering"),
("UI-07", "Keyboard/focus/accessibility, responsive layouts and understandable German"),
("UI-08", "Authorization, optional-module boundaries and save/cancel/retry data integrity"),
("UI-09", "Revision evidence and retroactive checks for changed design principles"),
)
def source_url(repository: str, path: str) -> str:
return f"{BASE_URL}/{OWNER}/{repository}/src/branch/main/{path}"
def marker(scope_id: str) -> str:
return f"<!-- govoplan-ui-review:v1:module:{scope_id} -->"
def normalized_title(value: str) -> str:
return " ".join(value.casefold().split())
def find_existing(issues: list[dict[str, Any]], scope_id: str, title: str) -> dict[str, Any] | None:
matches = [
issue for issue in issues if issue.get("pull_request") is None and (
marker(scope_id) in (issue.get("body") or "")
or normalized_title(issue.get("title") or "") == normalized_title(title)
)
]
if len(matches) > 1:
raise GiteaError(f"Ambiguous review issue matches for {scope_id}; inspect before making changes.")
if matches and marker(scope_id) not in (matches[0].get("body") or ""):
raise GiteaError(f"Unmanaged exact-title issue for {scope_id}; preserve it and resolve the duplicate manually.")
return matches[0] if matches else None
def extract_manifests(catalog: dict[str, Any], workspace_root: Path) -> list[dict[str, Any]]:
path = META_ROOT / "tools/inventory/platform-interface-inventory.py"
spec = importlib.util.spec_from_file_location("ui_review_source_inventory", path)
assert spec and spec.loader
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module._extract_manifests(catalog, workspace_root)
def source_groups(root: Path) -> dict[str, list[str]]:
groups: dict[str, list[str]] = {
"Pages and navigation entrypoints": [],
"Dialogs and embedded editing surfaces": [],
"Settings and administrator surfaces": [],
"Widgets, public/operator and contributed surfaces": [],
"Shared components and other UI entrypoints": [],
}
source_root = root / "webui/src"
for path in sorted(source_root.rglob("*.tsx")):
relative = path.relative_to(root).as_posix()
lower = relative.casefold()
name = path.stem.casefold()
if "page" in name or "navigation" in name or name in {"app", "routes", "index"}:
groups["Pages and navigation entrypoints"].append(relative)
elif re.search(r"dialog|modal|drawer|chooser|overlay", name):
groups["Dialogs and embedded editing surfaces"].append(relative)
elif re.search(r"settings|configur|admin", lower):
groups["Settings and administrator surfaces"].append(relative)
elif re.search(r"widget|public|operator|contribution", lower):
groups["Widgets, public/operator and contributed surfaces"].append(relative)
else:
groups["Shared components and other UI entrypoints"].append(relative)
return groups
def build_scopes(
catalog: dict[str, Any], workspace_root: Path, manifests: list[dict[str, Any]],
) -> list[dict[str, Any]]:
selected = [
repo for repo in catalog["repositories"]
if repo["category"] in {"module", "connector"} or repo["name"] == "govoplan-core"
]
by_repository: dict[str, dict[str, Any]] = {}
ids: set[str] = set()
for manifest in manifests:
if manifest["repository"] in by_repository or manifest["id"] in ids:
raise GiteaError("Duplicate source manifest repository or module ID.")
by_repository[manifest["repository"]] = manifest
ids.add(manifest["id"])
selected_names = {repo["name"] for repo in selected}
if set(by_repository) - selected_names:
raise GiteaError("Source manifests contain repositories absent from the module review catalog.")
scopes: list[dict[str, Any]] = []
for repo in selected:
root = workspace_root / repo["path"]
if not root.is_dir():
raise GiteaError(f"Missing source checkout for {repo['name']}; cannot infer review scope safely.")
manifest = by_repository.get(repo["name"])
paths = sorted(path.relative_to(root).as_posix() for path in root.glob("src/*/backend/manifest.py"))
if repo["name"] == "govoplan-core":
scope_id, name, kind = "core", "Core / shared shell", "core"
elif manifest:
scope_id, name, kind = manifest["id"], manifest["name"], "manifest"
else:
if paths or (root / "pyproject.toml").exists() or (root / "webui/package.json").exists():
raise GiteaError(f"{repo['name']} has implementation but no extracted manifest; inspect instead of calling it a placeholder.")
scope_id = "catalog:" + repo["name"]
name = repo["name"].removeprefix("govoplan-").replace("-", " ").title()
kind = "placeholder"
groups = source_groups(root)
scopes.append({
"scope_id": scope_id, "name": name, "repository": repo["name"],
"kind": kind, "manifest_paths": paths,
"frontend": manifest.get("frontend") if manifest else None,
"source_groups": groups,
"ui_source_count": sum(len(paths) for paths in groups.values()),
})
if len({scope["scope_id"] for scope in scopes}) != len(scopes):
raise GiteaError("Duplicate review scope IDs.")
return sorted(scopes, key=lambda scope: (scope["kind"] == "placeholder", scope["scope_id"] != "core", scope["name"].casefold()))
def issue_title(scope: dict[str, Any]) -> str:
suffix = "readiness and future UI review" if scope["kind"] == "placeholder" else "visual and interaction conformance"
return f"[UI review] {scope['name']}: {suffix}"
def source_seed(scope: dict[str, Any]) -> str:
repo = scope["repository"]
lines = [
"This is a **source-derived starting inventory, not a completed runtime audit**. Verify nested routes, embedded dialogs and contributions in the installed module context; add missing surfaces to this issue.",
"", f"Repository: [{repo}]({BASE_URL}/{OWNER}/{repo}).",
]
if scope["kind"] == "placeholder":
lines += [
"", f"Catalog entry `{repo}` is currently README-only; there is no runtime module ID, manifest or standalone WebUI to claim as reviewed. Source: [README]({source_url(repo, 'README.md')}).",
"", "- [ ] Confirm the catalog/readiness scope and record prerequisites for the first implementation.",
"- [ ] Keep the future interface review pending until actual configuration, public/operator or UI surfaces exist; do not manufacture N/A evidence to close this track.",
]
return "\n".join(lines)
for path in scope["manifest_paths"]:
lines.append(f"Manifest source: [{path}]({source_url(repo, path)}).")
if scope["kind"] == "core":
lines += [
"", "Core/shared shell additionally owns navigation/rail/breadcrumbs, module routing, page/action archetypes, cards/tables/forms/dialogs, loading and error surfaces, help affordances, authentication and user settings. Review optional-module and permission contexts, not just standalone primitives.",
]
frontend = scope["frontend"]
if frontend:
for key, label in (
("routes", "Declared routes"), ("public_routes", "Declared public routes"),
("settings_routes", "Declared settings routes"), ("nav_items", "Declared navigation"),
("view_surfaces", "Declared view, settings and contributed surfaces"),
):
entries = frontend.get(key) or []
lines += ["", f"**{label} ({len(entries)}):**"]
if not entries:
lines.append("None declared in this manifest; verify indirect/contributed surfaces before marking anything not applicable.")
for entry in entries:
identity = entry.get("path") or entry.get("id") or entry.get("component") or "unnamed declaration"
detail = entry.get("component") or entry.get("label") or entry.get("kind") or ""
lines.append(f"- `{identity}`" + (f"{detail}" if detail else ""))
elif scope["kind"] != "core":
lines += [
"", "No standalone frontend is declared. **The review is still pending:** inspect owned configuration/admin workflows, manifest documentation, errors and any interfaces contributed through host modules, public routes or operator tools. Record concrete evidence before claiming a principle does not apply.",
]
lines += ["", f"<details><summary>Source entrypoint seed ({scope['ui_source_count']} TSX files; classification is heuristic)</summary>", ""]
for label, paths in scope["source_groups"].items():
if not paths:
continue
lines += [f"**{label} ({len(paths)}):**", ""]
for path in paths[:40]:
lines.append(f"- [{path}]({source_url(repo, path)})")
if len(paths) > 40:
lines.append(f"- {len(paths) - 40} further files: inspect [the source tree]({source_url(repo, 'webui/src')}); expand the issue inventory during review.")
lines.append("")
lines += ["</details>"]
return "\n".join(lines)
def issue_body(scope: dict[str, Any], epic_url: str) -> str:
lines = [
marker(scope["scope_id"]), "## Status and objective", "",
f"**Pending / not reviewed.** This module track belongs to the [product-wide UI review epic]({epic_url}).",
f"Apply the [Core design principles]({PRINCIPLES_URL}) using the [shared review process]({PROCESS_URL}). The current heading-help icon pass and any existing isolated fixes are preparation, not evidence that this whole module is complete. Source documents are being prepared in the current working tree; this issue does not claim they are released.",
"", "Prioritize usability defects, consistent interaction and shared-component fixes before broader features. Preserve permissions, security, optional-module boundaries and data integrity.",
"", "## Source inventory to verify", "", source_seed(scope),
"", "## Review and implementation TODO", "",
"- [ ] Confirm every actual page, nested route, dialog, field/form, table/tree, settings level, public/operator surface, widget and cross-module contribution; document role/module prerequisites.",
"- [ ] Exercise EN/DE, keyboard/focus, narrow and wide layouts, long content, empty/loading/error states and realistic datasets.",
"- [ ] Check UI-01 heading/label help placement and UI-02 display-first/scoped editing; document any justified large-grid bulk-edit exception with explicit mode, Save/Cancel and dirty-navigation guard.",
"- [ ] Verify consistent top-right actions (Reload left of New), clean/dirty Save/Cancel behavior, destructive separation and safe navigation/reload.",
"- [ ] Verify full-width cards/tables, visible last-column actions, pagination, initial sizing, two-way pointer/keyboard resizing and preference reload with fixed columns and horizontal overflow.",
"- [ ] Check scoped progress/error feedback and predictable tree selection versus expansion; no unnecessary global blocking or repeated background reload.",
"- [ ] Record findings and implement shared-contract corrections plus all affected consumers, not local CSS/action-row exceptions without justification.",
"- [ ] Verify save/cancel/retry and partial-failure behavior without unintended writes, sends, deletes or loss of persisted/unsaved data.",
"- [ ] Update owning manifest-driven EN/DE documentation; record targeted automated checks and manual evidence against actual module surfaces.",
"- [ ] Complete the principle matrix, list unresolved decisions/manual checks and link follow-ups before proposing closure.",
]
if scope["scope_id"] == "campaigns":
lines += [
"", "### Campaign-specific starting direction", "",
"- [ ] Present a compact read-only campaign settings dashboard/overview and use explicit scoped edit dialogs for settings instead of a permanently editable form wall.",
"- [ ] Keep large recipient/attachment tables practical through an explicit bulk-edit mode where appropriate, with Save/Cancel, dirty-state protection and reload persistence; this is the documented UI-02 exception, not silent autosave.",
"- [ ] Review the complete compose → attachments → validation/review → delivery/report/operator workflow, including mail-profile migration, ZIP policy, multiple recipients and SMTP/IMAP progress, without sending live messages just to collect UI evidence.",
]
lines += [
"", "## Principle applicability / application / evidence / exceptions", "",
"Reviewed Core principle revision: **not yet recorded**. No exceptions approved.", "",
"| Principle | Applicable surfaces / justified N/A | Applied / remaining work | Evidence | Exception / owner / follow-up |",
"| --- | --- | --- | --- | --- |",
]
for identity, description in PRINCIPLES:
lines.append(f"| {identity}: {description} | Pending inventory | Pending review | Not yet recorded | None approved |")
lines += [
"", "When a principle changes after this review, re-check applicability and record current evidence. Reopen this issue or link an owned follow-up for outstanding work; notify the central epic. A past review must not silently remain green against an obsolete rule.",
"", "## Findings / TODO / done ledger", "",
"| Finding / surface / reproduction | Principle and expected behavior | TODO / implementation or follow-up | Verified done evidence |",
"| --- | --- | --- | --- |",
"| Full review not started | UI-01UI-09 | Inventory and review pending | None; no completed review claimed |",
"", "## Manual checks, decisions and closure evidence", "",
"- Manual work pending: safe actual-module walkthrough in both languages and realistic viewports, keyboard/focus, permissions/optional-module contexts, loading/error/empty states, edits/Save/Cancel/navigation/reload, tables/trees and progress.",
"- Decisions: none invented by this bootstrap. Record any product or policy choice with context and a recommendation; isolate independent implementation work from blocked decisions.",
"- Automated evidence: not yet recorded for this complete module review. Existing targeted fixes/tests may be linked as partial evidence only.",
"- Closure gate: verified inventory, complete current principle matrix, resolved required findings, owning EN/DE documentation and automated/manual evidence. Unimplemented placeholders remain pending until real surfaces can be reviewed or an explicit catalog/product decision changes scope.",
]
return "\n".join(lines) + "\n"
def result_record(scope: dict[str, Any], issue: dict[str, Any] | None, action: str) -> dict[str, Any]:
return {
"scope_id": scope["scope_id"], "name": scope["name"], "repository": scope["repository"],
"kind": scope["kind"], "ui_source_count": scope["ui_source_count"],
"number": issue.get("number") if issue else None,
"url": issue.get("html_url") if issue else None,
"state_at_verification": issue.get("state") if issue else None,
"operation": action,
}
def render_links(records: list[dict[str, Any]]) -> str:
lines = ["### Implemented scopes — pending review", ""]
for placeholder in (False, True):
if placeholder:
lines += ["", "### Catalogued placeholders — pending readiness / future UI review", ""]
for item in records:
if (item["kind"] == "placeholder") == placeholder:
if not item["url"]:
raise GiteaError("Cannot initialize an incomplete issue link inventory.")
lines.append(f"- [ ] [{item['name']}]({item['url']}) — `{item['scope_id']}` / `{item['repository']}`; pending.")
return "\n".join(lines)
def initialized_epic_body(body: str, records: list[dict[str, Any]]) -> str:
if body.count(LIST_START) != 1 or body.count(LIST_END) != 1 or EPIC_MARKER not in body:
raise GiteaError("Epic managed-list markers are absent or ambiguous; preserve its body.")
before, tail = body.split(LIST_START, 1)
current, after = tail.split(LIST_END, 1)
if current.strip() != INITIAL_LIST:
if all(item["url"] and f"]({item['url']})" in current for item in records):
return body # Never reset human checkboxes, evidence or subsequent edits.
raise GiteaError("Epic list was already edited; update missing links manually without replacing progress.")
return before + LIST_START + "\n" + render_links(records) + "\n" + LIST_END + after
@contextmanager
def ipv4_for_target(enabled: bool) -> Iterator[None]:
original = socket.getaddrinfo
def scoped(host: Any, port: Any, family: int = 0, type: int = 0, proto: int = 0, flags: int = 0) -> Any:
return original(host, port, socket.AF_INET if host == "git.add-ideas.de" else family, type, proto, flags)
if enabled:
socket.getaddrinfo = scoped
try:
yield
finally:
socket.getaddrinfo = original
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--env-file", type=Path)
parser.add_argument("--epic", type=int, required=True, help="Existing managed UI-review epic number in GovOPlaN/govoplan")
parser.add_argument("--apply", action="store_true", help="Create missing review issues; existing issues remain untouched")
parser.add_argument("--initialize-links", action="store_true", help="One-time initialization of the untouched epic module-list placeholder")
parser.add_argument("--ipv4", action="store_true", help="Prefer IPv4 only for git.add-ideas.de; keep HTTPS verification")
args = parser.parse_args()
if args.epic <= 0 or (args.initialize_links and not args.apply):
parser.error("A positive --epic is required; --initialize-links requires --apply.")
try:
catalog = json.loads((META_ROOT / "repositories.json").read_text(encoding="utf-8"))
workspace_root = Path(catalog["default_parent"])
scopes = build_scopes(catalog, workspace_root, extract_manifests(catalog, workspace_root))
load_dotenv(args.env_file)
token = require_token()
target = RepoTarget(BASE_URL, OWNER, "govoplan")
with ipv4_for_target(args.ipv4), GiteaClient(target, token) as central:
epic = central.request_json("GET", repo_path(OWNER, "govoplan", f"/issues/{args.epic}"))
if EPIC_MARKER not in (epic.get("body") or "") or epic.get("state") != "open":
raise GiteaError("Expected an open, managed UI-review epic; no child issues created.")
epic_url = f"{BASE_URL}/{OWNER}/govoplan/issues/{args.epic}"
org_labels = {item["name"]: item["id"] for item in central.paginate(org_path(OWNER, "/labels"))}
def reconcile(scope: dict[str, Any]) -> dict[str, Any]:
repo = scope["repository"]
with GiteaClient(RepoTarget(BASE_URL, OWNER, repo), token) as client:
issues = client.paginate(repo_path(OWNER, repo, "/issues"), query={"state": "all", "type": "issues"})
existing = find_existing(issues, scope["scope_id"], issue_title(scope))
if existing:
return result_record(scope, existing, "existing")
if not args.apply:
return result_record(scope, None, "would-create")
labels = dict(org_labels)
labels.update({item["name"]: item["id"] for item in client.paginate(repo_path(OWNER, repo, "/labels"))})
desired = ["type/task", "area/webui", "area/docs", "priority/p2", f"module/{repo.removeprefix('govoplan-')}"]
desired += ["status/triage"] if scope["kind"] == "placeholder" else ["status/ready", "codex/ready"]
issue = client.request_json("POST", repo_path(OWNER, repo, "/issues"), body={
"title": issue_title(scope), "body": issue_body(scope, epic_url),
"labels": [labels[name] for name in desired if name in labels],
})
verified = client.request_json("GET", repo_path(OWNER, repo, f"/issues/{issue['number']}"))
if marker(scope["scope_id"]) not in (verified.get("body") or "") or verified.get("state") != "open":
raise GiteaError(f"New review issue verification failed for {scope['scope_id']}.")
print(f"Created {repo}#{verified['number']} (pending)", file=sys.stderr, flush=True)
return result_record(scope, verified, "created")
with ThreadPoolExecutor(max_workers=4) as executor:
records = list(executor.map(reconcile, scopes))
if args.initialize_links:
fresh = central.request_json("GET", repo_path(OWNER, "govoplan", f"/issues/{args.epic}"))
body = initialized_epic_body(fresh["body"], records)
if body != fresh["body"]:
central.request_json("PATCH", repo_path(OWNER, "govoplan", f"/issues/{args.epic}"), body={"body": body})
summary = {
"schema_version": 1, "snapshot_purpose": "Issue discovery links; live Gitea issues own review state and evidence.",
"epic": {"repository": "govoplan", "number": args.epic, "url": epic_url},
"scope_count": len(scopes),
"manifest_modules": sum(scope["kind"] == "manifest" for scope in scopes),
"implemented_scopes": sum(scope["kind"] != "placeholder" for scope in scopes),
"catalogued_placeholders": sum(scope["kind"] == "placeholder" for scope in scopes),
"created": sum(item["operation"] == "created" for item in records),
"missing": sum(item["operation"] == "would-create" for item in records),
"issues": records,
}
print(json.dumps(summary, ensure_ascii=False, indent=2))
return 0
except (GiteaError, OSError, ValueError) as exc:
print(f"UI review program: {exc}", file=sys.stderr)
return 1
if __name__ == "__main__":
raise SystemExit(main())
+28 -10
View File
@@ -5,9 +5,16 @@ import path from "node:path";
import { createHash } from "node:crypto";
import { pathToFileURL } from "node:url";
const [metaRootArgument] = process.argv.slice(2);
const [metaRootArgument, ...argumentsRest] = process.argv.slice(2);
if (!metaRootArgument) {
throw new Error("Usage: extract-webui-structure.mjs META_ROOT");
throw new Error("Usage: extract-webui-structure.mjs META_ROOT [--workspace-root ROOT]");
}
let explicitWorkspaceRoot;
for (let index = 0; index < argumentsRest.length; index++) {
if (argumentsRest[index] !== "--workspace-root" || explicitWorkspaceRoot !== undefined || !argumentsRest[index + 1]) {
throw new Error("Expected one --workspace-root ROOT option");
}
explicitWorkspaceRoot = path.resolve(argumentsRest[++index]);
}
const metaRoot = path.resolve(metaRootArgument);
@@ -15,12 +22,18 @@ const repositoryCatalog = JSON.parse(
fs.readFileSync(path.join(metaRoot, "repositories.json"), "utf8")
);
const siblingWorkspaceRoot = path.dirname(metaRoot);
const configuredWorkspaceRoot = path.resolve(repositoryCatalog.default_parent);
const workspaceRoot = fs.existsSync(
// Discovery is retained only for old direct callers. An explicit root wins
// even when another configured checkout contains more optional modules.
const workspaceRoot = explicitWorkspaceRoot ?? (fs.existsSync(
path.join(siblingWorkspaceRoot, "govoplan-core", "webui")
)
? siblingWorkspaceRoot
: configuredWorkspaceRoot;
) ? siblingWorkspaceRoot : path.resolve(repositoryCatalog.default_parent));
if (!fs.statSync(workspaceRoot).isDirectory()) throw new Error("Inventory workspace root must be an existing directory");
const actualWorkspaceRoot = fs.realpathSync(workspaceRoot);
function withinWorkspace(candidate) {
const relative = path.relative(actualWorkspaceRoot, fs.realpathSync(candidate));
return relative === "" || (!relative.startsWith(`..${path.sep}`) && relative !== ".." && !path.isAbsolute(relative));
}
if (!Array.isArray(repositoryCatalog.repositories)) throw new Error("Repository catalog requires a repositories array");
const typescriptPath = path.join(
workspaceRoot,
"govoplan-core",
@@ -96,6 +109,7 @@ const contributionTypes = new Map([
]);
const result = {
workspaceRoot: actualWorkspaceRoot,
fields: [],
actions: [],
labels: [],
@@ -111,8 +125,12 @@ const result = {
};
for (const repository of repositoryCatalog.repositories) {
const sourceRoot = path.join(workspaceRoot, repository.path, "webui", "src");
if (!repository || typeof repository.path !== "string" || !repository.path || path.isAbsolute(repository.path) || repository.path.split(/[\\/]/).includes("..")) {
throw new Error("Inventory repository paths must remain inside the selected workspace");
}
const sourceRoot = path.join(actualWorkspaceRoot, repository.path, "webui", "src");
if (!fs.existsSync(sourceRoot)) continue;
if (!withinWorkspace(sourceRoot)) throw new Error("Inventory source root escapes the selected workspace");
for (const sourcePath of sourceFiles(sourceRoot)) {
inspectSource(repository.name, sourceRoot, sourcePath);
}
@@ -135,7 +153,7 @@ function sourceFiles(root) {
}
const candidate = path.join(current, entry.name);
if (entry.isDirectory()) pending.push(candidate);
else if (/\.(?:ts|tsx)$/.test(entry.name)) files.push(candidate);
else if (entry.isFile() && /\.(?:ts|tsx)$/.test(entry.name)) files.push(candidate);
}
}
return files.sort();
@@ -150,7 +168,7 @@ function inspectSource(repository, sourceRoot, sourcePath) {
true,
sourcePath.endsWith(".tsx") ? ts.ScriptKind.TSX : ts.ScriptKind.TS
);
const relativeFile = path.relative(path.join(workspaceRoot, repository), sourcePath);
const relativeFile = path.relative(path.resolve(sourceRoot, "..", ".."), sourcePath);
const identityCounters = new Map();
function location(node) {
@@ -9,6 +9,7 @@ from collections import Counter
from dataclasses import asdict, is_dataclass
import importlib
import json
import os
from pathlib import Path
import re
import subprocess
@@ -40,6 +41,11 @@ REFERENCE_LOCALE = "de"
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--workspace-root",
type=Path,
help="Authoritative directory containing registered checkouts; never falls back to another workspace.",
)
parser.add_argument(
"--output-dir",
type=Path,
@@ -87,8 +93,9 @@ def main() -> int:
args = parser.parse_args()
catalog = json.loads((META_ROOT / "repositories.json").read_text(encoding="utf-8"))
workspace_root = _resolve_workspace_root(catalog)
webui = _extract_webui()
workspace_root = _resolve_workspace_root(catalog, args.workspace_root)
_validate_repository_roots(catalog, workspace_root)
webui = _extract_webui(workspace_root)
backend_endpoints = _extract_backend_endpoints(catalog, workspace_root)
manifests = _extract_manifests(catalog, workspace_root)
endpoint_declarations = _load_endpoint_declarations(
@@ -109,6 +116,10 @@ def main() -> int:
else None
),
)
inventory["workspace_root"] = str(workspace_root)
inventory["workspace_selection"] = (
"explicit" if args.workspace_root is not None else "legacy-discovery"
)
output_dir = args.output_dir.resolve()
output_dir.mkdir(parents=True, exist_ok=True)
@@ -212,7 +223,16 @@ def _strict_failures(
return failures
def _resolve_workspace_root(catalog: dict[str, Any]) -> Path:
def _resolve_workspace_root(
catalog: dict[str, Any], explicit_root: Path | None = None
) -> Path:
if explicit_root is not None:
root = explicit_root.expanduser().resolve()
if not root.is_dir():
raise ValueError("The explicit inventory workspace root must be an existing directory")
return root
# Compatibility for direct legacy callers only. Managed callers always
# supply their selected root; a partial checkout must not borrow sources.
sibling_root = META_ROOT.parent.resolve()
configured_root = Path(str(catalog["default_parent"])).expanduser().resolve()
repositories = catalog.get("repositories")
@@ -233,15 +253,44 @@ def _resolve_workspace_root(catalog: dict[str, Any]) -> Path:
return sibling_root if sibling_count >= configured_count else configured_root
def _extract_webui() -> dict[str, Any]:
def _validate_repository_roots(catalog: dict[str, Any], workspace_root: Path) -> None:
repositories = catalog.get("repositories")
if not isinstance(repositories, list):
raise ValueError("repository catalog has no repositories array")
for repository in repositories:
if not isinstance(repository, dict) or not isinstance(repository.get("path"), str):
raise ValueError("Invalid inventory repository path")
relative = Path(repository["path"])
if not repository["path"] or relative.is_absolute() or ".." in relative.parts:
raise ValueError("Inventory repository paths must remain inside the selected workspace")
root = workspace_root / relative
# Missing optional checkouts are allowed; links to another checkout are
# not evidence for the selected workspace.
if not root.resolve().is_relative_to(workspace_root):
raise ValueError("Inventory repository path escapes the selected workspace")
for source in (root / "src", root / "webui/src"):
if not source.resolve().is_relative_to(workspace_root):
raise ValueError("Inventory source root escapes the selected workspace")
def _extract_webui(workspace_root: Path | None = None) -> dict[str, Any]:
helper = META_ROOT / "tools" / "inventory" / "extract-webui-structure.mjs"
argv = [os.environ.get("NODE", "node"), str(helper), str(META_ROOT)]
if workspace_root is not None:
argv.extend(["--workspace-root", str(workspace_root)])
completed = subprocess.run(
["node", str(helper), str(META_ROOT)],
argv,
check=True,
capture_output=True,
text=True,
)
return json.loads(completed.stdout)
result = json.loads(completed.stdout)
if workspace_root is not None and (
not isinstance(result, dict)
or result.get("workspaceRoot") != str(workspace_root.resolve())
):
raise ValueError("WebUI collector did not confirm the selected inventory workspace")
return result
def _extract_backend_endpoints(
@@ -255,6 +304,8 @@ def _extract_backend_endpoints(
if not source_root.is_dir():
continue
for source_path in sorted(source_root.rglob("*.py")):
if not source_path.resolve().is_relative_to(workspace_root):
raise ValueError("Backend source path escapes the selected workspace")
try:
tree = ast.parse(
source_path.read_text(encoding="utf-8"),
@@ -346,15 +397,26 @@ def _extract_manifests(
catalog: dict[str, Any],
workspace_root: Path,
) -> list[dict[str, Any]]:
_validate_repository_roots(catalog, workspace_root)
source_roots = [
workspace_root / repository["path"] / "src"
for repository in catalog["repositories"]
if (workspace_root / repository["path"] / "src").is_dir()
]
core_root = next(
(workspace_root / repository["path"] / "src"
for repository in catalog["repositories"]
if repository.get("name") == "govoplan-core"),
workspace_root / "govoplan-core/src",
)
if not (core_root / "govoplan_core/core/platform_interfaces.py").is_file():
raise ValueError("Inventory requires Core interface sources in the selected workspace")
_assert_workspace_imports(workspace_root)
sys.path[:0] = [str(path) for path in source_roots]
from govoplan_core.core.platform_interfaces import ( # noqa: PLC0415
manifest_interface_catalog,
)
_assert_workspace_imports(workspace_root)
manifests: list[dict[str, Any]] = []
for repository in catalog["repositories"]:
@@ -366,7 +428,12 @@ def _extract_manifests(
manifest_path.relative_to(source_root).with_suffix("").parts
)
loaded = importlib.import_module(module_name)
source = getattr(loaded, "__file__", None)
if not isinstance(source, str) or Path(source).resolve() != manifest_path.resolve():
raise ValueError("Manifest import did not resolve to its selected workspace source")
_assert_workspace_imports(workspace_root)
manifest = loaded.get_manifest()
_assert_workspace_imports(workspace_root)
frontend = manifest.frontend
manifests.append(
{
@@ -452,6 +519,26 @@ def _extract_manifests(
return sorted(manifests, key=lambda item: item["id"])
def _assert_workspace_imports(workspace_root: Path) -> None:
# An editable installation or cached import must not stand in for a missing
# optional checkout. Direct callers with another workspace use a fresh
# process instead of replacing already-loaded application packages.
for name, module in list(sys.modules.items()):
# The Meta tools may audit a separate checkout; they are not module
# contributions and must not be confused with application packages.
package = name.partition(".")[0]
if not package.startswith("govoplan_") or package in {
"govoplan_devkit", "govoplan_release"
}:
continue
filename = getattr(module, "__file__", None)
locations = list(getattr(module, "__path__", ()))
if isinstance(filename, str):
locations.append(filename)
if any(not Path(location).resolve().is_relative_to(workspace_root) for location in locations):
raise ValueError("A GovOPlaN import originates outside the selected inventory workspace")
def _assemble_inventory(
*,
webui: dict[str, Any],