diff --git a/README.md b/README.md index b3402a2..f9419d3 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,8 @@ Canonical policy documents live in `docs/`: - [ACCESS_RBAC_MODEL.md](docs/ACCESS_RBAC_MODEL.md) - [GOVERNANCE_MODEL.md](docs/GOVERNANCE_MODEL.md) - [MODULE_ARCHITECTURE.md](docs/MODULE_ARCHITECTURE.md) +- [INTEGRITY_PERFORMANCE_CONTRACT.md](docs/INTEGRITY_PERFORMANCE_CONTRACT.md) +- [TABULAR_SOURCE_CONTRACT.md](docs/TABULAR_SOURCE_CONTRACT.md) - [DEPLOYMENT_OPERATOR_GUIDE.md](docs/DEPLOYMENT_OPERATOR_GUIDE.md) - [CODEX_WORKFLOW.md](docs/CODEX_WORKFLOW.md) diff --git a/alembic/env.py b/alembic/env.py index c6cc472..7cd54a3 100644 --- a/alembic/env.py +++ b/alembic/env.py @@ -26,7 +26,9 @@ from govoplan_core.tenancy.scope import scope_registry config = context.config database_url = config.attributes.get("database_url") or settings.database_url -config.set_main_option("sqlalchemy.url", database_url) +# Alembic stores options through ConfigParser: escape only its interpolation +# syntax so URL-encoded credentials/socket paths reach SQLAlchemy unchanged. +config.set_main_option("sqlalchemy.url", database_url.replace("%", "%%")) if config.config_file_name is not None: # Migrations can run inside the long-lived application process when module diff --git a/docs/INTEGRITY_PERFORMANCE_CONTRACT.md b/docs/INTEGRITY_PERFORMANCE_CONTRACT.md new file mode 100755 index 0000000..450e23d --- /dev/null +++ b/docs/INTEGRITY_PERFORMANCE_CONTRACT.md @@ -0,0 +1,101 @@ +# Integrity-preserving performance contracts + +These are implementation guarantees and regression-test boundaries, not a +security certification or production load-test result. Feature-specific policies +and help remain in the owning modules' English and German documentation topics. + +## Refreshes, edits and table rendering + +The shell runs at most one module-load refresh per authority generation. Multiple +invalidations coalesce into one trailing refresh; stale results and errors cannot +replace a newer generation. Focus refreshes are throttled to five seconds and +focus/visibility refreshes on hidden pages are suppressed; explicit module +invalidations still trigger a read. Authentication/tenant reset disposes the +previous controller. Authentication and authorization checks are not cached away. + +Editable modules must reconcile a save against the submitted draft and accepted +server revision: edits made while a request is pending remain dirty. Completion +must be fenced by security-relevant authority and selection, not merely an auth +object's reference identity. A harmless profile refresh must not discard an +accepted newly created ID and invite a duplicate create. An old account's mutation +continuation must not reload its catalogue into the current account's page. + +DataGrid precomputes first-occurrence row indices once for client sorting and +filtering. Duplicate object references, primitive values, `NaN` and sparse arrays +retain `Array.indexOf` behavior. Comparator order, visible pagination, server-side +pagination, sizing and resize rules are unchanged. Do not replace this with a +last-occurrence map or change ordering as a side effect of an optimization. + +## Conditional responses + +The shared JSON GET middleware performs route handling, including authorization, +before considering `If-None-Match`. It buffers only responses up to 1 MiB for a +body-derived ETag. Known larger responses bypass buffering; an unknown-length +stream crossing that limit replays its exact prefix and streams the remainder. +No data is truncated and no large joined copy is created. The crossing chunk is +already producer-owned: this is not a process-wide or route-output memory limit. +Empty chunks do not accumulate. Large responses may no longer receive a +middleware-generated ETag/304; explicit route ETags remain intact. Small-response +cache semantics and credential/language/context `Vary` fields are retained. + +## Shared helpers and concurrency + +Central helpers replace exact live-code duplicates only. Actor precedence, +whitespace handling and service-account differences remain explicit owner choices; +historical migration code is not redirected to mutable runtime helpers. Connector +search ACL token projection keeps its first-seen ordering and existing 500-token +cap, stopping work once that cap is reached. Provider schemas are inferred in one +pass without storing a second list of every column value. + +The keyed-list three-way merge retains insertion anchors when unrelated fields +change. Concurrent additions use deterministic ordering; contradictory anchors +produce a collection-order conflict instead of silently relocating an item. +No existing endpoint is newly opted into merge behavior by this change. + +SQL JSON authorization predicates support the explicitly tested SQLite and +PostgreSQL dialects, retain exact string membership and reject unsupported +dialects. Apply tenant and authorization predicates before counting/pagination; +never page a broader result first and filter away unauthorized records afterward. +SQLite execution and PostgreSQL SQL compilation are not substitutes for a +deployment's PostgreSQL concurrency and representative-data load tests. + +## Migration connection URLs + +Alembic preserves the configured database URL exactly in online and offline +migration modes, including percent-encoded credentials and PostgreSQL Unix-socket +paths. Escaping applies only at its ConfigParser boundary; operators must not +double-escape `%` in `DATABASE_URL` or alter working connection credentials to +work around interpolation errors. This does not change the target database, +authentication, TLS policy, or migration contents. + +## Deutsch: Integrität vor Geschwindigkeit + +Der zentrale Modul-Refresh bündelt gleichzeitige Auslöser und verwirft veraltete +Ergebnisse einschließlich Fehlermeldungen. Ein Wechsel von Anmeldung oder Mandant +beendet die bisherige Generation. Fokusaktualisierungen sind auf einen Auslöser +je fünf Sekunden begrenzt. Berechtigungsprüfungen bleiben erhalten. + +Speicherantworten dürfen zwischenzeitliche Bearbeitungen nicht überschreiben. +Ein unveränderter Berechtigungskontext mit einem neuen Profilobjekt darf eine +bereits bestätigte neue ID oder Revision nicht verwerfen. Umgekehrt dürfen alte +Anfragen nach einem Kontowechsel keine Daten in den neuen Kontext übernehmen. +Die DataGrid-Optimierung erhält Reihenfolge, Filter-, Seiten- und Größenverhalten. + +Die ETag-Middleware puffert höchstens 1 MiB Nutzdaten zuzüglich eines bereits vom +Erzeuger gelieferten Grenz-Chunks. Größere Antworten werden vollständig weitergereicht, +nicht abgeschnitten; automatisch erzeugte ETags können dabei entfallen. +Autorisierung läuft auch bei bedingten Anfragen. Das ist keine allgemeine +Speicherbegrenzung für Routen oder Prozesse. + +Gemeinsame Helfer erhalten die bisherigen fachlichen Unterschiede. Listen-Merges +bewahren Einfügepositionen oder melden widersprüchliche Reihenfolgen explizit als +Konflikt. Datenbankseitige Autorisierung erfolgt vor Zählung und Seitenauswahl. +Regressionstests belegen diese Verträge; reale Provider-, PostgreSQL- und Lasttests +in einer repräsentativen Umgebung bleiben Teil der Betriebsfreigabe. + +Alembic übernimmt die konfigurierte Datenbank-URL in Online- und Offline-Läufen +unverändert, einschließlich prozentkodierter Zugangsdaten und PostgreSQL- +Unix-Socket-Pfade. Die Maskierung erfolgt ausschließlich an der ConfigParser- +Grenze; `%` in `DATABASE_URL` nicht doppelt maskieren und funktionierende +Zugangsdaten nicht als Umgehung ändern. Zieldatenbank, Anmeldung, TLS-Vorgaben und +Migrationsinhalte bleiben unverändert. diff --git a/docs/TABULAR_SOURCE_CONTRACT.md b/docs/TABULAR_SOURCE_CONTRACT.md index c516f35..01be842 100644 --- a/docs/TABULAR_SOURCE_CONTRACT.md +++ b/docs/TABULAR_SOURCE_CONTRACT.md @@ -19,3 +19,72 @@ Connector health and preview diagnostics must contain no credentials, endpoint userinfo, row values, or unbounded remote error bodies. A Datasource origin preserves this contract so registration and staging do not erase source mode, health, pushdown, or preview-limit evidence. + +## Durable CSV imports and original evidence + +`TabularCsvSource` optionally accompanies a durable `TabularSnapshotInput` or +`DatasourceStageInput`. It carries the exact submitted Unicode text, delimiter, +explicit value mode and parser profile. It is not part of ordinary catalogue, +preview or stage DTOs. Transient inspection does not retain original content. + +The `text` mode preserves cell strings, including whitespace, leading zeroes, +decimal spelling, boolean-looking text and explicit empty cells. It rejects +malformed quoting and rows with missing or extra cells. Header normalization is +unchanged. The API default remains `legacy_typed` for existing integrations; +interactive CSV imports offer text mode by default and an explicit legacy choice. +JSON and existing stored snapshots are not reinterpreted. Core and Datasources +retain distinct versioned legacy parser profiles where their historical coercion +rules differ. Shared schema inference preserves first-seen column order, missing +value nullability and the owning provider's type naming. + +Owners verify that the source reparses to exactly the stored projection, including +scalar types: `true`, `1` and `1.0` are not equivalent evidence. Raw input and row +projections each have a 5,000,000-byte limit; row parsing is capped at 10,000 rows. +The original text has its own UTF-8 SHA-256 and byte count, separate from the +existing row fingerprint. Only a small allowlisted source summary enters metadata. +Checksums detect drift; they are not digital signatures or protection against an +attacker who can rewrite the database and all its evidence. + +Original exports are explicit owner APIs, tenant scoped, integrity checked and +`no-store`. Datasources additionally requires administrator scope, audits the +export, and denies the whole original when current or historical governance +restricts any row or field. Freezing verifies the prior summary before copying +source evidence and retains prior policy restrictions, including referenced +policy evaluations. At most 32 distinct governance snapshots may accompany one +original; further incompatible history fails explicitly. Payload disposal also +disposes retained original content. Connectors applies its own current read, +tenant and lifecycle checks. See the owning module's documentation for endpoints. + +Original UTF-8 text is not proof of pre-decoding file bytes, and does not undo +CSV spreadsheet formula semantics. Exported content is deliberately unmodified; +operators must treat it as untrusted input when opening it in a spreadsheet. +New nullable columns require the owning modules' additive migrations. Historical +rows remain unchanged and report original content unavailable, not reconstructed. +Back up retained originals before any schema downgrade that removes those columns. + +## Deutsch: CSV-Datentreue + +Dauerhafte CSV-Importe können den unveränderten übermittelten Unicode-Text mit +Trennzeichen, Parserprofil und explizitem Wertemodus aufbewahren. Der Textmodus +erhält Zellwerte einschließlich Leerzeichen, führender Nullen und Dezimalschreibweise. +Fehlerhafte Zeilen werden abgewiesen. Die API bleibt aus Kompatibilitätsgründen bei +der bisherigen Typumwandlung als Standard; im Importdialog ist Text voreingestellt. +Bestehende Daten und JSON-Importe werden nicht neu interpretiert. + +Original und Zeilenprojektion werden getrennt begrenzt und geprüft; boolesche +Werte, Ganzzahlen und Gleitkommazahlen sind keine austauschbaren Belege. Der +Originaltext erscheint weder im Katalog noch in Vorschauantworten. Die expliziten +Export-APIs prüfen Mandant, Berechtigungen, Lebenszyklus und gespeicherten Hash. +Datasources verlangt zusätzlich Administrationsrechte, protokolliert Exporte und +berücksichtigt aktuelle sowie historische Zeilen-, Feld- und Zugriffsrichtlinien. +Eingeschränkte Originale werden vollständig gesperrt, nicht teilweise freigegeben. +Eingefrorene Kopien übernehmen diese Einschränkungen; nach 32 unterschiedlichen +Richtlinienständen wird eine weitere Kopie mit neuer Richtlinie explizit abgewiesen. +Die Aufbewahrungsbereinigung entfernt auch das gespeicherte Original. + +Die Grenzen betragen jeweils 5.000.000 UTF-8-Bytes für Original und Projektion +sowie 10.000 Zeilen. Prüfsummen sind keine Signaturen. Ein CSV-Original bleibt beim +Export unverändert und kann Tabellenkalkulationsformeln enthalten. Frühere +Dateikodierungen lassen sich daraus nicht rekonstruieren. Additive Migrationen +ändern keine historischen Zeilen; fehlende Originale werden nicht erfunden. +Vor einem Schema-Downgrade sind aufbewahrte Originale zu sichern. diff --git a/docs/migration-release-baselines.json b/docs/migration-release-baselines.json index 1383552..cefd3c3 100644 --- a/docs/migration-release-baselines.json +++ b/docs/migration-release-baselines.json @@ -2768,6 +2768,498 @@ "release": "0.1.45", "squash_policy": "reviewed-manual", "track": "release" + }, + { + "heads": [ + { + "owner": "govoplan-notifications", + "revision": "6e2f91ab4c70" + }, + { + "owner": "govoplan-poll", + "revision": "6e7f8a9b0c1d" + }, + { + "owner": "govoplan-dashboard", + "revision": "7b9d2f4a6c8e" + }, + { + "owner": "govoplan-tasks", + "revision": "7c4d9a2e1f30" + }, + { + "owner": "govoplan-records", + "revision": "8a6c4e2f1b3d" + }, + { + "owner": "govoplan-voting", + "revision": "8b9c0d1e2f3a" + }, + { + "owner": "govoplan-tickets", + "revision": "8d1f4b7a2c5e" + }, + { + "owner": "govoplan-quick-access", + "revision": "9a4e6c2d8f10" + }, + { + "owner": "govoplan-helpdesk", + "revision": "9e2a5c8f1b4d" + }, + { + "owner": "govoplan-workflow-engine", + "revision": "9e6b3f8a2c7d" + }, + { + "owner": "govoplan-files", + "revision": "a2b3c4d5e701" + }, + { + "owner": "govoplan-dataflow", + "revision": "a3d7f1c5e9b2" + }, + { + "owner": "govoplan-templates", + "revision": "a3f7c9d2e1b4" + }, + { + "owner": "govoplan-organizations", + "revision": "a61e4d9c72b8" + }, + { + "owner": "govoplan-wiki", + "revision": "a7c2e9f4b1d6" + }, + { + "owner": "govoplan-cases", + "revision": "a7c4e2f9b1d6" + }, + { + "owner": "govoplan-mandates", + "revision": "a8b1c2d3e4f5" + }, + { + "owner": "govoplan-approvals", + "revision": "a91c4e72b5d8" + }, + { + "owner": "govoplan-policy", + "revision": "a9c4e7b2d5f8" + }, + { + "owner": "govoplan-search", + "revision": "b2c3d4e5f607" + }, + { + "owner": "govoplan-tenancy", + "revision": "b3d8e1f4a6c2" + }, + { + "owner": "govoplan-mail", + "revision": "b5d6e7f8091a" + }, + { + "owner": "govoplan-risk-compliance", + "revision": "b9c0d1e2f3a4" + }, + { + "owner": "govoplan-services", + "revision": "b9c2d3e4f5a6" + }, + { + "owner": "govoplan-audit", + "revision": "b9e2f5a8c3d6" + }, + { + "owner": "govoplan-parties", + "revision": "c0d3e4f5a6b7" + }, + { + "owner": "govoplan-idm", + "revision": "c2d3e4f5a6b7" + }, + { + "owner": "govoplan-identity-trust", + "revision": "c3f5a7b9d1e2" + }, + { + "owner": "govoplan-projects", + "revision": "c4a1e8f2d6b9" + }, + { + "owner": "govoplan-core", + "revision": "c58a2d7e9f10" + }, + { + "owner": "govoplan-views", + "revision": "c6f2a9d4e7b1" + }, + { + "owner": "govoplan-reporting", + "revision": "c8d5e2f6a9b3" + }, + { + "owner": "govoplan-decisions", + "revision": "d1e4f5a6b7c8" + }, + { + "owner": "govoplan-calendar", + "revision": "d24e5f607182" + }, + { + "owner": "govoplan-connectors", + "revision": "d2a4c6e8f0b1" + }, + { + "owner": "govoplan-docs", + "revision": "d3e7a1c5f9b2" + }, + { + "owner": "govoplan-forms-runtime", + "revision": "d6a8b0c2e4f6" + }, + { + "owner": "govoplan-addresses", + "revision": "d6e8f9a0b1c2" + }, + { + "owner": "govoplan-scheduling", + "revision": "d7a4c1e8f205" + }, + { + "owner": "govoplan-postbox", + "revision": "d8b4f1a6c9e2" + }, + { + "owner": "govoplan-committee", + "revision": "d8b9f0a1c2e3" + }, + { + "owner": "govoplan-datasources", + "revision": "e2b8d4a0f6c3" + }, + { + "owner": "govoplan-encryption", + "revision": "e5b7c9d1f3a4" + }, + { + "owner": "govoplan-payments", + "revision": "e7b9c1d3f5a7" + }, + { + "owner": "govoplan-dist-lists", + "revision": "e7c3a9d1b5f2" + }, + { + "owner": "govoplan-access", + "revision": "e9a2c5f8b1d4" + }, + { + "owner": "govoplan-campaign", + "revision": "f3c7a9d2e6b1" + } + ], + "owner_heads": [ + { + "owner": "govoplan-access", + "revisions": [ + "e9a2c5f8b1d4" + ] + }, + { + "owner": "govoplan-addresses", + "revisions": [ + "d6e8f9a0b1c2" + ] + }, + { + "owner": "govoplan-approvals", + "revisions": [ + "a91c4e72b5d8" + ] + }, + { + "owner": "govoplan-audit", + "revisions": [ + "b9e2f5a8c3d6" + ] + }, + { + "owner": "govoplan-calendar", + "revisions": [ + "d24e5f607182" + ] + }, + { + "owner": "govoplan-campaign", + "revisions": [ + "f3c7a9d2e6b1" + ] + }, + { + "owner": "govoplan-cases", + "revisions": [ + "a7c4e2f9b1d6" + ] + }, + { + "owner": "govoplan-committee", + "revisions": [ + "d8b9f0a1c2e3" + ] + }, + { + "owner": "govoplan-connectors", + "revisions": [ + "d2a4c6e8f0b1" + ] + }, + { + "owner": "govoplan-core", + "revisions": [ + "c58a2d7e9f10" + ] + }, + { + "owner": "govoplan-dashboard", + "revisions": [ + "7b9d2f4a6c8e" + ] + }, + { + "owner": "govoplan-dataflow", + "revisions": [ + "a3d7f1c5e9b2" + ] + }, + { + "owner": "govoplan-datasources", + "revisions": [ + "e2b8d4a0f6c3" + ] + }, + { + "owner": "govoplan-decisions", + "revisions": [ + "d1e4f5a6b7c8" + ] + }, + { + "owner": "govoplan-dist-lists", + "revisions": [ + "e7c3a9d1b5f2" + ] + }, + { + "owner": "govoplan-docs", + "revisions": [ + "d3e7a1c5f9b2" + ] + }, + { + "owner": "govoplan-encryption", + "revisions": [ + "e5b7c9d1f3a4" + ] + }, + { + "owner": "govoplan-files", + "revisions": [ + "a2b3c4d5e701" + ] + }, + { + "owner": "govoplan-forms", + "revisions": [ + "e1f2a3b4c5d6" + ] + }, + { + "owner": "govoplan-forms-runtime", + "revisions": [ + "d6a8b0c2e4f6" + ] + }, + { + "owner": "govoplan-helpdesk", + "revisions": [ + "9e2a5c8f1b4d" + ] + }, + { + "owner": "govoplan-identity", + "revisions": [ + "5c6d7e8f9a10" + ] + }, + { + "owner": "govoplan-identity-trust", + "revisions": [ + "c3f5a7b9d1e2" + ] + }, + { + "owner": "govoplan-idm", + "revisions": [ + "c2d3e4f5a6b7" + ] + }, + { + "owner": "govoplan-mail", + "revisions": [ + "b5d6e7f8091a" + ] + }, + { + "owner": "govoplan-mandates", + "revisions": [ + "a8b1c2d3e4f5" + ] + }, + { + "owner": "govoplan-notifications", + "revisions": [ + "6e2f91ab4c70" + ] + }, + { + "owner": "govoplan-organizations", + "revisions": [ + "a61e4d9c72b8" + ] + }, + { + "owner": "govoplan-parties", + "revisions": [ + "c0d3e4f5a6b7" + ] + }, + { + "owner": "govoplan-payments", + "revisions": [ + "e7b9c1d3f5a7" + ] + }, + { + "owner": "govoplan-policy", + "revisions": [ + "a9c4e7b2d5f8" + ] + }, + { + "owner": "govoplan-poll", + "revisions": [ + "6e7f8a9b0c1d" + ] + }, + { + "owner": "govoplan-postbox", + "revisions": [ + "d8b4f1a6c9e2" + ] + }, + { + "owner": "govoplan-projects", + "revisions": [ + "c4a1e8f2d6b9" + ] + }, + { + "owner": "govoplan-quick-access", + "revisions": [ + "9a4e6c2d8f10" + ] + }, + { + "owner": "govoplan-records", + "revisions": [ + "8a6c4e2f1b3d" + ] + }, + { + "owner": "govoplan-reporting", + "revisions": [ + "c8d5e2f6a9b3" + ] + }, + { + "owner": "govoplan-risk-compliance", + "revisions": [ + "b9c0d1e2f3a4" + ] + }, + { + "owner": "govoplan-scheduling", + "revisions": [ + "d7a4c1e8f205" + ] + }, + { + "owner": "govoplan-search", + "revisions": [ + "b2c3d4e5f607" + ] + }, + { + "owner": "govoplan-services", + "revisions": [ + "b9c2d3e4f5a6" + ] + }, + { + "owner": "govoplan-tasks", + "revisions": [ + "7c4d9a2e1f30" + ] + }, + { + "owner": "govoplan-templates", + "revisions": [ + "a3f7c9d2e1b4" + ] + }, + { + "owner": "govoplan-tenancy", + "revisions": [ + "b3d8e1f4a6c2" + ] + }, + { + "owner": "govoplan-tickets", + "revisions": [ + "8d1f4b7a2c5e" + ] + }, + { + "owner": "govoplan-views", + "revisions": [ + "c6f2a9d4e7b1" + ] + }, + { + "owner": "govoplan-voting", + "revisions": [ + "8b9c0d1e2f3a" + ] + }, + { + "owner": "govoplan-wiki", + "revisions": [ + "a7c2e9f4b1d6" + ] + }, + { + "owner": "govoplan-workflow-engine", + "revisions": [ + "9e6b3f8a2c7d" + ] + } + ], + "recorded_at": "2026-09-08T10:17:38Z", + "release": "0.1.46", + "squash_policy": "reviewed-manual", + "track": "release" } ], "version": 1 diff --git a/pyproject.toml b/pyproject.toml index 496ee9d..d48fe39 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "govoplan-core" -version = "0.1.45" +version = "0.1.46" description = "Reusable GovOPlaN platform core, access, tenancy, and RBAC components." readme = "README.md" requires-python = ">=3.12" diff --git a/src/govoplan_core/core/concurrency.py b/src/govoplan_core/core/concurrency.py index ba819d7..5036b6c 100644 --- a/src/govoplan_core/core/concurrency.py +++ b/src/govoplan_core/core/concurrency.py @@ -2,6 +2,7 @@ from __future__ import annotations import copy import hashlib +import heapq import json from dataclasses import dataclass, field from typing import Any, Callable, Iterable, Mapping, Sequence @@ -436,10 +437,33 @@ def _merge_list( result_by_id[identity] = merged.value order_source = local_order if local_reordered and not current_reordered else current_order - merged_order = [identity for identity in order_source if identity in result_by_id] - for identity in identities: - if identity in result_by_id and identity not in merged_order: - merged_order.append(identity) + secondary_order = current_order if order_source is local_order else local_order + # Keep the chosen side's order and both sides' insertion anchors. Appending + # missing IDs would silently relocate an insertion during a disjoint edit. + # An incompatible reorder/insertion cycle is an explicit conflict. + edges: dict[str, set[str]] = {identity: set() for identity in result_by_id} + incoming = dict.fromkeys(result_by_id, 0) + for order, insertions_only in ((order_source, False), (secondary_order, True)): + selected = [identity for identity in order if identity in result_by_id] + for left, right in zip(selected, selected[1:]): + if insertions_only and left in base_by_id and right in base_by_id: + continue + if right not in edges[left]: + edges[left].add(right) + incoming[right] += 1 + priority = {identity: index for index, identity in enumerate(identities)} + ready = [(priority[identity], identity) for identity, count in incoming.items() if count == 0] + heapq.heapify(ready) + merged_order: list[str] = [] + while ready: + _, identity = heapq.heappop(ready) + merged_order.append(identity) + for following in edges[identity]: + incoming[following] -= 1 + if incoming[following] == 0: + heapq.heappush(ready, (priority[following], following)) + if len(merged_order) != len(result_by_id): + return _conflict(path, "collection_reorder", base, local, current) return ThreeWayMergeResult( value=[result_by_id[identity] for identity in merged_order], conflicts=conflicts, diff --git a/src/govoplan_core/core/datasources.py b/src/govoplan_core/core/datasources.py index 3a60be0..c0dd175 100644 --- a/src/govoplan_core/core/datasources.py +++ b/src/govoplan_core/core/datasources.py @@ -14,6 +14,7 @@ from govoplan_core.core.tabular_sources import ( DEFAULT_PREVIEW_BYTES, DEFAULT_PREVIEW_TIMEOUT_MS, TabularPreviewDiagnostic, + TabularCsvSource, TabularPushdown, TabularSourceHealth, TabularSourceMode, @@ -369,6 +370,7 @@ class DatasourceStageInput: provenance: Mapping[str, object] = field(default_factory=dict) metadata: Mapping[str, object] = field(default_factory=dict) governance: DatasourceGovernance | None = None + csv_source: TabularCsvSource | None = None @dataclass(frozen=True, slots=True) diff --git a/src/govoplan_core/core/modules.py b/src/govoplan_core/core/modules.py index 954195d..8d2301f 100644 --- a/src/govoplan_core/core/modules.py +++ b/src/govoplan_core/core/modules.py @@ -409,6 +409,25 @@ class DocumentationTopic: metadata: Mapping[str, Any] = field(default_factory=dict) +def localize_documentation_topics( + topics: Iterable[DocumentationTopic], + *, + locale: str, + translations: Mapping[str, Mapping[str, str]], +) -> tuple[DocumentationTopic, ...]: + """Merge owner-supplied text translations without moving feature content.""" + localized: list[DocumentationTopic] = [] + for topic in topics: + translated = translations.get(topic.id) + if translated is None: + localized.append(topic) + continue + values = {name: dict(value) for name, value in topic.translations.items()} + values[locale] = {**values.get(locale, {}), **translated} + localized.append(replace(topic, translations=values)) + return tuple(localized) + + def localizable_documentation_metadata_keys( topic: DocumentationTopic, ) -> tuple[str, ...]: diff --git a/src/govoplan_core/core/principal_helpers.py b/src/govoplan_core/core/principal_helpers.py new file mode 100755 index 0000000..f8a027c --- /dev/null +++ b/src/govoplan_core/core/principal_helpers.py @@ -0,0 +1,37 @@ +"""Pure principal attribution mechanics, not authorization or tenant resolution. + +The two existing contracts intentionally differ in precedence and whitespace. +Callers retain their own service-account, scope and resource-access decisions. +""" + + +def principal_actor_ids(principal: object) -> tuple[str, ...]: + """Account-first legacy IDs, unique in encounter order; retain nonblank text.""" + user = getattr(principal, "user", None) + return tuple( + dict.fromkeys( + str(value) + for value in ( + getattr(principal, "account_id", None), + getattr(principal, "identity_id", None), + getattr(principal, "membership_id", None), + getattr(user, "id", None), + ) + if str(value or "").strip() + ) + ) + + +def principal_user_first_actor(principal: object) -> str | None: + """First nonblank user/account/identity/membership ID, with trimmed text.""" + user = getattr(principal, "user", None) + for value in ( + getattr(user, "id", None), + getattr(principal, "account_id", None), + getattr(principal, "identity_id", None), + getattr(principal, "membership_id", None), + ): + candidate = str(value or "").strip() + if candidate: + return candidate + return None diff --git a/src/govoplan_core/core/tabular_sources.py b/src/govoplan_core/core/tabular_sources.py index 680832c..3628906 100644 --- a/src/govoplan_core/core/tabular_sources.py +++ b/src/govoplan_core/core/tabular_sources.py @@ -1,11 +1,15 @@ from __future__ import annotations import csv +import hashlib import io +import json +import math import re -from collections.abc import Mapping, Sequence +from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass, field from datetime import datetime +from decimal import Decimal from typing import Literal, Protocol, runtime_checkable @@ -17,6 +21,77 @@ DEFAULT_PREVIEW_TIMEOUT_MS = 2_000 TabularSourceMode = Literal["live", "cached", "file_backed", "static"] TabularHealthStatus = Literal["healthy", "warning", "error", "unknown"] TabularDiagnosticSeverity = Literal["info", "warning", "error"] +CsvValueMode = Literal["legacy_typed", "text"] + + +@dataclass(frozen=True, slots=True) +class TabularCsvSource: + """Original upload text, retained only with an explicitly durable import. + + This is never catalogue metadata or transient preview retention. Owners + enforce access, size limits, lifecycle and export authorization separately. + """ + + text: str + delimiter: str = "," + value_mode: CsvValueMode = "legacy_typed" + parser_profile: str = "core.csv.v1" + + +def csv_source_payload(source: TabularCsvSource, *, max_bytes: int = 5_000_000) -> dict[str, object]: + encoded = _csv_utf8_bytes(source.text) + if len(encoded) > max_bytes: + raise TabularSourceValidationError(f"Original CSV input is limited to {max_bytes:,} UTF-8 bytes.") + if source.value_mode not in {"text", "legacy_typed"} or len(source.delimiter) != 1: + raise TabularSourceValidationError("Invalid CSV source parsing options.") + return { + "text": source.text, + "delimiter": source.delimiter, + "value_mode": source.value_mode, + "parser_profile": source.parser_profile, + "sha256": hashlib.sha256(encoded).hexdigest(), + "byte_count": len(encoded), + } + + +def csv_source_summary(payload: Mapping[str, object]) -> dict[str, object]: + """Allowlist the small, non-content evidence safe for catalogue DTOs.""" + result = {key: payload[key] for key in ("delimiter", "value_mode", "parser_profile", "sha256", "byte_count")} + if "governance_history" in payload: + result["governance_sha256"] = hashlib.sha256(json.dumps(payload["governance_history"], sort_keys=True, separators=(",", ":"), allow_nan=False).encode("utf-8")).hexdigest() + return result + + +def verified_csv_source_text(payload: Mapping[str, object], *, expected_summary: Mapping[str, object] | None = None) -> str: + text = payload.get("text") + if not isinstance(text, str): + raise TabularSourceUnavailableError("Original CSV source text is unavailable.") + try: + encoded = text.encode("utf-8") + except UnicodeError as exc: + raise TabularSourceUnavailableError("Original CSV source encoding is invalid.") from exc + if len(encoded) != payload.get("byte_count") or hashlib.sha256(encoded).hexdigest() != payload.get("sha256"): + raise TabularSourceUnavailableError("Original CSV source integrity verification failed.") + if expected_summary is not None: + try: + actual = json.dumps(csv_source_summary(payload), sort_keys=True, separators=(",", ":"), allow_nan=False) + expected = json.dumps(expected_summary, sort_keys=True, separators=(",", ":"), allow_nan=False) + except (KeyError, TypeError, ValueError) as exc: + raise TabularSourceUnavailableError("Original CSV source evidence is invalid.") from exc + if actual != expected: + raise TabularSourceUnavailableError("Original CSV source no longer matches its recorded evidence.") + return text + + +def csv_projection_matches(expected: Sequence[Mapping[str, object]], actual: Sequence[Mapping[str, object]]) -> bool: + """CSV cells are scalar: booleans, integers and floats are not interchangeable.""" + return len(expected) == len(actual) and all( + left.keys() == right.keys() and all( + type(value) is type(right[name]) and value == right[name] + for name, value in left.items() + ) + for left, right in zip(expected, actual, strict=True) + ) class TabularSourceError(ValueError): @@ -39,29 +114,44 @@ class TabularSourceUnavailableError(TabularSourceError): pass +def _csv_utf8_bytes(text: str) -> bytes: + try: + return text.encode("utf-8") + except UnicodeError as exc: + raise TabularSourceValidationError("CSV input must be valid Unicode encodable as UTF-8.") from exc + + def parse_tabular_csv( csv_text: str, *, delimiter: str = ",", max_rows: int = 10_000, + max_bytes: int = 5_000_000, + value_mode: CsvValueMode = "legacy_typed", ) -> tuple[Mapping[str, object], ...]: - """Parse a bounded CSV document into JSON-compatible tabular rows.""" + """Parse CSV with an explicit lexical-text or backward-compatible typed mode.""" if len(delimiter) != 1: raise TabularSourceValidationError("CSV delimiter must be one character.") + if value_mode not in {"legacy_typed", "text"}: + raise TabularSourceValidationError("Unsupported CSV value mode.") + if len(_csv_utf8_bytes(csv_text)) > max_bytes: + raise TabularSourceValidationError(f"CSV input is limited to {max_bytes:,} UTF-8 bytes.") try: - reader = csv.DictReader(io.StringIO(csv_text), delimiter=delimiter) + reader = csv.DictReader(io.StringIO(csv_text), delimiter=delimiter, strict=value_mode == "text") original_headers, normalized_headers = _csv_headers(reader.fieldnames) rows: list[dict[str, object]] = [] for row in reader: _validate_csv_row_shape(row) - if _csv_row_is_empty(row, original_headers): + if value_mode == "text" and (None in row or any(row.get(header) is None for header in original_headers)): + raise TabularSourceValidationError("CSV text rows must have exactly the number of values defined by the header.") + if value_mode == "legacy_typed" and _csv_row_is_empty(row, original_headers): continue if len(rows) >= max_rows: raise TabularSourceValidationError( f"CSV snapshots are limited to {max_rows:,} rows." ) - rows.append(_csv_row(row, original_headers, normalized_headers)) + rows.append(_csv_row(row, original_headers, normalized_headers, value_mode=value_mode)) return tuple(rows) except csv.Error as exc: raise TabularSourceValidationError(f"CSV input could not be parsed: {exc}") from exc @@ -106,9 +196,11 @@ def _csv_row( row: Mapping[str | None, str | list[str] | None], original_headers: Sequence[str], normalized_headers: Sequence[str], + *, + value_mode: CsvValueMode = "legacy_typed", ) -> dict[str, object]: return { - normalized: _csv_scalar(value if isinstance(value, str) else None) + normalized: (value if value_mode == "text" else _csv_scalar(value if isinstance(value, str) else None)) for original, normalized in zip( original_headers, normalized_headers, @@ -128,9 +220,15 @@ def _csv_scalar(value: str | None) -> object: if lowered in {"true", "false"}: return lowered == "true" if re.fullmatch(r"-?(?:0|[1-9][0-9]*)", text): - return int(text) + try: + return int(text) + except ValueError as exc: + raise TabularSourceValidationError("CSV integer exceeds the conversion limit; use text mode to preserve it.") from exc if re.fullmatch(r"-?(?:0|[1-9][0-9]*)\.[0-9]+", text): - return float(text) + value = float(text) + if not math.isfinite(value): + raise TabularSourceValidationError("CSV numeric value exceeds the finite number range; use text mode to preserve it.") + return value return text @@ -141,6 +239,48 @@ class TabularColumn: nullable: bool = True +def tabular_type_name(value: object, *, casefold_unknown: bool = False) -> str: + if isinstance(value, bool): + return "boolean" + if isinstance(value, int): + return "integer" + if isinstance(value, (float, Decimal)): + return "number" + if isinstance(value, str): + return "string" + if isinstance(value, list): + return "array" + if isinstance(value, dict): + return "object" + name = type(value).__name__ + return name.casefold() if casefold_unknown else name.lower() + + +def infer_tabular_schema( + rows: Sequence[Mapping[str, object]], + *, + type_name: Callable[[object], str] = tabular_type_name, +) -> tuple[TabularColumn, ...]: + """Infer first-seen columns in one pass without retaining column values. + + The classifier is explicit so legacy providers can preserve their exact + type naming. Missing keys and explicit None both make a column nullable. + """ + states: dict[str, tuple[str | None, int]] = {} + for row in rows: + for name, value in row.items(): + kind, concrete = states.get(name, (None, 0)) + if value is not None: + value_kind = type_name(value) + kind = value_kind if kind is None else kind if kind == value_kind else "mixed" + concrete += 1 + states[name] = (kind, concrete) + return tuple( + TabularColumn(name=name, data_type=kind if kind is not None else "unknown", nullable=concrete != len(rows)) + for name, (kind, concrete) in states.items() + ) + + @dataclass(frozen=True, slots=True) class TabularPushdown: projections: bool = False @@ -221,6 +361,7 @@ class TabularSnapshotInput: rows: tuple[Mapping[str, object], ...] description: str | None = None metadata: Mapping[str, object] = field(default_factory=dict) + csv_source: TabularCsvSource | None = None @runtime_checkable @@ -296,6 +437,10 @@ __all__ = [ "CAPABILITY_CONNECTORS_TABULAR_SOURCES", "DEFAULT_PREVIEW_BYTES", "DEFAULT_PREVIEW_TIMEOUT_MS", + "CsvValueMode", + "TabularCsvSource", + "tabular_type_name", + "infer_tabular_schema", "TabularColumn", "TabularPreviewDiagnostic", "TabularPushdown", @@ -313,6 +458,10 @@ __all__ = [ "TabularSourceUnavailableError", "TabularSourceValidationError", "parse_tabular_csv", + "csv_source_payload", + "csv_source_summary", + "verified_csv_source_text", + "csv_projection_matches", "tabular_snapshot_writer", "tabular_source_provider", ] diff --git a/src/govoplan_core/db/json_predicates.py b/src/govoplan_core/db/json_predicates.py new file mode 100755 index 0000000..7413366 --- /dev/null +++ b/src/govoplan_core/db/json_predicates.py @@ -0,0 +1,147 @@ +"""Exact, bound-parameter JSON permission predicates for SQLite/PostgreSQL. + +These primitives only match strings (never coerced numbers/booleans) and +objects inside actual arrays. Owners still define tenant, subject, permission, +purpose and current-state policy. Unsupported dialects fail at compilation. +""" +from __future__ import annotations + +import re +from collections.abc import Mapping + +from sqlalchemy import Boolean, literal +from sqlalchemy.exc import CompileError +from sqlalchemy.ext.compiler import compiles +from sqlalchemy.sql.functions import FunctionElement + + +class _ArrayString(FunctionElement): + type = Boolean() + inherit_cache = True + + +class _ObjectStrings(FunctionElement): + type = Boolean() + inherit_cache = True + + +class _ArrayObjectStrings(FunctionElement): + type = Boolean() + inherit_cache = True + + +def json_array_contains_string(column, value: str): + if type(value) is not str: + raise TypeError("JSON string membership requires a string value.") + return _ArrayString(column, literal(value)) + + +def _field_arguments(fields: Mapping[str, str]): + if not isinstance(fields, Mapping) or not 1 <= len(fields) <= 16: + raise ValueError("JSON object matching requires between 1 and 16 string fields.") + arguments = [] + for key, value in sorted(fields.items()): + if type(key) is not str or re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]{0,127}", key) is None: + raise ValueError("JSON object field names must be simple identifiers.") + if type(value) is not str: + raise TypeError("JSON object matching requires string values.") + arguments.extend((literal(key), literal(value))) + return arguments + + +def json_object_matches_strings(column, fields: Mapping[str, str]): + return _ObjectStrings(column, *_field_arguments(fields)) + + +def json_array_contains_object_strings(column, fields: Mapping[str, str]): + return _ArrayObjectStrings(column, *_field_arguments(fields)) + + +@compiles(_ArrayString) +@compiles(_ObjectStrings) +@compiles(_ArrayObjectStrings) +def _unsupported(element, compiler, **kwargs): + raise CompileError("Exact JSON permission predicates support only SQLite and PostgreSQL.") + + +def _parts(element, compiler, kwargs): + return [compiler.process(item, **kwargs) for item in element.clauses] + + +def _sqlite_array(value): + return f"CASE WHEN json_type({value}) = 'array' THEN {value} ELSE '[]' END" + + +def _postgres_array(value): + value = f"CAST({value} AS JSON)" + return f"CASE WHEN json_typeof({value}) = 'array' THEN {value} ELSE '[]'::json END" + + +def _sqlite_fields(value, fields): + terms = [] + for index in range(0, len(fields), 2): + key, expected = fields[index:index + 2] + path = f"('$.' || {key})" + terms.extend((f"json_type({value}, {path}) = 'text'", f"json_extract({value}, {path}) = {expected}")) + return " AND ".join(terms) + + +def _postgres_fields(value, fields): + terms = [] + for index in range(0, len(fields), 2): + key, expected = fields[index:index + 2] + terms.extend((f"json_typeof(({value}) -> {key}) = 'string'", f"(({value}) ->> {key}) = {expected}")) + return " AND ".join(terms) + + +@compiles(_ArrayString, "sqlite") +def _array_string_sqlite(element, compiler, **kwargs): + value, expected = _parts(element, compiler, kwargs) + return ( + f"EXISTS (SELECT 1 FROM json_each({_sqlite_array(value)}) AS gp_json_string " + f"WHERE gp_json_string.type = 'text' AND gp_json_string.value = {expected})" + ) + + +@compiles(_ArrayString, "postgresql") +def _array_string_postgres(element, compiler, **kwargs): + value, expected = _parts(element, compiler, kwargs) + return ( + f"EXISTS (SELECT 1 FROM json_array_elements({_postgres_array(value)}) AS gp_json_string(value) " + f"WHERE json_typeof(gp_json_string.value) = 'string' " + f"AND (gp_json_string.value #>> '{{}}') = {expected})" + ) + + +@compiles(_ObjectStrings, "sqlite") +def _object_sqlite(element, compiler, **kwargs): + value, *fields = _parts(element, compiler, kwargs) + value = f"CASE WHEN json_type({value}) = 'object' THEN {value} ELSE '{{}}' END" + return f"({_sqlite_fields(value, fields)})" + + +@compiles(_ObjectStrings, "postgresql") +def _object_postgres(element, compiler, **kwargs): + value, *fields = _parts(element, compiler, kwargs) + value = f"CAST({value} AS JSON)" + return f"(json_typeof({value}) = 'object' AND {_postgres_fields(value, fields)})" + + +@compiles(_ArrayObjectStrings, "sqlite") +def _array_object_sqlite(element, compiler, **kwargs): + value, *fields = _parts(element, compiler, kwargs) + item = "CASE WHEN gp_json_object.type = 'object' THEN gp_json_object.value ELSE '{}' END" + return ( + f"EXISTS (SELECT 1 FROM json_each({_sqlite_array(value)}) AS gp_json_object " + f"WHERE {_sqlite_fields(item, fields)})" + ) + + +@compiles(_ArrayObjectStrings, "postgresql") +def _array_object_postgres(element, compiler, **kwargs): + value, *fields = _parts(element, compiler, kwargs) + return ( + f"EXISTS (SELECT 1 FROM json_array_elements({_postgres_array(value)}) AS gp_json_object(value) " + f"WHERE json_typeof(gp_json_object.value) = 'object' " + f"AND {_postgres_fields('gp_json_object.value', fields)})" + ) diff --git a/src/govoplan_core/server/conditional_requests.py b/src/govoplan_core/server/conditional_requests.py index 82dec02..d7d78f8 100644 --- a/src/govoplan_core/server/conditional_requests.py +++ b/src/govoplan_core/server/conditional_requests.py @@ -1,12 +1,15 @@ from __future__ import annotations import hashlib -from collections.abc import Awaitable, Callable +from collections.abc import AsyncIterator, Awaitable, Callable from fastapi import Request from starlette.responses import Response JSON_CACHE_CONTROL = "private, no-cache" +# This bounds middleware-owned buffering, not the size of a route response. +# Larger responses retain their streaming iterator and are never truncated. +MAX_CONDITIONAL_JSON_BYTES = 1_048_576 JSON_ETAG_VARY_HEADERS = ( "Authorization", "Cookie", @@ -26,15 +29,34 @@ async def conditional_json_get_middleware( The middleware deliberately works after route handling. That keeps the contract platform-wide without requiring every module router to learn about - conditional requests, while still limiting buffering to successful JSON GET - responses. + conditional requests. Only small successful JSON responses are buffered; + larger responses stream unchanged. Authorization still runs on every GET. """ response = await call_next(request) if not _eligible_for_conditional_json_get(request, response): return response - body = b"".join([chunk async for chunk in response.body_iterator]) + response.headers["cache-control"] = _conditional_cache_control(response.headers.get("cache-control")) + response.headers["vary"] = _merge_vary(response.headers.get("vary"), JSON_ETAG_VARY_HEADERS) + content_length = response.headers.get("content-length", "") + if content_length.isascii() and content_length.isdecimal() and int(content_length) > MAX_CONDITIONAL_JSON_BYTES: + return response + + chunks: list[bytes] = [] + size = 0 + iterator = response.body_iterator + async for chunk in iterator: + if not chunk: + continue + chunks.append(chunk) + size += len(chunk) + if size > MAX_CONDITIONAL_JSON_BYTES: + # Include the crossing chunk exactly once, without draining the + # rest of the producer or copying a potentially large chunk. + response.body_iterator = _replay_prefix(chunks, iterator) + return response + body = b"".join(chunks) etag = response.headers.get("etag") or json_response_etag(body) headers = dict(response.headers) headers["etag"] = etag @@ -48,6 +70,14 @@ async def conditional_json_get_middleware( return Response(content=body, status_code=response.status_code, headers=headers, background=response.background) +async def _replay_prefix(chunks: list[bytes], iterator: AsyncIterator[bytes]) -> AsyncIterator[bytes]: + for chunk in chunks: + yield chunk + chunks.clear() + async for chunk in iterator: + yield chunk + + def json_response_etag(body: bytes) -> str: digest = hashlib.sha256(body).hexdigest() return f'W/"sha256-{digest}"' diff --git a/tests/test_conditional_requests.py b/tests/test_conditional_requests.py index 22a429c..7666c85 100644 --- a/tests/test_conditional_requests.py +++ b/tests/test_conditional_requests.py @@ -1,15 +1,85 @@ from __future__ import annotations import unittest +from unittest.mock import patch -from fastapi import APIRouter, Response +from fastapi import APIRouter, Request, Response from fastapi.responses import PlainTextResponse from fastapi.testclient import TestClient +from starlette.background import BackgroundTask +from starlette.responses import StreamingResponse from govoplan_core.auth import get_api_principal from govoplan_core.core.registry import PlatformRegistry from govoplan_core.server.fastapi import create_govoplan_app from govoplan_core.server.platform import create_platform_router +from govoplan_core.server.conditional_requests import conditional_json_get_middleware + + +class ConditionalBufferTests(unittest.IsolatedAsyncioTestCase): + async def test_unknown_length_overflow_replays_exact_bytes_without_eager_drain(self) -> None: + chunks = [b'{"value":"', b'', b'a' * 32, b'b' * 32, b'c' * 32, b'"}'] + consumed = [] + + async def body(): + for chunk in chunks: + consumed.append(chunk) + yield chunk + + background = BackgroundTask(lambda: None) + response = StreamingResponse(body(), media_type="application/json", background=background) + + async def route(request): + return response + + request = Request({"type": "http", "method": "GET", "headers": [(b'if-none-match', b'*')]}) + with patch("govoplan_core.server.conditional_requests.MAX_CONDITIONAL_JSON_BYTES", 64, create=True): + result = await conditional_json_get_middleware(request, route) + self.assertIs(result, response) + self.assertEqual(4, len(consumed)) + self.assertEqual(200, result.status_code) + self.assertNotIn("etag", result.headers) + self.assertIn("private", result.headers["cache-control"]) + self.assertIn("Authorization", result.headers["vary"]) + self.assertIs(background, result.background) + self.assertEqual(b"".join(chunks), b"".join([chunk async for chunk in result.body_iterator])) + self.assertEqual(chunks, consumed) + + async def test_known_large_body_is_not_consumed(self) -> None: + consumed = [] + + async def body(): + consumed.append(True) + yield b"x" * 65 + + response = StreamingResponse(body(), media_type="application/json", headers={"Content-Length": "65"}) + + async def route(request): + return response + + request = Request({"type": "http", "method": "GET", "headers": []}) + with patch("govoplan_core.server.conditional_requests.MAX_CONDITIONAL_JSON_BYTES", 64, create=True): + result = await conditional_json_get_middleware(request, route) + self.assertIs(result, response) + self.assertEqual([], consumed) + self.assertEqual("65", result.headers["content-length"]) + self.assertEqual(b"x" * 65, b"".join([chunk async for chunk in result.body_iterator])) + + async def test_matching_small_response_still_runs_current_route_authorization(self) -> None: + calls = [] + + async def route(request): + calls.append(True) + if len(calls) > 1: + return Response(status_code=403) + return StreamingResponse(iter([b'{"ok":true}']), media_type="application/json") + + request = Request({"type": "http", "method": "GET", "headers": []}) + first = await conditional_json_get_middleware(request, route) + conditional = Request({"type": "http", "method": "GET", "headers": [(b'if-none-match', first.headers['etag'].encode())]}) + second = await conditional_json_get_middleware(conditional, route) + self.assertEqual(403, second.status_code) + self.assertEqual(2, len(calls)) class ConditionalRequestTests(unittest.TestCase): diff --git a/tests/test_json_predicates.py b/tests/test_json_predicates.py new file mode 100755 index 0000000..4cef601 --- /dev/null +++ b/tests/test_json_predicates.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +import unittest + +from sqlalchemy import JSON, Column, Integer, MetaData, Table, create_engine, select +from sqlalchemy.dialects import mysql, postgresql +from sqlalchemy.exc import CompileError + +from govoplan_core.db.json_predicates import ( + json_array_contains_object_strings, + json_array_contains_string, + json_object_matches_strings, +) + + +class JsonPredicateTests(unittest.TestCase): + def setUp(self): + self.engine = create_engine("sqlite:///:memory:") + self.metadata = MetaData() + self.records = Table("records", self.metadata, Column("id", Integer, primary_key=True), Column("value", JSON)) + self.metadata.create_all(self.engine) + + def tearDown(self): + self.engine.dispose() + + def matched(self, values, predicate): + with self.engine.begin() as connection: + connection.execute(self.records.insert(), [{"id": index, "value": value} for index, value in enumerate(values)]) + return list(connection.scalars(select(self.records.c.id).where(predicate).order_by(self.records.c.id))) + + def test_string_membership_is_array_and_type_exact(self): + self.assertEqual([0], self.matched( + [["1"], [1], [True], [None], {"key": "1"}, "1", None, ["11"]], + json_array_contains_string(self.records.c.value, "1"), + )) + + def test_object_fields_are_exact_strings_not_coerced_or_substrings(self): + self.assertEqual([0], self.matched( + [{"kind": "account", "id": "1", "label": "Extra field allowed"}, + {"kind": "account", "id": 1}, {"kind": "account", "id": "11"}, + "not-json", None, ["account", "1"]], + json_object_matches_strings(self.records.c.value, {"kind": "account", "id": "1"}), + )) + + def test_array_objects_do_not_match_encoded_objects_or_scalar_elements(self): + self.assertEqual([0], self.matched( + [[{"kind": "account", "id": "1"}], ['{"kind":"account","id":"1"}'], + [{"kind": "account", "id": 1}], ["not-json", None, True, 1], + {"kind": "account", "id": "1"}, None], + json_array_contains_object_strings(self.records.c.value, {"kind": "account", "id": "1"}), + )) + + def test_values_remain_bound_for_both_dialects(self): + value = "x' OR 1=1 --" + predicates = [ + json_array_contains_string(self.records.c.value, value), + json_object_matches_strings(self.records.c.value, {"id": value}), + json_array_contains_object_strings(self.records.c.value, {"id": value}), + ] + for predicate in predicates: + for dialect in (self.engine.dialect, postgresql.dialect()): + with self.subTest(predicate=type(predicate).__name__, dialect=dialect.name): + compiled = select(self.records.c.id).where(predicate).compile(dialect=dialect) + self.assertNotIn(value, str(compiled)) + self.assertIn(value, compiled.params.values()) + with self.assertRaises(CompileError): + select(self.records.c.id).where(predicate).compile(dialect=mysql.dialect()) + + def test_invalid_field_names_and_non_string_matches_are_rejected(self): + for fields in ({"id": 1}, {"not.a.field": "1"}, {}): + with self.subTest(fields=fields), self.assertRaises((TypeError, ValueError)): + json_array_contains_object_strings(self.records.c.value, fields) + with self.assertRaises(TypeError): + json_array_contains_string(self.records.c.value, True) diff --git a/tests/test_migration_url_config.py b/tests/test_migration_url_config.py new file mode 100644 index 0000000..1870701 --- /dev/null +++ b/tests/test_migration_url_config.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +from pathlib import Path +import runpy +from types import SimpleNamespace +import unittest +from unittest.mock import MagicMock, patch + +from alembic import context +from alembic.config import Config + + +class MigrationUrlConfigurationTests(unittest.TestCase): + def test_database_urls_round_trip_exactly_in_online_and_offline_modes(self) -> None: + urls = ( + "postgresql+psycopg://localhost/example?host=%2Ftmp%2Fexample", + "postgresql+psycopg://synthetic%40user:synthetic%25%40pass@localhost/example", + "sqlite:////tmp/synthetic%25-database.db", + "sqlite:////tmp/synthetic%%-database.db", + "sqlite:////tmp/%(here)s-literal.db", + "sqlite:////tmp/synthetic-database.db", + ) + environment = Path(__file__).resolve().parents[1] / "alembic" / "env.py" + for url in urls: + for offline in (False, True): + for from_settings in (False, True): + with self.subTest(url=url, offline=offline, from_settings=from_settings): + config = Config() + config.attributes.update(enabled_modules=(), manifest_factories=()) + if not from_settings: + config.attributes["database_url"] = url + engine = MagicMock() + connection = engine.connect.return_value.__enter__.return_value + with ( + patch.object(context, "config", config, create=True), + patch.object(context, "is_offline_mode", return_value=offline), + patch.object(context, "configure") as configure, + patch.object(context, "begin_transaction"), + patch.object(context, "run_migrations") as run_migrations, + patch("sqlalchemy.engine_from_config", return_value=engine) as engine_from_config, + patch( + "govoplan_core.server.default_config.get_server_config", + return_value=SimpleNamespace(enabled_modules=(), manifest_factories=()), + ), + patch("govoplan_core.server.registry.build_platform_registry"), + patch( + "govoplan_core.core.migrations.migration_metadata_plan", + return_value=SimpleNamespace(metadata=()), + ), + patch("govoplan_core.settings.settings.database_url", url), + ): + runpy.run_path(str(environment)) + + self.assertEqual(config.get_main_option("sqlalchemy.url"), url) + self.assertEqual(config.get_section(config.config_ini_section)["sqlalchemy.url"], url) + run_migrations.assert_called_once_with() + if offline: + engine_from_config.assert_not_called() + self.assertEqual(configure.call_args.kwargs["url"], url) + else: + engine_from_config.assert_called_once() + self.assertEqual(engine_from_config.call_args.args[0]["sqlalchemy.url"], url) + self.assertIs(configure.call_args.kwargs["connection"], connection) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_optimistic_concurrency.py b/tests/test_optimistic_concurrency.py index 4f1b8c3..ff904f6 100644 --- a/tests/test_optimistic_concurrency.py +++ b/tests/test_optimistic_concurrency.py @@ -27,6 +27,25 @@ class _RevisionFixture(Base): class OptimisticConcurrencyTests(unittest.TestCase): + def test_keyed_insertions_keep_their_anchors_during_disjoint_edits(self) -> None: + base = [{"id": "a", "value": 1}, {"id": "b", "value": 2}] + local = [base[0], {"id": "x", "value": 3}, base[1]] + current = [base[0], {"id": "b", "value": 20}] + result = three_way_merge(base, local, current) + self.assertTrue(result.merged) + self.assertEqual(["a", "x", "b"], [item["id"] for item in result.value]) + self.assertEqual(20, result.value[-1]["value"]) + self.assertEqual(["a", "b"], [item["id"] for item in base]) + + def test_concurrent_insertions_are_stable_and_incompatible_anchors_conflict(self) -> None: + base = [{"id": "a"}, {"id": "b"}] + result = three_way_merge(base, [base[0], {"id": "x"}, base[1]], [base[0], {"id": "y"}, base[1]]) + self.assertTrue(result.merged) + self.assertEqual(["a", "y", "x", "b"], [item["id"] for item in result.value]) + conflict = three_way_merge(base, [base[0], {"id": "x"}, base[1]], [base[1], base[0]]) + self.assertFalse(conflict.merged) + self.assertEqual("collection_reorder", conflict.conflicts[0].kind) + def test_strong_etags_and_if_match_use_strong_comparison(self) -> None: etag = strong_resource_etag("campaign_version", "version-1", 3) diff --git a/tests/test_principal_helpers.py b/tests/test_principal_helpers.py new file mode 100755 index 0000000..5ffad55 --- /dev/null +++ b/tests/test_principal_helpers.py @@ -0,0 +1,43 @@ +from types import SimpleNamespace +import unittest + +from govoplan_core.core.principal_helpers import principal_actor_ids, principal_user_first_actor + + +class PrincipalHelperTests(unittest.TestCase): + def test_contracts_keep_their_distinct_precedence_and_whitespace(self) -> None: + principal = SimpleNamespace(account_id=" account ", identity_id=" identity ", membership_id=" membership ", user=SimpleNamespace(id=" user ")) + self.assertEqual((" account ", " identity ", " membership ", " user "), principal_actor_ids(principal)) + self.assertEqual("user", principal_user_first_actor(principal)) + self.assertEqual(" account ", principal.account_id) + self.assertEqual(" user ", principal.user.id) + + def test_duplicate_ids_are_removed_in_stable_order_without_normalizing(self) -> None: + principal = SimpleNamespace(account_id="same", identity_id=" same ", membership_id="same", user=SimpleNamespace(id="same")) + self.assertEqual(("same", " same "), principal_actor_ids(principal)) + self.assertEqual("same", principal_user_first_actor(principal)) + + def test_missing_blank_and_service_only_principals_remain_unattributed(self) -> None: + for principal in ( + object(), None, SimpleNamespace(), SimpleNamespace(service_account_id="service"), + SimpleNamespace(account_id=0, identity_id=False, membership_id="\t", user=SimpleNamespace(id="\u00a0")), + ): + with self.subTest(principal=principal): + self.assertEqual((), principal_actor_ids(principal)) + self.assertIsNone(principal_user_first_actor(principal)) + + def test_fallback_and_existing_string_coercion_are_unchanged(self) -> None: + for values, expected_ids, expected_actor in ( + ({"account_id": 17, "identity_id": "identity"}, ("17", "identity"), "17"), + ({"identity_id": " identity ", "membership_id": "member"}, (" identity ", "member"), "identity"), + ({"membership_id": " member "}, (" member ",), "member"), + ({"user": SimpleNamespace(id=" user ")}, (" user ",), "user"), + ): + with self.subTest(values=values): + principal = SimpleNamespace(**values) + self.assertEqual(expected_ids, principal_actor_ids(principal)) + self.assertEqual(expected_actor, principal_user_first_actor(principal)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_review_mechanics.py b/tests/test_review_mechanics.py new file mode 100755 index 0000000..423870c --- /dev/null +++ b/tests/test_review_mechanics.py @@ -0,0 +1,144 @@ +from __future__ import annotations + +import random +import unittest +from decimal import Decimal + +from govoplan_core.core.modules import DocumentationTopic, localize_documentation_topics +from govoplan_core.core.tabular_sources import ( + TabularColumn, + TabularCsvSource, + TabularSourceUnavailableError, + TabularSourceValidationError, + csv_source_payload, + csv_source_summary, + verified_csv_source_text, + csv_projection_matches, + infer_tabular_schema, + parse_tabular_csv, + tabular_type_name, +) + + +class SharedReviewMechanicsTests(unittest.TestCase): + def test_invalid_csv_unicode_fails_explicitly_without_echoing_content(self) -> None: + source = TabularCsvSource(text="value\nprivate-\ud800\n") + for action in ( + lambda: csv_source_payload(source), + lambda: parse_tabular_csv(source.text), + ): + with ( + self.subTest(action=action), + self.assertRaises(TabularSourceValidationError) as raised, + ): + action() + self.assertNotIn("private", str(raised.exception)) + with self.assertRaises(TabularSourceUnavailableError): + verified_csv_source_text({"text": source.text}) + + def test_csv_huge_integer_has_explicit_validation_and_lossless_text_option( + self, + ) -> None: + import sys + + maximum = sys.get_int_max_str_digits() + if not maximum: + self.skipTest("Interpreter integer conversion limit is disabled") + text = "9" * (maximum + 1) + with self.assertRaisesRegex(TabularSourceValidationError, "use text mode"): + parse_tabular_csv("value\n" + text + "\n") + self.assertEqual( + ({"value": text},), + parse_tabular_csv("value\n" + text + "\n", value_mode="text"), + ) + + def test_csv_projection_binding_distinguishes_boolean_integer_and_float( + self, + ) -> None: + for expected in (True, 1, 1.0): + for actual in (True, 1, 1.0): + with self.subTest( + expected_type=type(expected), actual_type=type(actual) + ): + self.assertEqual( + type(expected) is type(actual), + csv_projection_matches( + ({"value": expected},), ({"value": actual},) + ), + ) + self.assertFalse(csv_projection_matches(({"value": 1},), ({"other": 1},))) + + def test_translation_merge_preserves_owner_data_and_untranslated_identity( + self, + ) -> None: + translations = {"de": {"summary": "vorhanden"}, "fr": {"title": "Français"}} + first = DocumentationTopic( + id="one", title="One", summary="First", translations=translations + ) + unchanged = DocumentationTopic(id="two", title="Two", summary="Second") + localized = localize_documentation_topics( + iter((first, unchanged)), + locale="de", + translations={"one": {"title": "Eins"}}, + ) + self.assertEqual( + {"title": "Eins", "summary": "vorhanden"}, localized[0].translations["de"] + ) + self.assertEqual({"title": "Français"}, localized[0].translations["fr"]) + self.assertIs(unchanged, localized[1]) + self.assertEqual({"summary": "vorhanden"}, first.translations["de"]) + self.assertIsNot(translations["fr"], localized[0].translations["fr"]) + + def test_schema_inference_matches_legacy_projection_for_sparse_mixed_rows( + self, + ) -> None: + generator = random.Random(298) + values = [None, True, False, 1, 1.5, Decimal("1.00"), "001", [], {}] + rows = [ + { + f"field_{column}": generator.choice(values) + for column in generator.sample(range(80), 20) + } + for _ in range(100) + ] + names = list(dict.fromkeys(name for row in rows for name in row)) + expected = [] + for name in names: + concrete = [row[name] for row in rows if row.get(name) is not None] + types = {tabular_type_name(value) for value in concrete} + kind = ( + "unknown" + if not types + else next(iter(types)) + if len(types) == 1 + else "mixed" + ) + expected.append( + TabularColumn( + name=name, data_type=kind, nullable=len(concrete) != len(rows) + ) + ) + self.assertEqual(tuple(expected), infer_tabular_schema(rows)) + self.assertEqual( + (TabularColumn("only_null", "unknown", True),), + infer_tabular_schema([{"only_null": None}]), + ) + self.assertEqual((), infer_tabular_schema([])) + unknown = type("ẞ", (), {})() + self.assertEqual("ß", tabular_type_name(unknown)) + self.assertEqual("ss", tabular_type_name(unknown, casefold_unknown=True)) + + def test_csv_evidence_keeps_exact_text_but_summary_never_contains_content( + self, + ) -> None: + source = TabularCsvSource( + text='\ufeffvalue\r\n" text "\r\n', value_mode="text" + ) + payload = csv_source_payload(source) + self.assertEqual(source.text, verified_csv_source_text(payload)) + self.assertNotIn("text", csv_source_summary(payload)) + self.assertEqual(len(source.text.encode("utf-8")), payload["byte_count"]) + with self.assertRaises(TabularSourceUnavailableError): + verified_csv_source_text( + {**payload, "text": source.text.replace("text", "edited")} + ) diff --git a/tests/test_tabular_source_contract.py b/tests/test_tabular_source_contract.py index 7b58121..547099e 100644 --- a/tests/test_tabular_source_contract.py +++ b/tests/test_tabular_source_contract.py @@ -178,6 +178,29 @@ class TabularSourceContractTests(unittest.TestCase): with self.assertRaises(TabularSourceValidationError): parse_tabular_csv("id,name\n1,Ada,extra\n") + def test_text_csv_mode_preserves_lexical_values_and_explicit_empty_records(self) -> None: + source = 'id,value\r\n9007199254740993," keep me "\r\ntrue,0.123456789012345678901234567890\r\n" ",""\r\n' + self.assertEqual( + ( + {"id": "9007199254740993", "value": " keep me "}, + {"id": "true", "value": "0.123456789012345678901234567890"}, + {"id": " ", "value": ""}, + ), + parse_tabular_csv(source, value_mode="text"), + ) + self.assertEqual(({"value": " "},), parse_tabular_csv('value\n" "\n', value_mode="text")) + + def test_text_csv_mode_rejects_shape_loss_and_applies_input_and_row_bounds(self) -> None: + for source in ('id,name\n1\n', 'id,name\n1,Ada,\n'): + with self.subTest(source=source), self.assertRaises(TabularSourceValidationError): + parse_tabular_csv(source, value_mode="text") + with self.assertRaises(TabularSourceValidationError): + parse_tabular_csv('value\n""\n""\n', value_mode="text", max_rows=1) + with self.assertRaises(TabularSourceValidationError): + parse_tabular_csv('value\nä\n', value_mode="text", max_bytes=8) + with self.assertRaises(TabularSourceValidationError): + parse_tabular_csv('value\nx\n', value_mode="unknown") + if __name__ == "__main__": unittest.main() diff --git a/webui/package-lock.json b/webui/package-lock.json index 2b4a805..0d2c61c 100644 --- a/webui/package-lock.json +++ b/webui/package-lock.json @@ -1,12 +1,12 @@ { "name": "@govoplan/core-webui", - "version": "0.1.45", + "version": "0.1.46", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@govoplan/core-webui", - "version": "0.1.45", + "version": "0.1.46", "dependencies": { "@govoplan/access-webui": "file:../../govoplan-access/webui", "@govoplan/addresses-webui": "file:../../govoplan-addresses/webui", diff --git a/webui/package-lock.release.json b/webui/package-lock.release.json index c9748d7..1f1a5ad 100644 --- a/webui/package-lock.release.json +++ b/webui/package-lock.release.json @@ -1,37 +1,37 @@ { "name": "@govoplan/core-webui", - "version": "0.1.45", + "version": "0.1.46", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@govoplan/core-webui", - "version": "0.1.45", + "version": "0.1.46", "dependencies": { "@govoplan/access-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-access.git#v0.1.25", - "@govoplan/addresses-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-addresses.git#v0.1.22", + "@govoplan/addresses-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-addresses.git#v0.1.23", "@govoplan/admin-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-admin.git#v0.1.23", "@govoplan/approvals-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-approvals.git#v0.1.21", "@govoplan/audit-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-audit.git#v0.1.20", - "@govoplan/calendar-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-calendar.git#v0.1.23", - "@govoplan/campaign-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-campaign.git#v0.1.28", - "@govoplan/cases-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-cases.git#v0.1.24", - "@govoplan/committee-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-committee.git#v0.1.21", - "@govoplan/connectors-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-connectors.git#v0.1.26", + "@govoplan/calendar-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-calendar.git#v0.1.24", + "@govoplan/campaign-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-campaign.git#v0.1.29", + "@govoplan/cases-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-cases.git#v0.1.25", + "@govoplan/committee-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-committee.git#v0.1.22", + "@govoplan/connectors-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-connectors.git#v0.1.27", "@govoplan/dashboard-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-dashboard.git#v0.1.20", - "@govoplan/dataflow-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-dataflow.git#v0.1.24", - "@govoplan/datasources-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-datasources.git#v0.1.25", + "@govoplan/dataflow-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-dataflow.git#v0.1.25", + "@govoplan/datasources-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-datasources.git#v0.1.26", "@govoplan/dist-lists-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-dist-lists.git#v0.1.21", "@govoplan/docs-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-docs.git#v0.1.23", "@govoplan/encryption-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-encryption.git#v0.1.20", - "@govoplan/files-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-files.git#v0.1.26", - "@govoplan/forms-runtime-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-forms-runtime.git#v0.1.21", + "@govoplan/files-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-files.git#v0.1.27", + "@govoplan/forms-runtime-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-forms-runtime.git#v0.1.22", "@govoplan/forms-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-forms.git#v0.1.23", "@govoplan/helpdesk-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-helpdesk.git#v0.1.21", "@govoplan/identity-trust-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-identity-trust.git#v0.1.21", "@govoplan/identity-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-identity.git#v0.1.21", - "@govoplan/idm-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-idm.git#v0.1.25", - "@govoplan/mail-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-mail.git#v0.1.27", + "@govoplan/idm-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-idm.git#v0.1.26", + "@govoplan/mail-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-mail.git#v0.1.28", "@govoplan/notifications-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-notifications.git#v0.1.20", "@govoplan/ops-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-ops.git#v0.1.22", "@govoplan/organizations-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-organizations.git#v0.1.21", @@ -42,14 +42,14 @@ "@govoplan/projects-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-projects.git#v0.1.20", "@govoplan/quick-access-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-quick-access.git#v0.1.21", "@govoplan/records-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-records.git#v0.1.24", - "@govoplan/reporting-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-reporting.git#v0.1.21", + "@govoplan/reporting-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-reporting.git#v0.1.22", "@govoplan/risk-compliance-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-risk-compliance.git#v0.1.21", "@govoplan/scheduling-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-scheduling.git#v0.1.22", "@govoplan/search-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-search.git#v0.1.20", "@govoplan/tasks-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-tasks.git#v0.1.23", "@govoplan/templates-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-templates.git#v0.1.22", "@govoplan/tenancy-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-tenancy.git#v0.1.22", - "@govoplan/tickets-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-tickets.git#v0.1.22", + "@govoplan/tickets-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-tickets.git#v0.1.23", "@govoplan/views-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-views.git#v0.1.22", "@govoplan/voting-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-voting.git#v0.1.21", "@govoplan/wiki-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-wiki.git#v0.1.22", @@ -807,8 +807,8 @@ } }, "node_modules/@govoplan/addresses-webui": { - "version": "0.1.22", - "resolved": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-addresses.git#53490e7be780e0b378b5aff3c929e5de59b5e44a", + "version": "0.1.23", + "resolved": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-addresses.git#d4fa024034e52924e9573b13a46dd0cee2287d9e", "peerDependencies": { "@govoplan/core-webui": "^0.1.45", "lucide-react": "^1.23.0", @@ -870,8 +870,8 @@ } }, "node_modules/@govoplan/calendar-webui": { - "version": "0.1.23", - "resolved": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-calendar.git#3e5fc05ca3728464131067d1fa1bc25befeae939", + "version": "0.1.24", + "resolved": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-calendar.git#8e36f8b3d203fc0b08ea0487540fb7ac597cb8c9", "peerDependencies": { "@govoplan/core-webui": "^0.1.44", "@vitejs/plugin-react": "^5.2.0", @@ -889,8 +889,8 @@ } }, "node_modules/@govoplan/campaign-webui": { - "version": "0.1.28", - "resolved": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-campaign.git#c51fc180fba75441dbd19c25b02a3ddc37c81a5d", + "version": "0.1.29", + "resolved": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-campaign.git#19437ce378553f6ab559e3c0cba44150e495ef52", "dependencies": { "read-excel-file": "9.2.0" }, @@ -922,8 +922,8 @@ } }, "node_modules/@govoplan/cases-webui": { - "version": "0.1.24", - "resolved": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-cases.git#afda3a5ac5f05898dfbc4daa91a74fd50d59f9c1", + "version": "0.1.25", + "resolved": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-cases.git#0337e0cf0b37a9b89dd51cccf28500649361fb37", "peerDependencies": { "@govoplan/core-webui": "^0.1.30", "lucide-react": "^1.23.0", @@ -938,8 +938,8 @@ } }, "node_modules/@govoplan/committee-webui": { - "version": "0.1.21", - "resolved": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-committee.git#7c2ec4ec17e650699164d99a656ba5e48068fdf7", + "version": "0.1.22", + "resolved": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-committee.git#96c48a7f2d3a8fa36f44d5806a14003e40cc4807", "peerDependencies": { "@govoplan/core-webui": "^0.1.18", "lucide-react": "^1.23.0", @@ -954,8 +954,8 @@ } }, "node_modules/@govoplan/connectors-webui": { - "version": "0.1.26", - "resolved": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-connectors.git#33de5cac5584ad41319a447f12d1eb5dbb11f1d7", + "version": "0.1.27", + "resolved": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-connectors.git#889cafaf207a2e7e495c64ccbac07d9faf5a5dce", "peerDependencies": { "@govoplan/core-webui": "^0.1.18", "react": ">=19.2.7 <20", @@ -984,10 +984,10 @@ } }, "node_modules/@govoplan/dataflow-webui": { - "version": "0.1.24", - "resolved": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-dataflow.git#618f10fe89223b8acbda080047f3c2ed39dae67a", + "version": "0.1.25", + "resolved": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-dataflow.git#a6c5bab3a4b7a52cb0f9b07911216ce61f362bbe", "peerDependencies": { - "@govoplan/core-webui": "^0.1.45", + "@govoplan/core-webui": "^0.1.46", "@xyflow/react": "^12.11.2", "lucide-react": "^1.23.0", "react": ">=19.2.7 <20", @@ -1002,10 +1002,10 @@ } }, "node_modules/@govoplan/datasources-webui": { - "version": "0.1.25", - "resolved": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-datasources.git#8b8c6c548ecd2801533b9c7d5003f13f3ca41f11", + "version": "0.1.26", + "resolved": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-datasources.git#9d067f1baded6e65a221fbba0b46fdd953404b51", "peerDependencies": { - "@govoplan/core-webui": "^0.1.45", + "@govoplan/core-webui": "^0.1.46", "lucide-react": "^1.23.0", "react": ">=19.2.7 <20", "react-dom": ">=19.2.7 <20", @@ -1070,8 +1070,8 @@ } }, "node_modules/@govoplan/files-webui": { - "version": "0.1.26", - "resolved": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-files.git#ff84812f7f15bab4385fdcbed8605a7cc1de8195", + "version": "0.1.27", + "resolved": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-files.git#61625fb00fedbd4474aab1854ba9cedaf5fd18f3", "peerDependencies": { "@govoplan/core-webui": "^0.1.45", "@vitejs/plugin-react": "^5.2.0", @@ -1089,8 +1089,8 @@ } }, "node_modules/@govoplan/forms-runtime-webui": { - "version": "0.1.21", - "resolved": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-forms-runtime.git#cd78800a6561a5cc1d04dc9430bd03a9466ea6db", + "version": "0.1.22", + "resolved": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-forms-runtime.git#1236b861b8128b9fa1ce30c51a32856554d8c3df", "peerDependencies": { "@govoplan/core-webui": "^0.1.18", "lucide-react": "^1.23.0", @@ -1167,8 +1167,8 @@ } }, "node_modules/@govoplan/idm-webui": { - "version": "0.1.25", - "resolved": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-idm.git#46e09d0c68866eaf7cf46ccfe773854eac0d0b0b", + "version": "0.1.26", + "resolved": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-idm.git#ba04593e29140298687a29a6bdf46d1376974c02", "peerDependencies": { "@govoplan/core-webui": "^0.1.45", "@vitejs/plugin-react": "^5.2.0", @@ -1186,8 +1186,8 @@ } }, "node_modules/@govoplan/mail-webui": { - "version": "0.1.27", - "resolved": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-mail.git#480c18c67cc345b462ec9dd0dde252fbfc13f571", + "version": "0.1.28", + "resolved": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-mail.git#d9262982147ff70b4edbc8990d0bf8ccb5f190a8", "peerDependencies": { "@govoplan/core-webui": "^0.1.45", "lucide-react": "^1.23.0", @@ -1371,8 +1371,8 @@ } }, "node_modules/@govoplan/reporting-webui": { - "version": "0.1.21", - "resolved": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-reporting.git#89755e1924e52a9f4ee225e5699b9e446adbcd30", + "version": "0.1.22", + "resolved": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-reporting.git#8253bd0dd6fe93b739b7482a04c058f2bed3aff4", "peerDependencies": { "@govoplan/core-webui": "^0.1.45", "lucide-react": "^1.23.0", @@ -1486,8 +1486,8 @@ } }, "node_modules/@govoplan/tickets-webui": { - "version": "0.1.22", - "resolved": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-tickets.git#0ad2ef96b43a83ea7e0e9188487392db735b4a85", + "version": "0.1.23", + "resolved": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-tickets.git#2be4a0c598a32a5ce044320f2a1311a192b9d5e3", "peerDependencies": { "@govoplan/core-webui": "^0.1.30", "lucide-react": "^1.23.0", @@ -1618,6 +1618,9 @@ "cpu": [ "x64" ], + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -1734,6 +1737,9 @@ "cpu": [ "arm" ], + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -1747,6 +1753,9 @@ "cpu": [ "arm" ], + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ @@ -1760,6 +1769,9 @@ "cpu": [ "arm64" ], + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -1773,6 +1785,9 @@ "cpu": [ "arm64" ], + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ @@ -1786,6 +1801,9 @@ "cpu": [ "loong64" ], + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -1799,6 +1817,9 @@ "cpu": [ "loong64" ], + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ @@ -1812,6 +1833,9 @@ "cpu": [ "ppc64" ], + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -1825,6 +1849,9 @@ "cpu": [ "ppc64" ], + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ @@ -1838,6 +1865,9 @@ "cpu": [ "riscv64" ], + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -1851,6 +1881,9 @@ "cpu": [ "riscv64" ], + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ @@ -1864,6 +1897,9 @@ "cpu": [ "s390x" ], + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -1877,6 +1913,9 @@ "cpu": [ "x64" ], + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -1890,6 +1929,9 @@ "cpu": [ "x64" ], + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ @@ -2840,9 +2882,9 @@ } }, "node_modules/electron-to-chromium": { - "version": "1.5.422", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.422.tgz", - "integrity": "sha512-UvA/32XqrLDdZSn7Jllo1AYNcWji/G0d5M0GTViE7KoGBiMunw3a34Sb2KO4ZZyrSEhqsxFoVhWWJshdyfKqJA==", + "version": "1.5.423", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.423.tgz", + "integrity": "sha512-rRZfTSY8ptHYMQxa+uIycJMFKmY1T0GIApNMXJYGehguTZa56TEEl19pKPCoBqk5Gpf7QizZn/jt7xur+DYxag==", "license": "ISC" }, "node_modules/esbuild": { diff --git a/webui/package.json b/webui/package.json index 96143a0..fb49099 100644 --- a/webui/package.json +++ b/webui/package.json @@ -1,6 +1,6 @@ { "name": "@govoplan/core-webui", - "version": "0.1.45", + "version": "0.1.46", "private": true, "type": "module", "main": "src/index.ts", diff --git a/webui/package.release.json b/webui/package.release.json index ec6d83c..88fe5e0 100644 --- a/webui/package.release.json +++ b/webui/package.release.json @@ -1,6 +1,6 @@ { "name": "@govoplan/core-webui", - "version": "0.1.45", + "version": "0.1.46", "private": true, "type": "module", "main": "src/index.ts", @@ -35,29 +35,29 @@ }, "dependencies": { "@govoplan/access-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-access.git#v0.1.25", - "@govoplan/addresses-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-addresses.git#v0.1.22", + "@govoplan/addresses-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-addresses.git#v0.1.23", "@govoplan/admin-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-admin.git#v0.1.23", "@govoplan/approvals-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-approvals.git#v0.1.21", "@govoplan/audit-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-audit.git#v0.1.20", - "@govoplan/calendar-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-calendar.git#v0.1.23", - "@govoplan/campaign-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-campaign.git#v0.1.28", - "@govoplan/cases-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-cases.git#v0.1.24", - "@govoplan/committee-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-committee.git#v0.1.21", - "@govoplan/connectors-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-connectors.git#v0.1.26", + "@govoplan/calendar-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-calendar.git#v0.1.24", + "@govoplan/campaign-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-campaign.git#v0.1.29", + "@govoplan/cases-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-cases.git#v0.1.25", + "@govoplan/committee-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-committee.git#v0.1.22", + "@govoplan/connectors-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-connectors.git#v0.1.27", "@govoplan/dashboard-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-dashboard.git#v0.1.20", - "@govoplan/dataflow-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-dataflow.git#v0.1.24", - "@govoplan/datasources-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-datasources.git#v0.1.25", + "@govoplan/dataflow-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-dataflow.git#v0.1.25", + "@govoplan/datasources-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-datasources.git#v0.1.26", "@govoplan/dist-lists-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-dist-lists.git#v0.1.21", "@govoplan/docs-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-docs.git#v0.1.23", "@govoplan/encryption-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-encryption.git#v0.1.20", - "@govoplan/files-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-files.git#v0.1.26", - "@govoplan/forms-runtime-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-forms-runtime.git#v0.1.21", + "@govoplan/files-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-files.git#v0.1.27", + "@govoplan/forms-runtime-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-forms-runtime.git#v0.1.22", "@govoplan/forms-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-forms.git#v0.1.23", "@govoplan/helpdesk-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-helpdesk.git#v0.1.21", "@govoplan/identity-trust-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-identity-trust.git#v0.1.21", "@govoplan/identity-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-identity.git#v0.1.21", - "@govoplan/idm-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-idm.git#v0.1.25", - "@govoplan/mail-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-mail.git#v0.1.27", + "@govoplan/idm-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-idm.git#v0.1.26", + "@govoplan/mail-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-mail.git#v0.1.28", "@govoplan/notifications-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-notifications.git#v0.1.20", "@govoplan/ops-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-ops.git#v0.1.22", "@govoplan/organizations-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-organizations.git#v0.1.21", @@ -68,14 +68,14 @@ "@govoplan/projects-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-projects.git#v0.1.20", "@govoplan/quick-access-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-quick-access.git#v0.1.21", "@govoplan/records-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-records.git#v0.1.24", - "@govoplan/reporting-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-reporting.git#v0.1.21", + "@govoplan/reporting-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-reporting.git#v0.1.22", "@govoplan/risk-compliance-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-risk-compliance.git#v0.1.21", "@govoplan/scheduling-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-scheduling.git#v0.1.22", "@govoplan/search-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-search.git#v0.1.20", "@govoplan/tasks-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-tasks.git#v0.1.23", "@govoplan/templates-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-templates.git#v0.1.22", "@govoplan/tenancy-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-tenancy.git#v0.1.22", - "@govoplan/tickets-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-tickets.git#v0.1.22", + "@govoplan/tickets-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-tickets.git#v0.1.23", "@govoplan/views-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-views.git#v0.1.22", "@govoplan/voting-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-voting.git#v0.1.21", "@govoplan/wiki-webui": "git+ssh://git@git.add-ideas.de/GovOPlaN/govoplan-wiki.git#v0.1.22", diff --git a/webui/src/App.tsx b/webui/src/App.tsx index bac6114..e173a28 100644 --- a/webui/src/App.tsx +++ b/webui/src/App.tsx @@ -1,6 +1,7 @@ import { Navigate, Route, Routes, useLocation } from "react-router"; import { lazy, useEffect, useMemo, useState } from "react"; import { fetchSession, fetchShellAuth, logout, updateProfile } from "./api/auth"; +import { createModuleRefresh } from "./platform/moduleRefresh"; import type { AuthActionUiCapability } from "./types"; import { fetchPlatformModules, fetchPlatformPublicModules, fetchPlatformStatus } from "./api/platform"; import { AUTH_REQUIRED_EVENT, apiSettingsForAuthUpdate, clearApiReadCache, isApiError, loadApiSettings, saveApiSettings, type AuthRequiredEventDetail } from "./api/client"; @@ -303,45 +304,34 @@ export default function App() { return; } - let cancelled = false; - let inFlight = false; - let lastRefreshAt = 0; - function loadModules() { - inFlight = true; - lastRefreshAt = Date.now(); - return fetchPlatformModules(settings). - then((response) => {if (!cancelled) setPlatformModules(response.modules);}). - catch(() => { - if (!cancelled) { - setPlatformModules(null); - setWebModulesLoading(false); - } - }). - finally(() => {inFlight = false;}); - } - - void loadModules(); + const refresh = createModuleRefresh({ + load: () => fetchPlatformModules(settings), + accept: (response) => setPlatformModules(response.modules), + reject: () => { + setPlatformModules(null); + setWebModulesLoading(false); + } + }); + refresh.invalidate(); function handleModulesChanged() { - void loadModules(); + refresh.invalidate(); } function refreshVisibleModules() { - if (document.visibilityState === "hidden" || inFlight) return; - if (Date.now() - lastRefreshAt < 5_000) return; - void loadModules(); + refresh.refreshVisible(document.visibilityState !== "hidden"); } window.addEventListener(PLATFORM_MODULES_CHANGED_EVENT, handleModulesChanged); window.addEventListener("focus", refreshVisibleModules); document.addEventListener("visibilitychange", refreshVisibleModules); return () => { - cancelled = true; + refresh.dispose(); window.removeEventListener(PLATFORM_MODULES_CHANGED_EVENT, handleModulesChanged); window.removeEventListener("focus", refreshVisibleModules); document.removeEventListener("visibilitychange", refreshVisibleModules); }; - }, [auth?.user?.id, auth?.active_tenant?.id, auth?.tenant.id, requiredAuthAction, settings.apiBaseUrl, settings.apiKey]); + }, [auth, requiredAuthAction, settings.apiBaseUrl, settings.apiKey, settings.accessToken]); useEffect(() => { let cancelled = false; diff --git a/webui/src/api/authAuthority.ts b/webui/src/api/authAuthority.ts new file mode 100755 index 0000000..e153785 --- /dev/null +++ b/webui/src/api/authAuthority.ts @@ -0,0 +1,31 @@ +import type { ApiSettings, AuthInfo } from "../types"; + +/** Stable in-memory async-work fence, not an authorization decision or cache. + * Includes credentials: never persist, log, display or send this key. Cosmetic + * profile refreshes must not invalidate a server-accepted mutation. + */ +export function authAuthorityKey(auth: AuthInfo | null | undefined, settings: ApiSettings): string { + const set = (values: readonly string[] | null | undefined) => [...new Set(values ?? [])].sort(); + const principal = auth?.principal; + return JSON.stringify([ + settings.apiBaseUrl, settings.apiKey, settings.accessToken, + auth ? [ + auth.user.id, auth.user.account_id, auth.user.email, + Boolean(auth.user.is_tenant_admin), Boolean(auth.user.password_reset_required), + auth.user.required_auth_action ?? null, Boolean(auth.user.local_password), + auth.tenant.id, auth.tenant.is_active !== false, + (auth.active_tenant ?? auth.tenant).id, (auth.active_tenant ?? auth.tenant).is_active !== false, + set(auth.scopes), + (auth.roles ?? []).map((role) => JSON.stringify([role.id, role.slug, role.level ?? null, set(role.permissions)])).sort(), + set((auth.groups ?? []).map((group) => group.id)), + principal ? [ + principal.account_id, principal.membership_id ?? null, principal.tenant_id ?? null, + principal.identity_id ?? null, principal.auth_method, principal.api_key_id ?? null, + principal.session_id ?? null, principal.service_account_id ?? null, + principal.acting_assignment_id ?? null, principal.acting_for_account_id ?? null, + principal.email ?? null, set(principal.scopes), set(principal.group_ids), + set(principal.role_ids), set(principal.function_assignment_ids), set(principal.delegation_ids) + ] : null + ] : null + ]); +} diff --git a/webui/src/components/table/DataGrid.tsx b/webui/src/components/table/DataGrid.tsx index d7742a7..17aee7d 100644 --- a/webui/src/components/table/DataGrid.tsx +++ b/webui/src/components/table/DataGrid.tsx @@ -4,6 +4,7 @@ import { ArrowDown, ArrowUp, ChevronLeft, ChevronRight, ChevronsLeft, ChevronsRi import StatusBadge from "../StatusBadge"; import ListSelectionFilter, { type ListFilterOption } from "../ListSelectionFilter"; import TableActionGroup from "./TableActionGroup"; +import { dataGridRowIndices } from "./dataGridRowIndices"; import { usePlatformLanguage, i18nMessage } from "../../i18n/LanguageContext"; import { DATA_GRID_MAX_TRACK_WIDTH, @@ -545,6 +546,8 @@ export default function DataGrid({ return result; }, [columns, rows]); + const originalRowIndex = useMemo(() => dataGridRowIndices(rows), [rows]); + const visibleRows = useMemo(() => { if (serverQueryMode) return rows; const filters = state.filters ?? {}; @@ -558,14 +561,14 @@ export default function DataGrid({ const sortColumn = columns.find((column) => column.id === state.sort?.columnId); if (!sortColumn) return filtered; return [...filtered].sort((a, b) => { - const aIndex = rows.indexOf(a); - const bIndex = rows.indexOf(b); + const aIndex = originalRowIndex(a); + const bIndex = originalRowIndex(b); const aValue = sortColumn.sortValue?.(a, aIndex) ?? sortColumn.value?.(a, aIndex) ?? ""; const bValue = sortColumn.sortValue?.(b, bIndex) ?? sortColumn.value?.(b, bIndex) ?? ""; const result = compareValues(aValue, bValue); return state.sort?.direction === "desc" ? -result : result; }); - }, [rows, columns, state.filters, state.sort, filterTypes, serverQueryMode]); + }, [rows, columns, state.filters, state.sort, filterTypes, serverQueryMode, originalRowIndex]); const paginationMode = pagination?.mode ?? "client"; const paginationTotal = pagination ? @@ -851,7 +854,7 @@ export default function DataGrid({ ); })() : renderedRows.map((row, visibleIndex) => { - const originalIndex = rows.indexOf(row); + const originalIndex = originalRowIndex(row); const rowKey = getRowKey(row, originalIndex); const rowClass = rowClassName?.(row, originalIndex); const parityClass = visibleIndex % 2 === 0 ? "data-grid-row-even" : "data-grid-row-odd"; diff --git a/webui/src/components/table/dataGridRowIndices.ts b/webui/src/components/table/dataGridRowIndices.ts new file mode 100755 index 0000000..6013155 --- /dev/null +++ b/webui/src/components/table/dataGridRowIndices.ts @@ -0,0 +1,9 @@ +/** Cache indexOf's first-occurrence semantics without scanning rows per comparison/cell. */ +export function dataGridRowIndices(rows: readonly T[]): (row: T) => number { + const indices = new Map(); + rows.forEach((row, index) => { + if (!indices.has(row)) indices.set(row, index); + }); + // Map uses SameValueZero, whereas indexOf never finds NaN. + return (row) => row !== row ? -1 : indices.get(row) ?? -1; +} diff --git a/webui/src/index.ts b/webui/src/index.ts index a82f2b7..e0e090f 100644 --- a/webui/src/index.ts +++ b/webui/src/index.ts @@ -3,6 +3,7 @@ export * from "./types"; export * from "./api/client"; export * from "./api/concurrency"; export * from "./api/auth"; +export * from "./api/authAuthority"; export * from "./api/platform"; export * from "./api/notificationSummary"; export * from "./api/adminCommon"; diff --git a/webui/src/platform/moduleRefresh.ts b/webui/src/platform/moduleRefresh.ts new file mode 100755 index 0000000..13fec0a --- /dev/null +++ b/webui/src/platform/moduleRefresh.ts @@ -0,0 +1,44 @@ +/** One request at a time, with a mandatory trailing read after an invalidation. + * Superseded responses/errors never publish. This does not cache authorization. + */ +export function createModuleRefresh({ + load, accept, reject, now = Date.now +}: { + load: () => Promise; + accept: (value: T) => void; + reject: (error: unknown) => void; + now?: () => number; +}) { + let disposed = false; + let inFlight = false; + let generation = 0; + let lastRefreshAt = -Infinity; + + async function run() { + if (disposed || inFlight) return; + inFlight = true; + const requestedGeneration = generation; + lastRefreshAt = now(); + try { + const value = await load(); + if (!disposed && requestedGeneration === generation) accept(value); + } catch (error) { + if (!disposed && requestedGeneration === generation) reject(error); + } finally { + inFlight = false; + if (!disposed && requestedGeneration !== generation) void run(); + } + } + + return { + invalidate() { + if (disposed) return; + generation += 1; + void run(); + }, + refreshVisible(visible: boolean) { + if (visible && !inFlight && now() - lastRefreshAt >= 5_000) void run(); + }, + dispose() { disposed = true; } + }; +} diff --git a/webui/tests/auth-action-state.test.mjs b/webui/tests/auth-action-state.test.mjs index 80b8943..38d9a19 100644 --- a/webui/tests/auth-action-state.test.mjs +++ b/webui/tests/auth-action-state.test.mjs @@ -44,3 +44,60 @@ test("lightweight session changes trigger full auth refresh for required actions assert.equal(sessionMatchesAuth({ ...session, user: { ...session.user, local_password: false } }, auth), false); assert.equal(sessionMatchesAuth({ ...session, session_id: "rotated-session" }, auth), false); }); + +const authorityContext = vm.createContext({ module: { exports: {} } }); +authorityContext.exports = authorityContext.module.exports; +vm.runInContext(transformSync(readFileSync(new URL("../src/api/authAuthority.ts", import.meta.url), "utf8"), { loader: "ts", format: "cjs" }).code, authorityContext); +const { authAuthorityKey } = authorityContext.module.exports; +const settings = { apiBaseUrl: "https://fixture.invalid", apiKey: "fixture-key", accessToken: "fixture-token" }; + +test("authority fences ignore fresh object identity and cosmetic profile changes", () => { + const auth = normalizeAuthInfo(base); + const key = authAuthorityKey(auth, settings); + const refreshed = structuredClone(auth); + refreshed.user.display_name = "Changed display name"; + refreshed.user.preferred_language = "de"; + refreshed.user.ui_preferences = { compact_tables: true }; + refreshed.tenant.name = "Changed tenant name"; + refreshed.profile_loaded = true; + assert.equal(authAuthorityKey(refreshed, { ...settings }), key); +}); + +test("authority fences include principal, tenant, credential, scope, acting and required-action changes", () => { + const auth = normalizeAuthInfo(base); + const key = authAuthorityKey(auth, settings); + for (const mutate of [ + (value) => { value.user.account_id = "different-account"; }, + (value) => { value.user.email = "different@example.test"; }, + (value) => { value.user.is_tenant_admin = true; }, + (value) => { value.user.required_auth_action = "change_password"; }, + (value) => { value.user.local_password = !value.user.local_password; }, + (value) => { value.tenant.id = "different-tenant"; }, + (value) => { value.active_tenant = { ...value.tenant, id: "active-tenant" }; }, + (value) => { value.tenant.is_active = false; }, + (value) => { value.scopes = ["new:permission"]; }, + (value) => { value.roles = [{ id: "role", slug: "role", permissions: ["new:permission"] }]; }, + (value) => { value.groups = [{ id: "group" }]; }, + (value) => { value.principal.session_id = "rotated-session"; }, + (value) => { value.principal.acting_assignment_id = "assignment"; }, + (value) => { value.principal.acting_for_account_id = "actor"; }, + (value) => { value.principal.delegation_ids = ["delegation"]; } + ]) { + const changed = structuredClone(auth); + mutate(changed); + assert.notEqual(authAuthorityKey(changed, settings), key); + } + for (const field of ["apiBaseUrl", "apiKey", "accessToken"]) { + assert.notEqual(authAuthorityKey(auth, { ...settings, [field]: "changed" }), key); + } +}); + +test("authority set ordering and duplicate entries do not invent a context change", () => { + const auth = normalizeAuthInfo(base); + auth.scopes = ["b", "a", "b"]; + auth.principal.group_ids = ["g2", "g1"]; + const reordered = structuredClone(auth); + reordered.scopes = ["a", "b"]; + reordered.principal.group_ids = ["g1", "g2", "g1"]; + assert.equal(authAuthorityKey(auth, settings), authAuthorityKey(reordered, settings)); +}); diff --git a/webui/tests/data-grid-actions.test.tsx b/webui/tests/data-grid-actions.test.tsx index f831c77..175f6e7 100644 --- a/webui/tests/data-grid-actions.test.tsx +++ b/webui/tests/data-grid-actions.test.tsx @@ -6,9 +6,47 @@ import { renderToStaticMarkup } from "react-dom/server"; import Button from "../src/components/Button"; import DataGrid, { DataGridEmptyAction, DataGridRowActions, type DataGridColumn } from "../src/components/table/DataGrid"; import TableActionGroup, { runTableAction } from "../src/components/table/TableActionGroup"; +import { dataGridRowIndices } from "../src/components/table/dataGridRowIndices"; function noop() {} +const repeatedRow = { label: "same" }; +const distinctRow = { label: "same" }; +const identityRows = [repeatedRow, distinctRow, repeatedRow]; +const identityIndex = dataGridRowIndices(identityRows); +assertEqual(identityIndex(repeatedRow), 0, "repeated object references retain their first original index"); +assertEqual(identityIndex(distinctRow), 1, "equal-looking objects keep distinct identities"); +assertEqual(identityIndex({ label: "same" }), -1, "absent objects retain indexOf semantics"); +const primitiveRows = [NaN, -0, 2, 2, 0]; +const primitiveIndex = dataGridRowIndices(primitiveRows); +for (const value of primitiveRows) { + assertEqual(primitiveIndex(value), primitiveRows.indexOf(value), "primitive indices preserve NaN, zero and duplicate behavior"); +} +let linearVisits = 0; +const measuredRows = Array.from({ length: 4_000 }, (_, index) => ({ index })); +measuredRows.forEach = (callback) => Array.prototype.forEach.call(measuredRows, (row, index, array) => { + linearVisits += 1; + callback(row, index, array); +}); +const measuredIndex = dataGridRowIndices(measuredRows); +for (let pass = 0; pass < 10; pass += 1) measuredRows.map(measuredIndex); +assertEqual(linearVisits, 4_000, "index construction traverses once, independent of lookup count"); + +const callbackIndices: number[] = []; +const stableRows = [{ id: "a", rank: 2 }, { id: "b", rank: 1 }, { id: "c", rank: 1 }]; +const stableMarkup = renderToStaticMarkup( { + assertEqual(stableRows[index], row, "sort callbacks receive original indices"); + return row.rank; + }, render: (row, index) => { callbackIndices.push(index); return row.id; } }]} + getRowKey={(row, index) => { assertEqual(stableRows[index], row, "row keys receive original indices"); return row.id; }} + initialSort={{ columnId: "rank", direction: "asc" }} +/>); +assertEqual(stableMarkup.includes("stable-original-row-index"), true, "sorted grid renders"); +assertEqual(callbackIndices.slice(-3).join(), "1,2,0", "equal sort values remain stable without mutating source order after the unchanged sizing pass"); +assertEqual(stableRows.map((row) => row.id).join(), "a,b,c", "sorting never reorders input data"); + function buttonCount(markup: string): number { return markup.match(/ { throw error; }); + +async function verifyModuleRefresh() { + const requests: Array<{ resolve: (value: number) => void; reject: (error: unknown) => void }> = []; + const accepted: number[] = []; + const errors: unknown[] = []; + let time = 0; + const refresh = createModuleRefresh({ + load: () => new Promise((resolve, reject) => requests.push({ resolve, reject })), + accept: (value) => accepted.push(value), reject: (error) => errors.push(error), now: () => time + }); + const settle = async () => { await Promise.resolve(); await Promise.resolve(); }; + refresh.invalidate(); + refresh.invalidate(); + refresh.invalidate(); + assert(requests.length === 1, "mutation bursts cannot start overlapping requests"); + requests[0].resolve(0); + await settle(); + assert(accepted.length === 0 && Number(requests.length) === 2, "obsolete success is ignored and invalidation gets one trailing read"); + refresh.invalidate(); + requests[1].reject(new Error("obsolete")); + await settle(); + assert(errors.length === 0 && Number(requests.length) === 3, "obsolete error cannot clear the current catalogue or drop invalidation"); + requests[2].resolve(2); + await settle(); + assert(accepted.join() === "2", "only the current generation publishes"); + refresh.refreshVisible(true); + time = 6_000; + refresh.refreshVisible(false); + assert(Number(requests.length) === 3, "focus throttle and hidden-document checks remain"); + refresh.refreshVisible(true); + assert(Number(requests.length) === 4, "visible focus refresh remains available after the throttle"); + requests[3].reject("current failure"); + await settle(); + assert(errors.join() === "current failure", "current errors retain fail-closed handling without automatic retries"); + refresh.invalidate(); + refresh.invalidate(); + refresh.dispose(); + requests[4].resolve(4); + await settle(); + refresh.invalidate(); + assert(Number(requests.length) === 5 && accepted.join() === "2", "authority change/unmount discards both old response and queued work"); + console.log("Module refresh coalescing and generation tests passed."); +} + +void verifyModuleRefresh().catch((error) => { throw error; });