feat: implement durable permission-aware search
This commit is contained in:
@@ -6,20 +6,108 @@ from types import SimpleNamespace
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from govoplan_core.core.search import SearchDocument, SearchQuery
|
||||
from govoplan_core.core.search import (
|
||||
SearchBackfillPage,
|
||||
SearchDocument,
|
||||
SearchIndexChange,
|
||||
SearchQuery,
|
||||
SearchResourceType,
|
||||
SearchResult,
|
||||
)
|
||||
from govoplan_core.db.base import Base
|
||||
from govoplan_search.backend.db.models import (
|
||||
SearchIndexAclToken,
|
||||
SearchIndexChangeQueue,
|
||||
SearchIndexDocument,
|
||||
SearchIndexState,
|
||||
)
|
||||
from govoplan_search.backend.service import (
|
||||
SearchIndexService,
|
||||
aggregate_search_page,
|
||||
)
|
||||
from govoplan_search.backend.service import SearchIndexService
|
||||
|
||||
|
||||
class _Registry:
|
||||
def __init__(self, source=None, active_modules=("search", "cases")):
|
||||
self.source = source
|
||||
self.active_modules = active_modules
|
||||
|
||||
def manifests(self):
|
||||
return tuple(
|
||||
SimpleNamespace(id=module_id)
|
||||
for module_id in self.active_modules
|
||||
)
|
||||
|
||||
def search_sources(self):
|
||||
if self.source is None:
|
||||
return ()
|
||||
return (
|
||||
SimpleNamespace(id="search"),
|
||||
SimpleNamespace(id="cases"),
|
||||
(
|
||||
SimpleNamespace(
|
||||
registration=SimpleNamespace(id="cases.records")
|
||||
),
|
||||
self.source,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class _Source:
|
||||
def __init__(self, pages=()):
|
||||
self.pages = list(pages)
|
||||
self.authorized_ids = {"case-1"}
|
||||
|
||||
def resource_types(self):
|
||||
return (
|
||||
SearchResourceType(
|
||||
provider_id="cases.records",
|
||||
module_id="cases",
|
||||
resource_type="case",
|
||||
label="Cases",
|
||||
requires_authorization_recheck=True,
|
||||
),
|
||||
)
|
||||
|
||||
def backfill(self, session, *, request):
|
||||
del session, request
|
||||
return self.pages.pop(0)
|
||||
|
||||
def authorize(self, session, principal, *, requests):
|
||||
del session, principal
|
||||
return {
|
||||
request.reference.key: (
|
||||
request.reference.resource_id
|
||||
in self.authorized_ids
|
||||
)
|
||||
for request in requests
|
||||
}
|
||||
|
||||
|
||||
class _ResultProvider:
|
||||
def search(self, session, principal, *, query):
|
||||
del session, principal
|
||||
return tuple(
|
||||
SearchResult(
|
||||
provider_id="ignored",
|
||||
module_id="cases",
|
||||
resource_type="case",
|
||||
resource_id=f"case-{number}",
|
||||
title=f"Permit {number}",
|
||||
url=f"/cases/case-{number}",
|
||||
score=float(10 - number),
|
||||
)
|
||||
for number in range(1, 7)
|
||||
)[: query.limit]
|
||||
|
||||
|
||||
class _AggregateRegistry:
|
||||
def search_providers(self):
|
||||
return (
|
||||
(
|
||||
SimpleNamespace(
|
||||
registration=SimpleNamespace(id="cases.live")
|
||||
),
|
||||
_ResultProvider(),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -31,6 +119,8 @@ class SearchServiceTests(unittest.TestCase):
|
||||
tables=(
|
||||
SearchIndexDocument.__table__,
|
||||
SearchIndexAclToken.__table__,
|
||||
SearchIndexState.__table__,
|
||||
SearchIndexChangeQueue.__table__,
|
||||
),
|
||||
)
|
||||
self.session = Session(self.engine)
|
||||
@@ -164,6 +254,267 @@ class SearchServiceTests(unittest.TestCase):
|
||||
),
|
||||
)
|
||||
|
||||
with self.assertRaisesRegex(ValueError, "secret field"):
|
||||
self.service.upsert_document(
|
||||
self.session,
|
||||
self.principal,
|
||||
document=SearchDocument(
|
||||
tenant_id="tenant-1",
|
||||
module_id="cases",
|
||||
resource_type="case",
|
||||
resource_id="case-secret",
|
||||
title="Permit",
|
||||
url="/cases/case-secret",
|
||||
acl_tokens=("account:account-1",),
|
||||
metadata={"access_token": "must-not-index"},
|
||||
),
|
||||
)
|
||||
|
||||
def test_query_rejects_cross_tenant_principal(self) -> None:
|
||||
with self.assertRaises(PermissionError):
|
||||
self.service.search(
|
||||
self.session,
|
||||
self.principal,
|
||||
query=SearchQuery(
|
||||
text="permit",
|
||||
tenant_id="tenant-2",
|
||||
),
|
||||
)
|
||||
|
||||
def test_source_authorization_recheck_is_fail_closed(self) -> None:
|
||||
source = _Source()
|
||||
service = SearchIndexService(_Registry(source))
|
||||
for resource_id in ("case-1", "case-2"):
|
||||
service.upsert_document(
|
||||
self.session,
|
||||
self.principal,
|
||||
document=_source_document(resource_id),
|
||||
)
|
||||
self.session.flush()
|
||||
|
||||
results = service.search(
|
||||
self.session,
|
||||
self.principal,
|
||||
query=SearchQuery(
|
||||
text="permit",
|
||||
tenant_id="tenant-1",
|
||||
),
|
||||
)
|
||||
self.assertEqual(
|
||||
["case-1"],
|
||||
[item.resource_id for item in results],
|
||||
)
|
||||
unavailable_results = SearchIndexService(
|
||||
_Registry()
|
||||
).search(
|
||||
self.session,
|
||||
self.principal,
|
||||
query=SearchQuery(
|
||||
text="permit",
|
||||
tenant_id="tenant-1",
|
||||
),
|
||||
)
|
||||
self.assertEqual((), unavailable_results)
|
||||
|
||||
def test_durable_change_queue_is_idempotent(self) -> None:
|
||||
document = _source_document("case-1")
|
||||
change = SearchIndexChange(
|
||||
change_id="change-1",
|
||||
provider_id="cases.records",
|
||||
kind="upsert",
|
||||
reference=document.reference,
|
||||
source_revision=document.source_revision,
|
||||
cursor=document.change_cursor or "",
|
||||
document=document,
|
||||
)
|
||||
|
||||
self.assertTrue(
|
||||
self.service.enqueue_change(
|
||||
self.session,
|
||||
change=change,
|
||||
)
|
||||
)
|
||||
self.assertFalse(
|
||||
self.service.enqueue_change(
|
||||
self.session,
|
||||
change=change,
|
||||
)
|
||||
)
|
||||
result = self.service.process_changes(self.session)
|
||||
self.assertEqual(1, result["applied"])
|
||||
self.assertEqual(
|
||||
"case-1",
|
||||
self.session.query(SearchIndexDocument).one().resource_id,
|
||||
)
|
||||
self.assertEqual(
|
||||
"applied",
|
||||
self.session.query(SearchIndexChangeQueue).one().status,
|
||||
)
|
||||
|
||||
delete_change = SearchIndexChange(
|
||||
change_id="change-2",
|
||||
provider_id="cases.records",
|
||||
kind="delete",
|
||||
reference=document.reference,
|
||||
source_revision="3",
|
||||
cursor="cursor-2",
|
||||
)
|
||||
self.service.enqueue_change(
|
||||
self.session,
|
||||
change=delete_change,
|
||||
)
|
||||
self.service.process_changes(self.session)
|
||||
self.assertEqual(
|
||||
0,
|
||||
self.session.query(SearchIndexDocument).count(),
|
||||
)
|
||||
|
||||
def test_rebuild_resumes_and_removes_stale_documents(self) -> None:
|
||||
stale = _source_document("stale-case")
|
||||
self.service.upsert_document(
|
||||
self.session,
|
||||
self.principal,
|
||||
document=stale,
|
||||
)
|
||||
source = _Source(
|
||||
pages=(
|
||||
SearchBackfillPage(
|
||||
documents=(_source_document("case-1"),),
|
||||
next_cursor="page-2",
|
||||
complete=False,
|
||||
high_watermark="changes-9",
|
||||
),
|
||||
SearchBackfillPage(
|
||||
documents=(_source_document("case-2"),),
|
||||
next_cursor=None,
|
||||
complete=True,
|
||||
high_watermark="changes-9",
|
||||
),
|
||||
)
|
||||
)
|
||||
service = SearchIndexService(_Registry(source))
|
||||
|
||||
started = service.start_rebuild(
|
||||
self.session,
|
||||
self.principal,
|
||||
provider_id="cases.records",
|
||||
resource_type="case",
|
||||
)
|
||||
first_page = service.continue_rebuild(
|
||||
self.session,
|
||||
self.principal,
|
||||
provider_id="cases.records",
|
||||
resource_type="case",
|
||||
)
|
||||
self.assertEqual(started.id, first_page.id)
|
||||
self.assertEqual("backfilling", first_page.status)
|
||||
self.assertEqual("page-2", first_page.checkpoint_cursor)
|
||||
|
||||
complete = service.continue_rebuild(
|
||||
self.session,
|
||||
self.principal,
|
||||
provider_id="cases.records",
|
||||
resource_type="case",
|
||||
)
|
||||
self.assertEqual("ready", complete.status)
|
||||
self.assertEqual("changes-9", complete.high_watermark)
|
||||
self.assertEqual(
|
||||
["case-1", "case-2"],
|
||||
sorted(
|
||||
item.resource_id
|
||||
for item in self.session.query(
|
||||
SearchIndexDocument
|
||||
).all()
|
||||
),
|
||||
)
|
||||
|
||||
def test_module_reconciliation_disables_derived_rows(self) -> None:
|
||||
self.service.upsert_document(
|
||||
self.session,
|
||||
self.principal,
|
||||
document=SearchDocument(
|
||||
tenant_id="tenant-1",
|
||||
module_id="cases",
|
||||
resource_type="case",
|
||||
resource_id="case-1",
|
||||
title="Permit",
|
||||
url="/cases/case-1",
|
||||
acl_tokens=("account:account-1",),
|
||||
),
|
||||
)
|
||||
disabled_service = SearchIndexService(
|
||||
_Registry(active_modules=("search",))
|
||||
)
|
||||
result = disabled_service.reconcile_active_modules(
|
||||
self.session
|
||||
)
|
||||
self.assertEqual(1, result["disabled_documents"])
|
||||
self.assertFalse(
|
||||
self.session.query(SearchIndexDocument).one().active
|
||||
)
|
||||
|
||||
def test_aggregate_cursor_is_stable_and_non_overlapping(self) -> None:
|
||||
registry = _AggregateRegistry()
|
||||
first = aggregate_search_page(
|
||||
registry,
|
||||
self.session,
|
||||
self.principal,
|
||||
query=SearchQuery(
|
||||
text="permit",
|
||||
tenant_id="tenant-1",
|
||||
limit=2,
|
||||
),
|
||||
)
|
||||
second = aggregate_search_page(
|
||||
registry,
|
||||
self.session,
|
||||
self.principal,
|
||||
query=SearchQuery(
|
||||
text="permit",
|
||||
tenant_id="tenant-1",
|
||||
limit=2,
|
||||
cursor=first.next_cursor,
|
||||
),
|
||||
)
|
||||
|
||||
self.assertIsNotNone(first.next_cursor)
|
||||
self.assertEqual(
|
||||
{"case-1", "case-2"},
|
||||
{item.resource_id for item in first.results},
|
||||
)
|
||||
self.assertEqual(
|
||||
{"case-3", "case-4"},
|
||||
{item.resource_id for item in second.results},
|
||||
)
|
||||
with self.assertRaisesRegex(ValueError, "cursor"):
|
||||
aggregate_search_page(
|
||||
registry,
|
||||
self.session,
|
||||
self.principal,
|
||||
query=SearchQuery(
|
||||
text="different",
|
||||
tenant_id="tenant-1",
|
||||
limit=2,
|
||||
cursor=first.next_cursor,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _source_document(resource_id: str) -> SearchDocument:
|
||||
return SearchDocument(
|
||||
tenant_id="tenant-1",
|
||||
module_id="cases",
|
||||
resource_type="case",
|
||||
resource_id=resource_id,
|
||||
title=f"Permit {resource_id}",
|
||||
url=f"/cases/{resource_id}",
|
||||
acl_tokens=("account:account-1",),
|
||||
provider_id="cases.records",
|
||||
source_revision="2",
|
||||
change_cursor=f"cursor-{resource_id}",
|
||||
requires_authorization_recheck=True,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user