feat: implement permission-aware search baseline
This commit is contained in:
3
src/govoplan_search/__init__.py
Normal file
3
src/govoplan_search/__init__.py
Normal file
@@ -0,0 +1,3 @@
|
||||
"""GovOPlaN search module."""
|
||||
|
||||
__version__ = "0.1.14"
|
||||
1
src/govoplan_search/backend/__init__.py
Normal file
1
src/govoplan_search/backend/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
"""Search backend."""
|
||||
1
src/govoplan_search/backend/db/__init__.py
Normal file
1
src/govoplan_search/backend/db/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
"""Search persistence."""
|
||||
103
src/govoplan_search/backend/db/models.py
Normal file
103
src/govoplan_search/backend/db/models.py
Normal file
@@ -0,0 +1,103 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import (
|
||||
DateTime,
|
||||
ForeignKey,
|
||||
Index,
|
||||
JSON,
|
||||
String,
|
||||
Text,
|
||||
UniqueConstraint,
|
||||
)
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
|
||||
from govoplan_core.db.base import Base, TimestampMixin
|
||||
|
||||
|
||||
def new_uuid() -> str:
|
||||
return str(uuid.uuid4())
|
||||
|
||||
|
||||
class SearchIndexDocument(Base, TimestampMixin):
|
||||
__tablename__ = "search_index_documents"
|
||||
__table_args__ = (
|
||||
UniqueConstraint(
|
||||
"tenant_id",
|
||||
"module_id",
|
||||
"resource_type",
|
||||
"resource_id",
|
||||
name="uq_search_document_resource",
|
||||
),
|
||||
Index(
|
||||
"ix_search_document_scope",
|
||||
"tenant_id",
|
||||
"module_id",
|
||||
"resource_type",
|
||||
),
|
||||
Index("ix_search_document_visibility", "tenant_id", "visibility"),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||
module_id: Mapped[str] = mapped_column(String(100), nullable=False, index=True)
|
||||
resource_type: Mapped[str] = mapped_column(
|
||||
String(100), nullable=False, index=True
|
||||
)
|
||||
resource_id: Mapped[str] = mapped_column(
|
||||
String(255), nullable=False, index=True
|
||||
)
|
||||
title: Mapped[str] = mapped_column(String(500), nullable=False)
|
||||
summary: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
body: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
keywords: Mapped[list[str]] = mapped_column(JSON, default=list, nullable=False)
|
||||
search_text: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
url: Mapped[str] = mapped_column(String(1500), nullable=False)
|
||||
visibility: Mapped[str] = mapped_column(
|
||||
String(20), default="restricted", nullable=False
|
||||
)
|
||||
external_reference: Mapped[dict[str, Any] | None] = mapped_column(
|
||||
JSON, nullable=True
|
||||
)
|
||||
metadata_: Mapped[dict[str, Any]] = mapped_column(
|
||||
"metadata", JSON, default=dict, nullable=False
|
||||
)
|
||||
content_hash: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
indexed_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False
|
||||
)
|
||||
|
||||
acl_tokens: Mapped[list["SearchIndexAclToken"]] = relationship(
|
||||
back_populates="document",
|
||||
cascade="all, delete-orphan",
|
||||
)
|
||||
|
||||
|
||||
class SearchIndexAclToken(Base):
|
||||
__tablename__ = "search_index_acl_tokens"
|
||||
__table_args__ = (
|
||||
UniqueConstraint(
|
||||
"document_id",
|
||||
"token",
|
||||
name="uq_search_index_acl_document_token",
|
||||
),
|
||||
Index("ix_search_index_acl_token", "token", "document_id"),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||
document_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("search_index_documents.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
index=True,
|
||||
)
|
||||
token: Mapped[str] = mapped_column(String(500), nullable=False, index=True)
|
||||
|
||||
document: Mapped[SearchIndexDocument] = relationship(
|
||||
back_populates="acl_tokens"
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["SearchIndexAclToken", "SearchIndexDocument", "new_uuid"]
|
||||
222
src/govoplan_search/backend/manifest.py
Normal file
222
src/govoplan_search/backend/manifest.py
Normal file
@@ -0,0 +1,222 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from govoplan_core.core.access import (
|
||||
CAPABILITY_AUTH_PERMISSION_EVALUATOR,
|
||||
CAPABILITY_AUTH_PRINCIPAL_RESOLVER,
|
||||
)
|
||||
from govoplan_core.core.module_guards import (
|
||||
drop_table_retirement_provider,
|
||||
persistent_table_uninstall_guard,
|
||||
)
|
||||
from govoplan_core.core.modules import (
|
||||
DocumentationTopic,
|
||||
FrontendModule,
|
||||
FrontendRoute,
|
||||
MigrationSpec,
|
||||
ModuleContext,
|
||||
ModuleInterfaceProvider,
|
||||
ModuleManifest,
|
||||
PermissionDefinition,
|
||||
RoleTemplate,
|
||||
)
|
||||
from govoplan_core.core.search import (
|
||||
CAPABILITY_SEARCH_INDEX_WRITER,
|
||||
SearchProviderRegistration,
|
||||
)
|
||||
from govoplan_core.core.views import ViewSurface
|
||||
from govoplan_core.db.base import Base
|
||||
from govoplan_search.backend.db import models as search_models
|
||||
|
||||
|
||||
MODULE_ID = "search"
|
||||
MODULE_NAME = "Search"
|
||||
MODULE_VERSION = "0.1.14"
|
||||
READ_SCOPE = "search:result:read"
|
||||
INDEX_SCOPE = "search:index:write"
|
||||
ADMIN_SCOPE = "search:index:admin"
|
||||
|
||||
|
||||
def _permission(scope: str, label: str, description: str) -> PermissionDefinition:
|
||||
module_id, resource, action = scope.split(":", 2)
|
||||
return PermissionDefinition(
|
||||
scope=scope,
|
||||
label=label,
|
||||
description=description,
|
||||
category="Search",
|
||||
level="tenant",
|
||||
module_id=module_id,
|
||||
resource=resource,
|
||||
action=action,
|
||||
)
|
||||
|
||||
|
||||
PERMISSIONS = (
|
||||
_permission(
|
||||
READ_SCOPE,
|
||||
"Search available content",
|
||||
"Search content the current principal is authorized to read.",
|
||||
),
|
||||
_permission(
|
||||
INDEX_SCOPE,
|
||||
"Write search index entries",
|
||||
"Publish and remove module-owned entries in the search index.",
|
||||
),
|
||||
_permission(
|
||||
ADMIN_SCOPE,
|
||||
"Administer search index",
|
||||
"Inspect providers and rebuild tenant search indexes.",
|
||||
),
|
||||
)
|
||||
|
||||
ROLE_TEMPLATES = (
|
||||
RoleTemplate(
|
||||
slug="search_user",
|
||||
name="Search user",
|
||||
description="Search content available to the current account.",
|
||||
permissions=(READ_SCOPE,),
|
||||
default_authenticated=True,
|
||||
),
|
||||
RoleTemplate(
|
||||
slug="search_manager",
|
||||
name="Search manager",
|
||||
description="Search content and administer indexing.",
|
||||
permissions=(READ_SCOPE, INDEX_SCOPE, ADMIN_SCOPE),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _service(context: ModuleContext):
|
||||
from govoplan_search.backend.service import SearchIndexService
|
||||
|
||||
return SearchIndexService(context.registry)
|
||||
|
||||
|
||||
def _router(_context: ModuleContext):
|
||||
from govoplan_search.backend.router import router
|
||||
|
||||
return router
|
||||
|
||||
|
||||
manifest = ModuleManifest(
|
||||
id=MODULE_ID,
|
||||
name=MODULE_NAME,
|
||||
version=MODULE_VERSION,
|
||||
optional_dependencies=(
|
||||
"access",
|
||||
"views",
|
||||
"connectors",
|
||||
"wiki",
|
||||
"projects",
|
||||
"tickets",
|
||||
"cases",
|
||||
),
|
||||
required_capabilities=(
|
||||
CAPABILITY_AUTH_PRINCIPAL_RESOLVER,
|
||||
CAPABILITY_AUTH_PERMISSION_EVALUATOR,
|
||||
),
|
||||
provides_interfaces=(
|
||||
ModuleInterfaceProvider(name="search.provider", version="1.0.0"),
|
||||
ModuleInterfaceProvider(name="search.index_writer", version="1.0.0"),
|
||||
),
|
||||
permissions=PERMISSIONS,
|
||||
role_templates=ROLE_TEMPLATES,
|
||||
route_factory=_router,
|
||||
frontend=FrontendModule(
|
||||
module_id=MODULE_ID,
|
||||
package_name="@govoplan/search-webui",
|
||||
routes=(
|
||||
FrontendRoute(
|
||||
path="/search",
|
||||
component="SearchPage",
|
||||
required_any=(READ_SCOPE,),
|
||||
order=12,
|
||||
),
|
||||
),
|
||||
view_surfaces=(
|
||||
ViewSurface(
|
||||
id="search.global",
|
||||
module_id=MODULE_ID,
|
||||
kind="selector",
|
||||
label="Global search",
|
||||
description="Search entry in the title bar.",
|
||||
order=10,
|
||||
),
|
||||
ViewSurface(
|
||||
id="search.results",
|
||||
module_id=MODULE_ID,
|
||||
kind="route",
|
||||
label="Search results",
|
||||
order=20,
|
||||
),
|
||||
),
|
||||
),
|
||||
migration_spec=MigrationSpec(
|
||||
module_id=MODULE_ID,
|
||||
metadata=Base.metadata,
|
||||
script_location=str(Path(__file__).with_name("migrations") / "versions"),
|
||||
retirement_supported=True,
|
||||
retirement_provider=drop_table_retirement_provider(
|
||||
search_models.SearchIndexAclToken,
|
||||
search_models.SearchIndexDocument,
|
||||
label="Search",
|
||||
),
|
||||
retirement_notes=(
|
||||
"Destructive retirement removes the derived search index. "
|
||||
"Source module data remains authoritative."
|
||||
),
|
||||
),
|
||||
uninstall_guard_providers=(
|
||||
persistent_table_uninstall_guard(
|
||||
search_models.SearchIndexDocument,
|
||||
label="Search index",
|
||||
),
|
||||
),
|
||||
capability_factories={
|
||||
CAPABILITY_SEARCH_INDEX_WRITER: _service,
|
||||
},
|
||||
search_providers=(
|
||||
SearchProviderRegistration(
|
||||
id="search.index",
|
||||
factory=_service,
|
||||
order=10,
|
||||
),
|
||||
),
|
||||
documentation=(
|
||||
DocumentationTopic(
|
||||
id="search.global-and-contextual",
|
||||
title="Global and contextual search",
|
||||
summary=(
|
||||
"Search authorized native and connected objects from one "
|
||||
"permission-aware interface."
|
||||
),
|
||||
body=(
|
||||
"Search works with the built-in database index and can aggregate "
|
||||
"optional providers. Source modules announce searchable types, "
|
||||
"context scopes, and ACL-aware index entries. External engines "
|
||||
"remain optional adapters."
|
||||
),
|
||||
layer="available",
|
||||
documentation_types=("admin", "user"),
|
||||
audience=("administrator", "user"),
|
||||
related_modules=("connectors", "views"),
|
||||
order=12,
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def get_manifest() -> ModuleManifest:
|
||||
return manifest
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ADMIN_SCOPE",
|
||||
"INDEX_SCOPE",
|
||||
"MODULE_ID",
|
||||
"MODULE_VERSION",
|
||||
"READ_SCOPE",
|
||||
"get_manifest",
|
||||
"manifest",
|
||||
]
|
||||
1
src/govoplan_search/backend/migrations/__init__.py
Normal file
1
src/govoplan_search/backend/migrations/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
"""Search migrations."""
|
||||
@@ -0,0 +1 @@
|
||||
"""Search migration revisions."""
|
||||
@@ -0,0 +1,119 @@
|
||||
"""Create the built-in search index.
|
||||
|
||||
Revision ID: a1b2c3d4e5f6
|
||||
Revises:
|
||||
Create Date: 2026-07-29
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
revision = "a1b2c3d4e5f6"
|
||||
down_revision = None
|
||||
branch_labels = ("search",)
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"search_index_documents",
|
||||
sa.Column("id", sa.String(length=36), nullable=False),
|
||||
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
||||
sa.Column("module_id", sa.String(length=100), nullable=False),
|
||||
sa.Column("resource_type", sa.String(length=100), nullable=False),
|
||||
sa.Column("resource_id", sa.String(length=255), nullable=False),
|
||||
sa.Column("title", sa.String(length=500), nullable=False),
|
||||
sa.Column("summary", sa.Text(), nullable=True),
|
||||
sa.Column("body", sa.Text(), nullable=True),
|
||||
sa.Column("keywords", sa.JSON(), nullable=False),
|
||||
sa.Column("search_text", sa.Text(), nullable=False),
|
||||
sa.Column("url", sa.String(length=1500), nullable=False),
|
||||
sa.Column("visibility", sa.String(length=20), nullable=False),
|
||||
sa.Column("external_reference", sa.JSON(), nullable=True),
|
||||
sa.Column("metadata", sa.JSON(), nullable=False),
|
||||
sa.Column("content_hash", sa.String(length=64), nullable=False),
|
||||
sa.Column("indexed_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_search_index_documents")),
|
||||
sa.UniqueConstraint(
|
||||
"tenant_id",
|
||||
"module_id",
|
||||
"resource_type",
|
||||
"resource_id",
|
||||
name="uq_search_document_resource",
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_search_document_scope",
|
||||
"search_index_documents",
|
||||
["tenant_id", "module_id", "resource_type"],
|
||||
unique=False,
|
||||
)
|
||||
op.create_index(
|
||||
"ix_search_document_visibility",
|
||||
"search_index_documents",
|
||||
["tenant_id", "visibility"],
|
||||
unique=False,
|
||||
)
|
||||
for column in ("tenant_id", "module_id", "resource_type", "resource_id"):
|
||||
op.create_index(
|
||||
op.f(f"ix_search_index_documents_{column}"),
|
||||
"search_index_documents",
|
||||
[column],
|
||||
unique=False,
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"search_index_acl_tokens",
|
||||
sa.Column("id", sa.String(length=36), nullable=False),
|
||||
sa.Column("document_id", sa.String(length=36), nullable=False),
|
||||
sa.Column("token", sa.String(length=500), nullable=False),
|
||||
sa.ForeignKeyConstraint(
|
||||
["document_id"],
|
||||
["search_index_documents.id"],
|
||||
name=op.f(
|
||||
"fk_search_index_acl_tokens_document_id_search_index_documents"
|
||||
),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_search_index_acl_tokens")),
|
||||
sa.UniqueConstraint(
|
||||
"document_id",
|
||||
"token",
|
||||
name="uq_search_index_acl_document_token",
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
op.f("ix_search_index_acl_tokens_document_id"),
|
||||
"search_index_acl_tokens",
|
||||
["document_id"],
|
||||
unique=False,
|
||||
)
|
||||
op.create_index(
|
||||
op.f("ix_search_index_acl_tokens_token"),
|
||||
"search_index_acl_tokens",
|
||||
["token"],
|
||||
unique=False,
|
||||
)
|
||||
op.create_index(
|
||||
"ix_search_index_acl_token",
|
||||
"search_index_acl_tokens",
|
||||
["token", "document_id"],
|
||||
unique=False,
|
||||
)
|
||||
|
||||
if op.get_bind().dialect.name == "postgresql":
|
||||
op.execute(
|
||||
"CREATE INDEX ix_search_document_fts "
|
||||
"ON search_index_documents USING gin "
|
||||
"(to_tsvector('simple', search_text))"
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table("search_index_acl_tokens")
|
||||
op.drop_table("search_index_documents")
|
||||
121
src/govoplan_search/backend/router.py
Normal file
121
src/govoplan_search/backend/router.py
Normal file
@@ -0,0 +1,121 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, Request, status
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from govoplan_core.auth import ApiPrincipal, get_api_principal, has_scope
|
||||
from govoplan_core.core.registry import PlatformRegistry
|
||||
from govoplan_core.core.search import SearchQuery
|
||||
from govoplan_core.db.session import get_session
|
||||
from govoplan_search.backend.manifest import READ_SCOPE
|
||||
from govoplan_search.backend.schemas import (
|
||||
SearchProviderListResponse,
|
||||
SearchProviderResponse,
|
||||
SearchResponse,
|
||||
SearchResultResponse,
|
||||
)
|
||||
from govoplan_search.backend.service import aggregate_search
|
||||
|
||||
|
||||
router = APIRouter(prefix="/search", tags=["search"])
|
||||
|
||||
|
||||
def _registry(request: Request) -> PlatformRegistry:
|
||||
registry = getattr(request.app.state, "govoplan_registry", None)
|
||||
if not isinstance(registry, PlatformRegistry):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
detail="Search registry is not available.",
|
||||
)
|
||||
return registry
|
||||
|
||||
|
||||
def _require_read(principal: ApiPrincipal) -> None:
|
||||
if not has_scope(principal, READ_SCOPE):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail=f"Missing scope: {READ_SCOPE}",
|
||||
)
|
||||
|
||||
|
||||
@router.get("", response_model=SearchResponse)
|
||||
def api_search(
|
||||
request: Request,
|
||||
q: str = Query(default="", max_length=500),
|
||||
module: list[str] = Query(default=[]),
|
||||
resource_type: list[str] = Query(default=[]),
|
||||
context_kind: str = Query(default="global", pattern="^(global|module|resource)$"),
|
||||
context_id: str | None = Query(default=None, max_length=255),
|
||||
limit: int = Query(default=25, ge=1, le=100),
|
||||
offset: int = Query(default=0, ge=0, le=10_000),
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(get_api_principal),
|
||||
) -> SearchResponse:
|
||||
_require_read(principal)
|
||||
registry = _registry(request)
|
||||
query = SearchQuery(
|
||||
text=q,
|
||||
tenant_id=principal.tenant_id,
|
||||
module_ids=tuple(dict.fromkeys(module)),
|
||||
resource_types=tuple(dict.fromkeys(resource_type)),
|
||||
context_kind=context_kind, # type: ignore[arg-type]
|
||||
context_id=context_id,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
)
|
||||
results, diagnostics = aggregate_search(
|
||||
registry,
|
||||
session,
|
||||
principal,
|
||||
query=query,
|
||||
)
|
||||
return SearchResponse(
|
||||
query=query.text,
|
||||
results=[
|
||||
SearchResultResponse.model_validate(
|
||||
{
|
||||
"provider_id": result.provider_id,
|
||||
"module_id": result.module_id,
|
||||
"resource_type": result.resource_type,
|
||||
"resource_id": result.resource_id,
|
||||
"title": result.title,
|
||||
"summary": result.summary,
|
||||
"url": result.url,
|
||||
"score": result.score,
|
||||
"highlights": list(result.highlights),
|
||||
"breadcrumbs": list(result.breadcrumbs),
|
||||
"external_reference": (
|
||||
result.external_reference.to_dict()
|
||||
if result.external_reference is not None
|
||||
else None
|
||||
),
|
||||
"metadata": dict(result.metadata),
|
||||
}
|
||||
)
|
||||
for result in results
|
||||
],
|
||||
diagnostics=list(diagnostics),
|
||||
)
|
||||
|
||||
|
||||
@router.get("/providers", response_model=SearchProviderListResponse)
|
||||
def api_search_providers(
|
||||
request: Request,
|
||||
principal: ApiPrincipal = Depends(get_api_principal),
|
||||
) -> SearchProviderListResponse:
|
||||
_require_read(principal)
|
||||
registrations = _registry(request).search_provider_registrations()
|
||||
return SearchProviderListResponse(
|
||||
providers=[
|
||||
SearchProviderResponse(
|
||||
id=item.registration.id,
|
||||
module_id=item.module_id,
|
||||
resource_types=list(item.registration.resource_types),
|
||||
order=item.registration.order,
|
||||
)
|
||||
for item in registrations
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["router"]
|
||||
65
src/govoplan_search/backend/schemas.py
Normal file
65
src/govoplan_search/backend/schemas.py
Normal file
@@ -0,0 +1,65 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class SearchExternalReferenceResponse(BaseModel):
|
||||
system: str
|
||||
object_type: str
|
||||
object_id: str
|
||||
maturity: str
|
||||
connector_id: str | None = None
|
||||
canonical_url: str | None = None
|
||||
version: str | None = None
|
||||
etag: str | None = None
|
||||
observed_at: str | None = None
|
||||
metadata: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class SearchResultResponse(BaseModel):
|
||||
provider_id: str
|
||||
module_id: str
|
||||
resource_type: str
|
||||
resource_id: str
|
||||
title: str
|
||||
summary: str | None = None
|
||||
url: str
|
||||
score: float = 0.0
|
||||
highlights: list[str] = Field(default_factory=list)
|
||||
breadcrumbs: list[str] = Field(default_factory=list)
|
||||
external_reference: SearchExternalReferenceResponse | None = None
|
||||
metadata: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class SearchProviderDiagnosticResponse(BaseModel):
|
||||
provider_id: str
|
||||
status: Literal["unavailable"]
|
||||
message: str
|
||||
|
||||
|
||||
class SearchResponse(BaseModel):
|
||||
query: str
|
||||
results: list[SearchResultResponse]
|
||||
diagnostics: list[SearchProviderDiagnosticResponse] = Field(default_factory=list)
|
||||
|
||||
|
||||
class SearchProviderResponse(BaseModel):
|
||||
id: str
|
||||
module_id: str
|
||||
resource_types: list[str]
|
||||
order: int
|
||||
|
||||
|
||||
class SearchProviderListResponse(BaseModel):
|
||||
providers: list[SearchProviderResponse]
|
||||
|
||||
|
||||
__all__ = [
|
||||
"SearchProviderDiagnosticResponse",
|
||||
"SearchProviderListResponse",
|
||||
"SearchProviderResponse",
|
||||
"SearchResponse",
|
||||
"SearchResultResponse",
|
||||
]
|
||||
334
src/govoplan_search/backend/service.py
Normal file
334
src/govoplan_search/backend/service.py
Normal file
@@ -0,0 +1,334 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
from dataclasses import replace
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import delete, exists, func, literal, or_, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from govoplan_core.core.external_references import ExternalObjectReference
|
||||
from govoplan_core.core.search import SearchDocument, SearchQuery, SearchResult
|
||||
from govoplan_search.backend.db.models import (
|
||||
SearchIndexAclToken,
|
||||
SearchIndexDocument,
|
||||
)
|
||||
|
||||
|
||||
LOGGER = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SearchIndexService:
|
||||
def __init__(self, registry: object | None = None) -> None:
|
||||
self.registry = registry
|
||||
|
||||
def upsert_document(
|
||||
self,
|
||||
session: object,
|
||||
principal: object,
|
||||
*,
|
||||
document: SearchDocument,
|
||||
) -> None:
|
||||
db = _session(session)
|
||||
_require_tenant(principal, document.tenant_id)
|
||||
normalized_tokens = tuple(
|
||||
dict.fromkeys(token.strip() for token in document.acl_tokens if token.strip())
|
||||
)
|
||||
if document.visibility == "restricted" and not normalized_tokens:
|
||||
raise ValueError("Restricted search documents require ACL tokens.")
|
||||
identity = (
|
||||
document.tenant_id,
|
||||
document.module_id,
|
||||
document.resource_type,
|
||||
document.resource_id,
|
||||
)
|
||||
model = db.scalar(
|
||||
select(SearchIndexDocument).where(
|
||||
SearchIndexDocument.tenant_id == identity[0],
|
||||
SearchIndexDocument.module_id == identity[1],
|
||||
SearchIndexDocument.resource_type == identity[2],
|
||||
SearchIndexDocument.resource_id == identity[3],
|
||||
)
|
||||
)
|
||||
payload = _document_payload(document)
|
||||
if model is None:
|
||||
model = SearchIndexDocument(**payload)
|
||||
db.add(model)
|
||||
db.flush()
|
||||
else:
|
||||
for key, value in payload.items():
|
||||
setattr(model, key, value)
|
||||
db.execute(
|
||||
delete(SearchIndexAclToken).where(
|
||||
SearchIndexAclToken.document_id == model.id
|
||||
)
|
||||
)
|
||||
for token in normalized_tokens:
|
||||
db.add(SearchIndexAclToken(document_id=model.id, token=token))
|
||||
|
||||
def delete_document(
|
||||
self,
|
||||
session: object,
|
||||
principal: object,
|
||||
*,
|
||||
tenant_id: str,
|
||||
module_id: str,
|
||||
resource_type: str,
|
||||
resource_id: str,
|
||||
) -> bool:
|
||||
db = _session(session)
|
||||
_require_tenant(principal, tenant_id)
|
||||
model = db.scalar(
|
||||
select(SearchIndexDocument).where(
|
||||
SearchIndexDocument.tenant_id == tenant_id,
|
||||
SearchIndexDocument.module_id == module_id,
|
||||
SearchIndexDocument.resource_type == resource_type,
|
||||
SearchIndexDocument.resource_id == resource_id,
|
||||
)
|
||||
)
|
||||
if model is None:
|
||||
return False
|
||||
db.delete(model)
|
||||
return True
|
||||
|
||||
def search(
|
||||
self,
|
||||
session: object,
|
||||
principal: object,
|
||||
*,
|
||||
query: SearchQuery,
|
||||
) -> tuple[SearchResult, ...]:
|
||||
db = _session(session)
|
||||
_require_tenant(principal, query.tenant_id)
|
||||
if not query.text:
|
||||
return ()
|
||||
active_modules = _active_module_ids(self.registry)
|
||||
if active_modules == ():
|
||||
return ()
|
||||
|
||||
tokens = principal_acl_tokens(principal)
|
||||
acl_match = exists(
|
||||
select(SearchIndexAclToken.id).where(
|
||||
SearchIndexAclToken.document_id == SearchIndexDocument.id,
|
||||
SearchIndexAclToken.token.in_(tokens or ("__no_acl_token__",)),
|
||||
)
|
||||
)
|
||||
filters = [
|
||||
SearchIndexDocument.tenant_id == query.tenant_id,
|
||||
SearchIndexDocument.module_id.in_(active_modules),
|
||||
or_(
|
||||
SearchIndexDocument.visibility == "tenant",
|
||||
acl_match,
|
||||
),
|
||||
]
|
||||
if query.module_ids:
|
||||
filters.append(SearchIndexDocument.module_id.in_(query.module_ids))
|
||||
if query.resource_types:
|
||||
filters.append(
|
||||
SearchIndexDocument.resource_type.in_(query.resource_types)
|
||||
)
|
||||
|
||||
dialect = db.get_bind().dialect.name
|
||||
if dialect == "postgresql":
|
||||
search_query = func.websearch_to_tsquery("simple", query.text)
|
||||
vector = func.to_tsvector("simple", SearchIndexDocument.search_text)
|
||||
rank = func.ts_rank_cd(vector, search_query)
|
||||
statement = (
|
||||
select(SearchIndexDocument, rank.label("search_rank"))
|
||||
.where(*filters, vector.op("@@")(search_query))
|
||||
.order_by(rank.desc(), SearchIndexDocument.title.asc())
|
||||
)
|
||||
else:
|
||||
lowered = func.lower(SearchIndexDocument.search_text)
|
||||
text_filters = [
|
||||
lowered.contains(token.casefold())
|
||||
for token in query.text.split()
|
||||
if token.strip()
|
||||
]
|
||||
statement = (
|
||||
select(SearchIndexDocument, literal(1.0).label("search_rank"))
|
||||
.where(*filters, *text_filters)
|
||||
.order_by(SearchIndexDocument.title.asc())
|
||||
)
|
||||
rows = db.execute(
|
||||
statement.limit(query.limit).offset(query.offset)
|
||||
).all()
|
||||
return tuple(
|
||||
_search_result(model, float(rank or 0.0))
|
||||
for model, rank in rows
|
||||
)
|
||||
|
||||
|
||||
def aggregate_search(
|
||||
registry: object,
|
||||
session: Session,
|
||||
principal: object,
|
||||
*,
|
||||
query: SearchQuery,
|
||||
) -> tuple[tuple[SearchResult, ...], tuple[dict[str, str], ...]]:
|
||||
provider_limit = min(200, query.limit + query.offset)
|
||||
provider_query = replace(query, limit=provider_limit, offset=0)
|
||||
results: dict[tuple[str, str, str], SearchResult] = {}
|
||||
diagnostics: list[dict[str, str]] = []
|
||||
for registered, provider in registry.search_providers():
|
||||
provider_id = registered.registration.id
|
||||
try:
|
||||
provided = provider.search(
|
||||
session,
|
||||
principal,
|
||||
query=provider_query,
|
||||
)
|
||||
except Exception:
|
||||
LOGGER.exception("Search provider failed: %s", provider_id)
|
||||
diagnostics.append(
|
||||
{
|
||||
"provider_id": provider_id,
|
||||
"status": "unavailable",
|
||||
"message": "This search provider is temporarily unavailable.",
|
||||
}
|
||||
)
|
||||
continue
|
||||
for item in provided[:provider_limit]:
|
||||
normalized = replace(item, provider_id=provider_id)
|
||||
key = (
|
||||
normalized.module_id,
|
||||
normalized.resource_type,
|
||||
normalized.resource_id,
|
||||
)
|
||||
current = results.get(key)
|
||||
if current is None or normalized.score > current.score:
|
||||
results[key] = normalized
|
||||
ordered = sorted(
|
||||
results.values(),
|
||||
key=lambda item: (-item.score, item.title.casefold(), item.resource_id),
|
||||
)
|
||||
return (
|
||||
tuple(ordered[query.offset : query.offset + query.limit]),
|
||||
tuple(diagnostics),
|
||||
)
|
||||
|
||||
|
||||
def principal_acl_tokens(principal: object) -> tuple[str, ...]:
|
||||
token_groups: tuple[tuple[str, Any], ...] = (
|
||||
("account", getattr(principal, "account_id", None)),
|
||||
("membership", getattr(principal, "membership_id", None)),
|
||||
("identity", getattr(principal, "identity_id", None)),
|
||||
)
|
||||
tokens = [
|
||||
f"{prefix}:{value}"
|
||||
for prefix, value in token_groups
|
||||
if value
|
||||
]
|
||||
for prefix, attribute in (
|
||||
("group", "group_ids"),
|
||||
("role", "role_ids"),
|
||||
("function", "function_assignment_ids"),
|
||||
("scope", "scopes"),
|
||||
):
|
||||
tokens.extend(
|
||||
f"{prefix}:{value}"
|
||||
for value in getattr(principal, attribute, ())
|
||||
if value
|
||||
)
|
||||
return tuple(dict.fromkeys(tokens))
|
||||
|
||||
|
||||
def _active_module_ids(registry: object | None) -> tuple[str, ...]:
|
||||
if registry is None or not hasattr(registry, "manifests"):
|
||||
return ()
|
||||
return tuple(manifest.id for manifest in registry.manifests())
|
||||
|
||||
|
||||
def _document_payload(document: SearchDocument) -> dict[str, Any]:
|
||||
search_text = " ".join(
|
||||
part
|
||||
for part in (
|
||||
document.title,
|
||||
document.summary or "",
|
||||
document.body or "",
|
||||
" ".join(document.keywords),
|
||||
)
|
||||
if part
|
||||
)
|
||||
external_reference = (
|
||||
document.external_reference.to_dict()
|
||||
if document.external_reference is not None
|
||||
else None
|
||||
)
|
||||
content = {
|
||||
"title": document.title,
|
||||
"summary": document.summary,
|
||||
"body": document.body,
|
||||
"keywords": document.keywords,
|
||||
"url": document.url,
|
||||
"visibility": document.visibility,
|
||||
"external_reference": external_reference,
|
||||
"metadata": dict(document.metadata),
|
||||
}
|
||||
content_hash = hashlib.sha256(
|
||||
json.dumps(content, sort_keys=True, default=str).encode("utf-8")
|
||||
).hexdigest()
|
||||
return {
|
||||
"tenant_id": document.tenant_id,
|
||||
"module_id": document.module_id,
|
||||
"resource_type": document.resource_type,
|
||||
"resource_id": document.resource_id,
|
||||
"title": document.title,
|
||||
"summary": document.summary,
|
||||
"body": document.body,
|
||||
"keywords": list(document.keywords),
|
||||
"search_text": search_text,
|
||||
"url": document.url,
|
||||
"visibility": document.visibility,
|
||||
"external_reference": external_reference,
|
||||
"metadata_": dict(document.metadata),
|
||||
"content_hash": content_hash,
|
||||
"indexed_at": datetime.now(timezone.utc),
|
||||
}
|
||||
|
||||
|
||||
def _search_result(
|
||||
document: SearchIndexDocument,
|
||||
rank: float,
|
||||
) -> SearchResult:
|
||||
reference = None
|
||||
if document.external_reference:
|
||||
payload = dict(document.external_reference)
|
||||
observed_at = payload.get("observed_at")
|
||||
if isinstance(observed_at, str) and observed_at:
|
||||
payload["observed_at"] = datetime.fromisoformat(observed_at)
|
||||
reference = ExternalObjectReference(**payload)
|
||||
return SearchResult(
|
||||
provider_id="search.index",
|
||||
module_id=document.module_id,
|
||||
resource_type=document.resource_type,
|
||||
resource_id=document.resource_id,
|
||||
title=document.title,
|
||||
summary=document.summary,
|
||||
url=document.url,
|
||||
score=rank,
|
||||
external_reference=reference,
|
||||
metadata=dict(document.metadata_),
|
||||
)
|
||||
|
||||
|
||||
def _require_tenant(principal: object, tenant_id: str) -> None:
|
||||
principal_tenant = str(getattr(principal, "tenant_id", "") or "")
|
||||
if principal_tenant != tenant_id:
|
||||
raise PermissionError("Search operations cannot cross tenant boundaries.")
|
||||
|
||||
|
||||
def _session(value: object) -> Session:
|
||||
if not isinstance(value, Session):
|
||||
raise TypeError("Search index operations require a SQLAlchemy session.")
|
||||
return value
|
||||
|
||||
|
||||
__all__ = [
|
||||
"SearchIndexService",
|
||||
"aggregate_search",
|
||||
"principal_acl_tokens",
|
||||
]
|
||||
1
src/govoplan_search/py.typed
Normal file
1
src/govoplan_search/py.typed
Normal file
@@ -0,0 +1 @@
|
||||
|
||||
Reference in New Issue
Block a user