297 lines
10 KiB
Python
297 lines
10 KiB
Python
from __future__ import annotations
|
|
|
|
from collections.abc import Mapping, Sequence
|
|
from urllib.parse import quote
|
|
|
|
from sqlalchemy import and_, func, select
|
|
from sqlalchemy.orm import Session
|
|
|
|
from govoplan_core.auth import ApiPrincipal
|
|
from govoplan_core.core.events import PlatformEvent
|
|
from govoplan_core.core.modules import ModuleContext
|
|
from govoplan_core.core.search import (
|
|
SearchAuthorizationRequest,
|
|
SearchBackfillPage,
|
|
SearchBackfillRequest,
|
|
SearchDocument,
|
|
SearchIndexChange,
|
|
SearchResourceReference,
|
|
SearchResourceType,
|
|
)
|
|
|
|
from govoplan_docs.backend.db.models import (
|
|
SemanticDocumentationEntry,
|
|
SemanticDocumentationRevision,
|
|
)
|
|
from govoplan_docs.backend.semantic_service import (
|
|
prefetch_semantic_revisions,
|
|
published_revision,
|
|
semantic_entry_payload,
|
|
)
|
|
|
|
|
|
PROVIDER_ID = "docs.semantic_documentation"
|
|
RESOURCE_TYPE = "semantic_documentation"
|
|
DOCS_READ_SCOPE = "docs:documentation:read"
|
|
|
|
|
|
class SemanticDocumentationSearchSource:
|
|
def __init__(self, registry: object) -> None:
|
|
self._registry = registry
|
|
|
|
def resource_types(self) -> Sequence[SearchResourceType]:
|
|
return (
|
|
SearchResourceType(
|
|
provider_id=PROVIDER_ID,
|
|
module_id="docs",
|
|
resource_type=RESOURCE_TYPE,
|
|
label="Semantic documentation",
|
|
requires_authorization_recheck=True,
|
|
),
|
|
)
|
|
|
|
def backfill(
|
|
self,
|
|
session: object,
|
|
*,
|
|
request: SearchBackfillRequest,
|
|
) -> SearchBackfillPage:
|
|
_assert_source(request.provider_id, request.resource_type)
|
|
db = _session(session)
|
|
statement = (
|
|
select(SemanticDocumentationEntry, SemanticDocumentationRevision)
|
|
.join(
|
|
SemanticDocumentationRevision,
|
|
and_(
|
|
SemanticDocumentationRevision.id == SemanticDocumentationEntry.published_revision_id,
|
|
SemanticDocumentationRevision.entry_id == SemanticDocumentationEntry.id,
|
|
SemanticDocumentationRevision.tenant_id == SemanticDocumentationEntry.tenant_id,
|
|
SemanticDocumentationRevision.lifecycle_state == "published",
|
|
),
|
|
)
|
|
.where(
|
|
SemanticDocumentationEntry.tenant_id == request.tenant_id,
|
|
SemanticDocumentationEntry.lifecycle_state.in_(("draft", "published")),
|
|
)
|
|
)
|
|
if request.cursor:
|
|
statement = statement.where(
|
|
SemanticDocumentationEntry.id > request.cursor
|
|
)
|
|
rows = list(
|
|
db.execute(
|
|
statement.order_by(SemanticDocumentationEntry.id).limit(
|
|
request.limit + 1
|
|
)
|
|
).all()
|
|
)
|
|
has_more = len(rows) > request.limit
|
|
selected = rows[: request.limit]
|
|
high_watermark = db.scalar(
|
|
select(func.max(SemanticDocumentationEntry.updated_at)).where(
|
|
SemanticDocumentationEntry.tenant_id == request.tenant_id,
|
|
SemanticDocumentationEntry.published_revision_id.is_not(None),
|
|
)
|
|
)
|
|
return SearchBackfillPage(
|
|
documents=tuple(_document(entry, revision) for entry, revision in selected),
|
|
next_cursor=selected[-1][0].id if has_more and selected else None,
|
|
complete=not has_more,
|
|
high_watermark=(
|
|
high_watermark.isoformat() if high_watermark is not None else None
|
|
),
|
|
)
|
|
|
|
def authorize(
|
|
self,
|
|
session: object,
|
|
principal: object,
|
|
*,
|
|
requests: Sequence[SearchAuthorizationRequest],
|
|
) -> Mapping[str, bool]:
|
|
decisions = {item.reference.key: False for item in requests}
|
|
if not isinstance(principal, ApiPrincipal) or not principal.has(DOCS_READ_SCOPE):
|
|
return decisions
|
|
db = _session(session)
|
|
valid_requests = [
|
|
item
|
|
for item in requests
|
|
if (
|
|
item.reference.tenant_id == principal.tenant_id
|
|
and item.reference.module_id == "docs"
|
|
and item.reference.resource_type == RESOURCE_TYPE
|
|
)
|
|
]
|
|
ids = sorted({item.reference.resource_id for item in valid_requests})
|
|
entries: dict[str, SemanticDocumentationEntry] = {}
|
|
for offset in range(0, len(ids), 400):
|
|
rows = db.scalars(
|
|
select(SemanticDocumentationEntry).where(
|
|
SemanticDocumentationEntry.id.in_(ids[offset:offset + 400]),
|
|
SemanticDocumentationEntry.tenant_id == principal.tenant_id,
|
|
SemanticDocumentationEntry.lifecycle_state.in_(("draft", "published")),
|
|
)
|
|
)
|
|
entries.update((entry.id, entry) for entry in rows)
|
|
revisions = prefetch_semantic_revisions(
|
|
db, principal, entries=tuple(entries.values()), editor=False,
|
|
)
|
|
for item in valid_requests:
|
|
reference = item.reference
|
|
entry = entries.get(reference.resource_id)
|
|
if (
|
|
entry is None
|
|
or entry.tenant_id != principal.tenant_id
|
|
or entry.published_revision_id != item.source_revision
|
|
or entry.lifecycle_state not in {"draft", "published"}
|
|
):
|
|
continue
|
|
payload = semantic_entry_payload(
|
|
db,
|
|
self._registry,
|
|
principal,
|
|
entry=entry,
|
|
editor=False,
|
|
revisions=revisions,
|
|
)
|
|
decisions[reference.key] = bool(
|
|
payload
|
|
and payload["subject_resolution"]["availability"]
|
|
in {"available", "changed"}
|
|
)
|
|
return decisions
|
|
|
|
def index_changes_for_event(
|
|
self,
|
|
session: object,
|
|
*,
|
|
event: PlatformEvent,
|
|
delivery_key: str,
|
|
) -> Sequence[SearchIndexChange]:
|
|
if (
|
|
event.module_id != "docs"
|
|
or event.tenant is None
|
|
or event.resource is None
|
|
or event.resource.type != RESOURCE_TYPE
|
|
or event.resource.id is None
|
|
):
|
|
return ()
|
|
db = _session(session)
|
|
entry = db.get(SemanticDocumentationEntry, event.resource.id)
|
|
revision = (
|
|
published_revision(db, entry)
|
|
if (
|
|
entry is not None
|
|
and entry.tenant_id == event.tenant.id
|
|
and entry.lifecycle_state in {"draft", "published"}
|
|
)
|
|
else None
|
|
)
|
|
visible = bool(
|
|
entry is not None
|
|
and entry.tenant_id == event.tenant.id
|
|
and entry.lifecycle_state in {"draft", "published"}
|
|
and revision is not None
|
|
and revision.entry_id == entry.id
|
|
and revision.tenant_id == entry.tenant_id
|
|
and revision.lifecycle_state == "published"
|
|
)
|
|
cursor = event.event_id
|
|
document = _document(entry, revision, change_cursor=cursor) if visible else None
|
|
reference = SearchResourceReference(
|
|
tenant_id=event.tenant.id,
|
|
module_id="docs",
|
|
resource_type=RESOURCE_TYPE,
|
|
resource_id=event.resource.id,
|
|
)
|
|
return (
|
|
SearchIndexChange(
|
|
change_id=f"{delivery_key}:{PROVIDER_ID}",
|
|
provider_id=PROVIDER_ID,
|
|
kind="upsert" if document is not None else "delete",
|
|
reference=reference,
|
|
source_revision=(
|
|
document.source_revision if document is not None else cursor
|
|
),
|
|
cursor=cursor,
|
|
document=document,
|
|
occurred_at=event.occurred_at,
|
|
),
|
|
)
|
|
|
|
|
|
def create_semantic_documentation_search_source(
|
|
context: ModuleContext,
|
|
) -> SemanticDocumentationSearchSource:
|
|
return SemanticDocumentationSearchSource(context.registry)
|
|
|
|
|
|
def _document(
|
|
entry: SemanticDocumentationEntry,
|
|
revision: SemanticDocumentationRevision,
|
|
*,
|
|
change_cursor: str | None = None,
|
|
) -> SearchDocument:
|
|
content = revision.content
|
|
title = str(content.get("title") or "Semantic documentation")
|
|
summary = str(content.get("summary") or "")[:4000] or None
|
|
keywords = tuple(
|
|
str(value)[:200]
|
|
for value in (
|
|
entry.subject_module_id,
|
|
entry.subject_kind,
|
|
entry.locale,
|
|
"semantic documentation",
|
|
)
|
|
)
|
|
return SearchDocument(
|
|
tenant_id=entry.tenant_id,
|
|
module_id="docs",
|
|
provider_id=PROVIDER_ID,
|
|
resource_type=RESOURCE_TYPE,
|
|
resource_id=entry.id,
|
|
title=title,
|
|
url=(
|
|
f"/docs/semantic?entryId={quote(entry.id, safe='')}"
|
|
f"&locale={quote(entry.locale, safe='')}"
|
|
),
|
|
summary=summary,
|
|
body=revision.search_text[:200_000] or None,
|
|
keywords=keywords,
|
|
visibility="restricted",
|
|
acl_tokens=(f"scope:{DOCS_READ_SCOPE}",),
|
|
metadata={
|
|
"source_badge": "tenant_semantic",
|
|
"subject_module_id": entry.subject_module_id,
|
|
"subject_kind": entry.subject_kind,
|
|
"subject_id": entry.subject_id,
|
|
"anchor_kind": entry.anchor_kind,
|
|
"anchor_id": entry.anchor_id,
|
|
"locale": entry.locale,
|
|
"classification": content.get("classification", "internal"),
|
|
},
|
|
source_revision=revision.id,
|
|
change_cursor=change_cursor,
|
|
source_updated_at=entry.updated_at or entry.created_at,
|
|
requires_authorization_recheck=True,
|
|
)
|
|
|
|
|
|
def _assert_source(provider_id: str, resource_type: str) -> None:
|
|
if provider_id != PROVIDER_ID or resource_type != RESOURCE_TYPE:
|
|
raise ValueError("Unsupported Docs search source.")
|
|
|
|
|
|
def _session(value: object) -> Session:
|
|
if not isinstance(value, Session):
|
|
raise TypeError("Semantic documentation search requires a SQLAlchemy session.")
|
|
return value
|
|
|
|
|
|
__all__ = [
|
|
"PROVIDER_ID",
|
|
"RESOURCE_TYPE",
|
|
"SemanticDocumentationSearchSource",
|
|
"create_semantic_documentation_search_source",
|
|
]
|