GovOPlaN/govoplan-calendar
1
0
Fork 0
Code Issues 6 Pull Requests Actions Packages Projects Releases Wiki Activity

intermittent commit

Browse Source
This commit is contained in:
Albrecht Degering
2026-07-14 13:22:10 +02:00
parent c071235ad7
commit 371a00aff5
8 changed files with 561 additions and 281 deletions
Download Patch File Download Diff File Expand all files Collapse all files

323
src/govoplan_calendar/backend/service.py
View File

@@ -7,7 +7,6 @@ import os
import urllib.parse
import urllib.error
import urllib.request
import xml.etree.ElementTree as ET
from dataclasses import dataclass, field
from datetime import datetime, timedelta, timezone
from typing import Any, Callable
@@ -19,7 +18,7 @@ from sqlalchemy.orm import Session
from govoplan_calendar.backend.caldav import CalDAVClient, CalDAVError, CalDAVNotFound, CalDAVPreconditionFailed, CalDAVReportResult, CalDAVSyncUnsupported, ensure_collection_url
from govoplan_calendar.backend.db.models import CalendarCollection, CalendarEvent, CalendarSyncCredential, CalendarSyncSource
from govoplan_calendar.backend.ical import event_to_ics, events_to_ics, expand_event_occurrences, parse_vevent, parse_vevents
from govoplan_calendar.backend.ical import events_to_ics, expand_event_occurrences, parse_vevent, parse_vevents
from govoplan_calendar.backend.runtime import get_registry
from govoplan_calendar.backend.schemas import (
CalendarCalDavDiscoveryRequest,
@@ -33,6 +32,7 @@ from govoplan_calendar.backend.schemas import (
CalendarSyncSourceCreateRequest,
CalendarSyncSourceUpdateRequest,
)
from govoplan_core.core.notifications import NotificationDispatchRequest, notification_dispatch_provider
from govoplan_core.db.base import utcnow
from govoplan_core.core.change_sequence import record_change
from govoplan_core.security.secrets import CAPABILITY_SECURITY_SECRET_PROVIDER, decrypt_secret, encrypt_secret
@@ -54,6 +54,7 @@ EWS_TYPES_NS = "http://schemas.microsoft.com/exchange/services/2006/types"
CALENDAR_MODULE_ID = "calendar"
CALENDAR_EVENTS_COLLECTION = "calendar.events"
CALENDAR_EVENT_RESOURCE = "calendar_event"
SOURCE_EVENT_CLEANUP_BATCH_SIZE = 500
def calendar_event_change_payload(event: CalendarEvent, *, prefix: str = "") -> dict[str, Any]:
@@ -186,6 +187,107 @@ def sync_source_label(source_kind: str) -> str:
}.get(source_kind, source_kind)
def _sync_stats_payload(stats: CalendarCalDavSyncStats | None) -> dict[str, object]:
if stats is None:
return {}
return {
"created": stats.created,
"updated": stats.updated,
"deleted": stats.deleted,
"unchanged": stats.unchanged,
"fetched": stats.fetched,
"full_sync": stats.full_sync,
"used_sync_token": stats.used_sync_token,
"errors": list(stats.errors),
}
def _emit_calendar_sync_notification(
session: Session,
*,
source: CalendarSyncSource,
status: str,
previous_status: str | None,
stats: CalendarCalDavSyncStats | None = None,
error: str | None = None,
) -> None:
if status == "ok" and previous_status != "error":
return
if status == "error" and previous_status == "error":
return
provider = notification_dispatch_provider(get_registry())
if provider is None:
return
label = source.display_name or sync_source_label(source.source_kind)
calendar_name = source.calendar.name if source.calendar else "calendar"
if status == "ok":
subject = f"Calendar sync recovered: {calendar_name}"
body_text = f"{label} synchronized successfully after a previous failure."
priority = 1
else:
subject = f"Calendar sync failed: {calendar_name}"
body_text = f"{label} could not be synchronized. {error or 'No error detail was provided.'}"
priority = 5
try:
provider.enqueue_notification(
session,
NotificationDispatchRequest(
tenant_id=source.tenant_id,
source_module=CALENDAR_MODULE_ID,
source_resource_type="calendar_sync_source",
source_resource_id=source.id,
event_kind=f"calendar.sync.{status}",
channel="inbox",
subject=subject,
body_text=body_text,
action_url="/calendar",
priority=priority,
payload={
"calendar_id": source.calendar_id,
"source_id": source.id,
"source_kind": source.source_kind,
"status": status,
"previous_status": previous_status,
"error": error,
"stats": _sync_stats_payload(stats),
},
metadata={"calendar_id": source.calendar_id, "source_kind": source.source_kind},
),
enqueue_delivery=False,
)
except Exception:
return
def _finalize_sync_success(
session: Session,
*,
source: CalendarSyncSource,
stats: CalendarCalDavSyncStats | None = None,
mark_calendar: Callable[[CalendarCollection, CalendarSyncSource], None] | None = None,
update_stats_tokens: bool = False,
) -> None:
source.last_synced_at = utcnow()
source.last_status = "ok"
source.last_error = None
schedule_next_caldav_sync(source)
if stats is not None and update_stats_tokens:
stats.sync_token = source.sync_token
stats.ctag = source.ctag
if mark_calendar is not None and source.calendar is not None:
mark_calendar(source.calendar, source)
session.flush()
def _finalize_sync_error(session: Session, *, source: CalendarSyncSource, error: BaseException) -> None:
source.last_attempt_at = utcnow()
source.last_status = "error"
source.last_error = str(error)
schedule_next_caldav_sync(source)
session.flush()
def ensure_default_calendar(session: Session, *, tenant_id: str, user_id: str | None = None) -> CalendarCollection:
existing = (
session.query(CalendarCollection)
@@ -265,10 +367,10 @@ def update_calendar(session: Session, *, tenant_id: str, calendar_id: str, paylo
calendar.is_default = True
elif payload.is_default is False:
calendar.is_default = False
for field in ("name", "description", "timezone", "color", "visibility"):
value = getattr(payload, field)
for attr in ("name", "description", "timezone", "color", "visibility"):
value = getattr(payload, attr)
if value is not None:
setattr(calendar, field, value)
setattr(calendar, attr, value)
if payload.metadata is not None:
calendar.metadata_ = payload.metadata
session.flush()
@@ -753,7 +855,7 @@ def http_request(
url = validate_http_url(url, label="Calendar source URL")
request = urllib.request.Request(url, data=data, method=method, headers=headers or {})
try:
with urllib.request.urlopen(request, timeout=timeout) as response: # noqa: S310 - validated admin-configured sync source URL. # nosemgrep: python.lang.security.audit.dynamic-urllib-use-detected.dynamic-urllib-use-detected
with urllib.request.urlopen(request, timeout=timeout) as response: # noqa: S310 - validated admin-configured sync source URL. # nosec B310 # nosemgrep: python.lang.security.audit.dynamic-urllib-use-detected.dynamic-urllib-use-detected
response_headers = {key.lower(): value for key, value in response.headers.items()}
payload = response.read().decode(response_headers.get("content-charset") or "utf-8", errors="replace")
return int(response.status), response_headers, payload
@@ -857,21 +959,16 @@ def sync_caldav_source(
apply_caldav_report(session, source=source, client=client, report=report, stats=stats, user_id=user_id)
source.sync_token = report.sync_token or collection_props.sync_token or source.sync_token
source.ctag = report.ctag or collection_props.ctag or source.ctag
source.last_synced_at = utcnow()
source.last_status = "ok"
source.last_error = None
schedule_next_caldav_sync(source)
stats.sync_token = source.sync_token
stats.ctag = source.ctag
mark_calendar_caldav(source.calendar, source)
session.flush()
_finalize_sync_success(
session,
source=source,
stats=stats,
mark_calendar=mark_calendar_caldav,
update_stats_tokens=True,
)
return source, stats
except Exception as exc:
source.last_attempt_at = utcnow()
source.last_status = "error"
source.last_error = str(exc)
schedule_next_caldav_sync(source)
session.flush()
_finalize_sync_error(session, source=source, error=exc)
raise
@@ -900,11 +997,7 @@ def sync_ics_source(
try:
status_code, response_headers, body = http_request(source.collection_url, headers=headers)
if status_code == 304:
source.last_synced_at = utcnow()
source.last_status = "ok"
source.last_error = None
schedule_next_caldav_sync(source)
session.flush()
_finalize_sync_success(session, source=source, stats=stats)
return source, stats
stats.fetched = 1
stats.created, stats.updated, stats.unchanged, stats.deleted = import_ics_subscription(
@@ -919,19 +1012,10 @@ def sync_ics_source(
metadata["last_modified"] = response_headers["last-modified"]
source.metadata_ = metadata
source.ctag = response_headers.get("etag") or source.ctag
source.last_synced_at = utcnow()
source.last_status = "ok"
source.last_error = None
schedule_next_caldav_sync(source)
mark_calendar_sync_source(source.calendar, source)
session.flush()
_finalize_sync_success(session, source=source, stats=stats, mark_calendar=mark_calendar_sync_source)
return source, stats
except Exception as exc:
source.last_attempt_at = utcnow()
source.last_status = "error"
source.last_error = str(exc)
schedule_next_caldav_sync(source)
session.flush()
_finalize_sync_error(session, source=source, error=exc)
raise
@@ -1331,24 +1415,49 @@ def soft_delete_source_href(session: Session, *, source: CalendarSyncSource, hre
return count
def soft_delete_unseen_source_events(session: Session, *, source: CalendarSyncSource, seen_hrefs: set[str]) -> int:
def soft_delete_unseen_source_events(
session: Session,
*,
source: CalendarSyncSource,
seen_hrefs: set[str],
source_kind: str | None = None,
href_prefix: str | None = None,
user_id: str | None = None,
batch_size: int = SOURCE_EVENT_CLEANUP_BATCH_SIZE,
) -> int:
deleted_at = utcnow()
count = 0
for event in (
session.query(CalendarEvent)
.filter(
last_id: str | None = None
effective_source_kind = source_kind or source.source_kind
effective_batch_size = max(1, batch_size)
while True:
id_query = session.query(CalendarEvent.id).filter(
CalendarEvent.tenant_id == source.tenant_id,
CalendarEvent.calendar_id == source.calendar_id,
CalendarEvent.source_kind == source.source_kind,
CalendarEvent.source_kind == effective_source_kind,
CalendarEvent.deleted_at.is_(None),
)
.all()
):
if event.source_href not in seen_hrefs:
previous = calendar_event_change_payload(event, prefix="previous_")
event.deleted_at = deleted_at
record_calendar_event_change(session, event=event, operation="deleted", user_id=None, previous=previous)
count += 1
if href_prefix:
id_query = id_query.filter(CalendarEvent.source_href.like(f"{href_prefix}%"))
if last_id:
id_query = id_query.filter(CalendarEvent.id > last_id)
ids = [row[0] for row in id_query.order_by(CalendarEvent.id.asc()).limit(effective_batch_size).all()]
if not ids:
break
last_id = ids[-1]
events = (
session.query(CalendarEvent)
.filter(CalendarEvent.id.in_(ids))
.order_by(CalendarEvent.id.asc())
.all()
)
for event in events:
if event.source_href not in seen_hrefs:
previous = calendar_event_change_payload(event, prefix="previous_")
event.deleted_at = deleted_at
record_calendar_event_change(session, event=event, operation="deleted", user_id=user_id, previous=previous)
count += 1
session.flush()
return count
@@ -1438,12 +1547,12 @@ def parse_ews_datetime(value: str | None) -> datetime:
return normalize_datetime(datetime.fromisoformat(value.replace("Z", "+00:00")))
def text_of(item: ET.Element, name: str) -> str | None:
def text_of(item: Any, name: str) -> str | None:
child = item.find(f"./{{{EWS_TYPES_NS}}}{name}")
return child.text if child is not None else None
def ews_mailbox_record(mailbox: ET.Element | None) -> dict[str, Any] | None:
def ews_mailbox_record(mailbox: Any | None) -> dict[str, Any] | None:
if mailbox is None:
return None
return {
@@ -1453,7 +1562,7 @@ def ews_mailbox_record(mailbox: ET.Element | None) -> dict[str, Any] | None:
}
def ews_attendees(item: ET.Element) -> list[dict[str, Any]]:
def ews_attendees(item: Any) -> list[dict[str, Any]]:
attendees: list[dict[str, Any]] = []
for role, path in (("required", "RequiredAttendees"), ("optional", "OptionalAttendees")):
for attendee in item.findall(f"./{{{EWS_TYPES_NS}}}{path}/{{{EWS_TYPES_NS}}}Attendee"):
@@ -1467,14 +1576,14 @@ def ews_attendees(item: ET.Element) -> list[dict[str, Any]]:
return attendees
def ews_reminders(item: ET.Element) -> list[dict[str, Any]]:
def ews_reminders(item: Any) -> list[dict[str, Any]]:
if text_of(item, "ReminderIsSet") == "true":
minutes = int_or_default(text_of(item, "ReminderMinutesBeforeStart"), 15)
return [{"action": "DISPLAY", "trigger_minutes_before": minutes}]
return []
def element_to_dict(element: ET.Element) -> dict[str, Any]:
def element_to_dict(element: Any) -> dict[str, Any]:
tag = element.tag.rsplit("}", 1)[-1]
children = list(element)
result: dict[str, Any] = {"tag": tag}
@@ -1507,35 +1616,7 @@ def sync_due_sources(
now: datetime | None = None,
client_factory: Callable[[CalendarSyncSource], CalDAVClient] | None = None,
) -> list[CalendarCalDavDueSyncResult]:
due_at = normalize_datetime(now or utcnow())
query = session.query(CalendarSyncSource).filter(
CalendarSyncSource.source_kind.in_(SYNC_SOURCE_KINDS),
CalendarSyncSource.sync_enabled.is_(True),
CalendarSyncSource.deleted_at.is_(None),
or_(CalendarSyncSource.next_sync_at.is_(None), CalendarSyncSource.next_sync_at <= due_at),
)
if tenant_id is not None:
query = query.filter(CalendarSyncSource.tenant_id == tenant_id)
sources = query.order_by(CalendarSyncSource.next_sync_at.asc(), CalendarSyncSource.created_at.asc()).limit(limit).all()
results: list[CalendarCalDavDueSyncResult] = []
for source in sources:
try:
if source.source_kind == "caldav":
client = client_factory(source) if client_factory else None
refreshed, stats = sync_caldav_source(
session,
tenant_id=source.tenant_id,
user_id=user_id,
source_id=source.id,
client=client,
)
else:
refreshed, stats = sync_source(session, tenant_id=source.tenant_id, user_id=user_id, source_id=source.id)
results.append(CalendarCalDavDueSyncResult(source_id=refreshed.id, calendar_id=refreshed.calendar_id, status="ok", stats=stats))
except Exception as exc:
results.append(CalendarCalDavDueSyncResult(source_id=source.id, calendar_id=source.calendar_id, status="error", error=str(exc)))
session.flush()
return results
return _sync_due_sources(session, source_kinds=SYNC_SOURCE_KINDS, **_due_sync_options(tenant_id, user_id, limit, now, client_factory))
def sync_due_caldav_sources(
@@ -1546,10 +1627,39 @@ def sync_due_caldav_sources(
limit: int = 50,
now: datetime | None = None,
client_factory: Callable[[CalendarSyncSource], CalDAVClient] | None = None,
) -> list[CalendarCalDavDueSyncResult]:
return _sync_due_sources(session, source_kinds={"caldav"}, **_due_sync_options(tenant_id, user_id, limit, now, client_factory))
def _due_sync_options(
tenant_id: str | None,
user_id: str | None,
limit: int,
now: datetime | None,
client_factory: Callable[[CalendarSyncSource], CalDAVClient] | None,
) -> dict[str, object]:
return {
"tenant_id": tenant_id,
"user_id": user_id,
"limit": limit,
"now": now,
"client_factory": client_factory,
}
def _sync_due_sources(
session: Session,
*,
tenant_id: str | None,
user_id: str | None,
limit: int,
now: datetime | None,
source_kinds: set[str],
client_factory: Callable[[CalendarSyncSource], CalDAVClient] | None,
) -> list[CalendarCalDavDueSyncResult]:
due_at = normalize_datetime(now or utcnow())
query = session.query(CalendarSyncSource).filter(
CalendarSyncSource.source_kind == "caldav",
CalendarSyncSource.source_kind.in_(source_kinds),
CalendarSyncSource.sync_enabled.is_(True),
CalendarSyncSource.deleted_at.is_(None),
or_(CalendarSyncSource.next_sync_at.is_(None), CalendarSyncSource.next_sync_at <= due_at),
@@ -1559,16 +1669,31 @@ def sync_due_caldav_sources(
sources = query.order_by(CalendarSyncSource.next_sync_at.asc(), CalendarSyncSource.created_at.asc()).limit(limit).all()
results: list[CalendarCalDavDueSyncResult] = []
for source in sources:
previous_status = source.last_status
try:
client = client_factory(source) if client_factory else None
refreshed, stats = sync_caldav_source(session, tenant_id=source.tenant_id, user_id=user_id, source_id=source.id, client=client)
refreshed, stats = _sync_due_source(session, source=source, user_id=user_id, client_factory=client_factory)
results.append(CalendarCalDavDueSyncResult(source_id=refreshed.id, calendar_id=refreshed.calendar_id, status="ok", stats=stats))
_emit_calendar_sync_notification(session, source=refreshed, status="ok", previous_status=previous_status, stats=stats)
except Exception as exc:
results.append(CalendarCalDavDueSyncResult(source_id=source.id, calendar_id=source.calendar_id, status="error", error=str(exc)))
_emit_calendar_sync_notification(session, source=source, status="error", previous_status=previous_status, error=str(exc))
session.flush()
return results
def _sync_due_source(
session: Session,
*,
source: CalendarSyncSource,
user_id: str | None,
client_factory: Callable[[CalendarSyncSource], CalDAVClient] | None,
) -> tuple[CalendarSyncSource, CalendarCalDavSyncStats]:
if source.source_kind == "caldav":
client = client_factory(source) if client_factory else None
return sync_caldav_source(session, tenant_id=source.tenant_id, user_id=user_id, source_id=source.id, client=client)
return sync_source(session, tenant_id=source.tenant_id, user_id=user_id, source_id=source.id)
def schedule_next_caldav_sync(source: CalendarSyncSource, *, now: datetime | None = None) -> None:
if not source.sync_enabled:
source.next_sync_at = None
@@ -1616,24 +1741,14 @@ def apply_caldav_report(
stats.deleted += deleted
if stats.full_sync:
query = (
session.query(CalendarEvent)
.filter(
CalendarEvent.tenant_id == source.tenant_id,
CalendarEvent.calendar_id == source.calendar_id,
CalendarEvent.source_kind == "caldav",
CalendarEvent.deleted_at.is_(None),
CalendarEvent.source_href.like(f"{source.collection_url}%"),
)
.all()
stats.deleted += soft_delete_unseen_source_events(
session,
source=source,
seen_hrefs=seen_hrefs,
source_kind="caldav",
href_prefix=source.collection_url,
user_id=user_id,
)
deleted_at = utcnow()
for event in query:
if event.source_href not in seen_hrefs:
previous = calendar_event_change_payload(event, prefix="previous_")
event.deleted_at = deleted_at
record_calendar_event_change(session, event=event, operation="deleted", user_id=user_id, previous=previous)
stats.deleted += 1
def import_caldav_resource(
@@ -2018,10 +2133,10 @@ def update_event(session: Session, *, tenant_id: str, user_id: str | None, event
"etag",
"icalendar",
)
for field in scalar_fields:
value = getattr(payload, field)
for attr in scalar_fields:
value = getattr(payload, attr)
if value is not None:
setattr(event, field, value.upper() if field in {"status", "transparency", "classification"} and isinstance(value, str) else value)
setattr(event, attr, value.upper() if attr in {"status", "transparency", "classification"} and isinstance(value, str) else value)
if payload.start_at is not None:
event.start_at = normalize_datetime(payload.start_at)
if "end_at" in payload.model_fields_set: