GovOPlaN/govoplan-dataflow
2
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

Fence and reconcile Dataflow runs

Browse Source
This commit is contained in:
zemion
2026-08-03 06:09:53 +02:00
parent e1c092ece7
commit 3fa7a29f48
13 changed files with 1314 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/govoplan_dataflow/backend/run_worker.py
+129 -3
View File
@@ -25,6 +25,13 @@ from govoplan_dataflow.backend.db.models import (
DataflowTrigger,
)
from govoplan_dataflow.backend.governance import require_definition_action
from govoplan_dataflow.backend.recovery import (
DataflowRecoveryError,
DataflowRunRecovery,
begin_dataflow_run_recovery,
claim_stale_dataflow_recovery,
dataflow_run_recovery_state,
)
from govoplan_dataflow.backend.service import (
_execute_pipeline_run,
_require_deployed_revision,
@@ -55,7 +62,10 @@ def dispatch_pending_runs(
if worker_id and str(worker_id).strip()
else socket.gethostname()
)
recovered = _recover_expired_leases(session, now=current)
recovered, recovered_outcome_unknown = _recover_expired_leases(
session,
now=current,
)
session.commit()
summary: dict[str, object] = {
"claimed": 0,
@@ -63,6 +73,7 @@ def dispatch_pending_runs(
"retrying": 0,
"failed": 0,
"cancelled": 0,
"outcome_unknown": recovered_outcome_unknown,
"recovered": recovered,
"runs": [],
}
@@ -193,7 +204,7 @@ def _recover_expired_leases(
session: Session,
*,
now: datetime,
) -> int:
) -> tuple[int, int]:
runs = list(
session.scalars(
select(DataflowRun)
@@ -205,13 +216,46 @@ def _recover_expired_leases(
.with_for_update(skip_locked=True)
)
)
recovered = 0
outcome_unknown = 0
for run in runs:
recovery = dataflow_run_recovery_state(session, run_id=run.id)
recovery_status: str | None = None
recovery_metadata = dict(run.authorization_).get("recovery")
effect_started = bool(
isinstance(recovery_metadata, Mapping)
and recovery_metadata.get("boundary") == "output-publication"
)
if recovery is not None and recovery.get("operation_id"):
try:
recovery_status = claim_stale_dataflow_recovery(
session,
operation_id=str(recovery["operation_id"]),
lease_ttl_seconds=DEFAULT_LEASE_SECONDS,
effect_started=effect_started,
)
except DataflowRecoveryError:
logger.warning(
"Could not claim stale Dataflow recovery operation %s",
recovery["operation_id"],
exc_info=True,
)
continue
run.worker_id = None
run.claimed_at = None
run.lease_expires_at = None
run.heartbeat_at = None
if run.cancellation_requested_at is not None:
_cancel_run(run, now=now)
elif recovery_status == "outcome_unknown" or effect_started:
run.status = "outcome_unknown"
run.finished_at = now
run.progress_phase = "outcome_unknown"
run.error = (
"The worker lease expired after output publication may have "
"started. Verify the sink before retrying."
)
outcome_unknown += 1
elif run.attempts < run.max_attempts:
run.status = "retrying"
run.available_at = now
@@ -223,7 +267,8 @@ def _recover_expired_leases(
now=now,
message="The worker lease expired after the final attempt.",
)
return len(runs)
recovered += 1
return recovered, outcome_unknown
def _claim_next_run(
@@ -291,6 +336,7 @@ def _execute_claimed_run(
if run.cancellation_requested_at is not None:
_cancel_run(run, now=now)
return "cancelled"
recovery: DataflowRunRecovery | None = None
try:
principal, provenance = _resolve_principal(
session,
@@ -329,6 +375,16 @@ def _execute_claimed_run(
revision=revision,
environment=run.environment,
)
session.commit()
recovery = begin_dataflow_run_recovery(
session,
run=run,
lease_ttl_seconds=_lease_seconds(run),
)
if recovery.replayed:
session.rollback()
run = session.get(DataflowRun, run_id)
return run.status if run is not None else "failed"
_notify_run(
session,
registry=registry,
@@ -344,6 +400,7 @@ def _execute_claimed_run(
request=pipeline_run_request(run),
principal=principal,
registry=registry,
recovery=recovery,
)
if (
run.status == "failed"
@@ -359,7 +416,47 @@ def _execute_claimed_run(
run=run,
pipeline_name=pipeline.name,
)
recovery.finish(session, run=run)
return run.status
except DataflowRecoveryError:
logger.exception(
"Dataflow recovery finalization failed for run %s",
run_id,
)
session.rollback()
run = session.get(DataflowRun, run_id)
if run is not None:
if recovery is not None and recovery.publication_started:
run.status = "outcome_unknown"
run.finished_at = now
run.progress_phase = "outcome_unknown"
run.error = (
"Output publication completed without verifiable terminal "
"recovery evidence; reconcile the sink."
)
_finish_claim(run)
return "outcome_unknown"
if recovery is None:
_fail_run(
run,
now=now,
message=(
"The recovery ledger was unavailable; no Dataflow "
"execution started."
),
)
_finish_claim(run)
else:
_fail_run(
run,
now=now,
message=(
"The database-only run could not commit verifiable "
"recovery evidence."
),
)
_finish_claim(run)
return "failed"
except (DataflowWorkerError, PermissionError, ValueError) as exc:
_fail_run(run, now=now, message=str(exc))
_finish_claim(run)
@@ -370,6 +467,8 @@ def _execute_claimed_run(
run=run,
pipeline_name="Dataflow",
)
if recovery is not None:
recovery.finish(session, run=run)
return "failed"
except Exception as exc:
logger.exception(
@@ -380,6 +479,24 @@ def _execute_claimed_run(
run = session.get(DataflowRun, run_id)
if run is None:
return "failed"
if recovery is not None and recovery.publication_started:
run.status = "outcome_unknown"
run.finished_at = now
run.error = (
"The output provider failed after dispatch began; verify the "
"sink before retrying."
)
run.progress_phase = "outcome_unknown"
_finish_claim(run)
try:
recovery.finish(session, run=run)
except DataflowRecoveryError:
logger.exception(
"Could not persist the uncertain output state for run %s",
run_id,
)
session.rollback()
return "outcome_unknown"
_fail_run(
run,
now=now,
@@ -389,6 +506,15 @@ def _execute_claimed_run(
),
)
_finish_claim(run)
if recovery is not None:
try:
recovery.finish(session, run=run)
except DataflowRecoveryError:
logger.exception(
"Could not finalize failed Dataflow recovery for run %s",
run_id,
)
session.rollback()
return "failed"