From d387769453834092ef61ab4e6bfdfd9ac7dd0a76 Mon Sep 17 00:00:00 2001 From: Robert Wojciechowski Date: Sun, 8 Mar 2026 03:11:50 +0000 Subject: [PATCH] Skip heartbeat timeout check for workers with no incomplete tasks Prevents false "heartbeat timed out" warnings when workers finish their work before the master completes local tile processing. --- upscale/job_timeout.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/upscale/job_timeout.py b/upscale/job_timeout.py index 0bc4d5d..7d4515e 100644 --- a/upscale/job_timeout.py +++ b/upscale/job_timeout.py @@ -99,6 +99,10 @@ async def _check_and_requeue_timed_out_workers(multi_job_id, total_tasks): f"Probe diagnostics: online={probe_queue is not None} queue_remaining={probe_queue}" ) + if incomplete_assigned == 0: + debug_log(f"Worker {worker} heartbeat stale but all tasks complete; skipping") + continue + if busy: workers_graced.append(worker) debug_log(f"Heartbeat grace: worker {worker} busy via probe; skipping requeue")